chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,27 @@
|
||||
# Tests
|
||||
|
||||
Before running any tests, make sure you have `uv` installed (and ideally run `make sync` after).
|
||||
|
||||
## Running tests
|
||||
|
||||
```
|
||||
make tests
|
||||
```
|
||||
|
||||
`make tests` runs the shard-safe suite in parallel and then runs tests marked `serial` in a separate serial pass.
|
||||
|
||||
## Snapshots
|
||||
|
||||
We use [inline-snapshots](https://15r10nk.github.io/inline-snapshot/latest/) for some tests. If your code adds new snapshot tests or breaks existing ones, you can fix/create them. After fixing/creating snapshots, run `make tests` again to verify the tests pass.
|
||||
|
||||
### Fixing snapshots
|
||||
|
||||
```
|
||||
make snapshots-fix
|
||||
```
|
||||
|
||||
### Creating snapshots
|
||||
|
||||
```
|
||||
make snapshots-create
|
||||
```
|
||||
@@ -0,0 +1,108 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import shlex
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import PurePosixPath
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FakeResolveWorkspaceResult:
|
||||
exit_code: int
|
||||
stdout: str = ""
|
||||
stderr: str = ""
|
||||
|
||||
|
||||
def resolve_fake_workspace_path(
|
||||
command: str | Sequence[str],
|
||||
*,
|
||||
symlinks: dict[str, str],
|
||||
home_dir: str,
|
||||
) -> FakeResolveWorkspaceResult | None:
|
||||
tokens = shlex.split(command) if isinstance(command, str) else list(command)
|
||||
helper_index = next(
|
||||
(
|
||||
index
|
||||
for index, token in enumerate(tokens)
|
||||
if token.startswith("/tmp/openai-agents/bin/resolve-workspace-path-")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if helper_index is None or len(tokens) < helper_index + 4:
|
||||
return None
|
||||
|
||||
root = _resolve_fake_path(tokens[helper_index + 1], symlinks=symlinks, home_dir=home_dir)
|
||||
candidate = _resolve_fake_path(tokens[helper_index + 2], symlinks=symlinks, home_dir=home_dir)
|
||||
for_write = tokens[helper_index + 3]
|
||||
grant_tokens = tokens[helper_index + 4 :]
|
||||
|
||||
if _fake_path_is_under(candidate, root):
|
||||
return FakeResolveWorkspaceResult(exit_code=0, stdout=candidate.as_posix())
|
||||
|
||||
best_grant: tuple[PurePosixPath, str, str] | None = None
|
||||
for index in range(0, len(grant_tokens), 2):
|
||||
grant_original = grant_tokens[index]
|
||||
read_only = grant_tokens[index + 1]
|
||||
grant_root = _resolve_fake_path(grant_original, symlinks=symlinks, home_dir=home_dir)
|
||||
if not _fake_path_is_under(candidate, grant_root):
|
||||
continue
|
||||
if best_grant is None or len(grant_root.parts) > len(best_grant[0].parts):
|
||||
best_grant = (grant_root, grant_original, read_only)
|
||||
|
||||
if best_grant is not None:
|
||||
_grant_root, grant_original, read_only = best_grant
|
||||
if for_write == "1" and read_only == "1":
|
||||
return FakeResolveWorkspaceResult(
|
||||
exit_code=114,
|
||||
stderr=(
|
||||
f"read-only extra path grant: {grant_original}\n"
|
||||
f"resolved path: {candidate.as_posix()}\n"
|
||||
),
|
||||
)
|
||||
return FakeResolveWorkspaceResult(exit_code=0, stdout=candidate.as_posix())
|
||||
|
||||
return FakeResolveWorkspaceResult(
|
||||
exit_code=111,
|
||||
stderr=f"workspace escape: {candidate.as_posix()}\n",
|
||||
)
|
||||
|
||||
|
||||
def _resolve_fake_path(
|
||||
raw_path: str,
|
||||
*,
|
||||
symlinks: dict[str, str],
|
||||
home_dir: str,
|
||||
depth: int = 0,
|
||||
) -> PurePosixPath:
|
||||
if depth > 64:
|
||||
raise RuntimeError(f"symlink resolution depth exceeded: {raw_path}")
|
||||
|
||||
path = PurePosixPath(raw_path)
|
||||
if not path.is_absolute():
|
||||
path = PurePosixPath(home_dir) / path
|
||||
|
||||
parts = path.parts
|
||||
current = PurePosixPath("/")
|
||||
for index, part in enumerate(parts[1:], start=1):
|
||||
current = current / part
|
||||
target = symlinks.get(current.as_posix())
|
||||
if target is None:
|
||||
continue
|
||||
|
||||
target_path = PurePosixPath(target)
|
||||
if not target_path.is_absolute():
|
||||
target_path = current.parent / target_path
|
||||
for remaining in parts[index + 1 :]:
|
||||
target_path /= remaining
|
||||
return _resolve_fake_path(
|
||||
target_path.as_posix(),
|
||||
symlinks=symlinks,
|
||||
home_dir=home_dir,
|
||||
depth=depth + 1,
|
||||
)
|
||||
|
||||
return path
|
||||
|
||||
|
||||
def _fake_path_is_under(path: PurePosixPath, root: PurePosixPath) -> bool:
|
||||
return path == root or root in path.parents
|
||||
@@ -0,0 +1,93 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from agents.models import _openai_shared
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from agents.models.openai_responses import OpenAIResponsesModel
|
||||
from agents.run import set_default_agent_runner
|
||||
from agents.tracing.provider import DefaultTraceProvider
|
||||
from agents.tracing.setup import set_trace_provider
|
||||
|
||||
from .testing_processor import SPAN_PROCESSOR_TESTING
|
||||
|
||||
collect_ignore: list[str] = []
|
||||
|
||||
if sys.platform == "win32":
|
||||
collect_ignore.extend(
|
||||
[
|
||||
"test_example_workflows.py",
|
||||
"test_run_state.py",
|
||||
"sandbox/capabilities/test_filesystem_capability.py",
|
||||
"sandbox/integration_tests/test_runner_pause_resume.py",
|
||||
"sandbox/test_client_options.py",
|
||||
"sandbox/test_exposed_ports.py",
|
||||
"sandbox/test_extract.py",
|
||||
"sandbox/test_memory.py",
|
||||
"sandbox/test_runtime.py",
|
||||
"sandbox/test_session_manager.py",
|
||||
"sandbox/test_session_sinks.py",
|
||||
"sandbox/test_snapshot.py",
|
||||
"sandbox/test_unix_local.py",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
# This fixture will run once before any tests are executed
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def setup_span_processor():
|
||||
provider = DefaultTraceProvider()
|
||||
provider.set_processors([SPAN_PROCESSOR_TESTING])
|
||||
set_trace_provider(provider)
|
||||
yield
|
||||
provider.shutdown()
|
||||
|
||||
|
||||
# Ensure a default OpenAI API key is present for tests that construct clients
|
||||
# without explicitly configuring a key/client. Tests that need no key use
|
||||
# monkeypatch.delenv("OPENAI_API_KEY", ...) to remove it locally.
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def ensure_openai_api_key():
|
||||
import os
|
||||
|
||||
if not os.environ.get("OPENAI_API_KEY"):
|
||||
os.environ["OPENAI_API_KEY"] = "test_key"
|
||||
|
||||
|
||||
# This fixture will run before each test
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_span_processor():
|
||||
SPAN_PROCESSOR_TESTING.force_flush()
|
||||
SPAN_PROCESSOR_TESTING.shutdown()
|
||||
SPAN_PROCESSOR_TESTING.clear()
|
||||
|
||||
|
||||
# This fixture will run before each test
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_openai_settings():
|
||||
_openai_shared._default_openai_key = None
|
||||
_openai_shared._default_openai_client = None
|
||||
_openai_shared._use_responses_by_default = True
|
||||
_openai_shared.set_default_openai_responses_transport("http")
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clear_default_runner():
|
||||
set_default_agent_runner(None)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def disable_real_model_clients(monkeypatch, request):
|
||||
# If the test is marked to allow the method call, don't override it.
|
||||
if request.node.get_closest_marker("allow_call_model_methods"):
|
||||
return
|
||||
|
||||
def failing_version(*args, **kwargs):
|
||||
pytest.fail("Real models should not be used in tests!")
|
||||
|
||||
monkeypatch.setattr(OpenAIResponsesModel, "get_response", failing_version)
|
||||
monkeypatch.setattr(OpenAIResponsesModel, "stream_response", failing_version)
|
||||
monkeypatch.setattr(OpenAIChatCompletionsModel, "get_response", failing_version)
|
||||
monkeypatch.setattr(OpenAIChatCompletionsModel, "stream_response", failing_version)
|
||||
@@ -0,0 +1,757 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import importlib
|
||||
import inspect
|
||||
import json
|
||||
import os
|
||||
from dataclasses import fields
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from agents.exceptions import UserError
|
||||
from agents.extensions.experimental.codex import Usage
|
||||
from agents.extensions.experimental.codex.codex import Codex, _normalize_env
|
||||
from agents.extensions.experimental.codex.codex_options import CodexOptions, coerce_codex_options
|
||||
from agents.extensions.experimental.codex.exec import CodexExec
|
||||
from agents.extensions.experimental.codex.output_schema_file import (
|
||||
OutputSchemaFile,
|
||||
create_output_schema_file,
|
||||
)
|
||||
from agents.extensions.experimental.codex.thread import Thread, _normalize_input
|
||||
from agents.extensions.experimental.codex.thread_options import ThreadOptions, coerce_thread_options
|
||||
from agents.extensions.experimental.codex.turn_options import TurnOptions
|
||||
|
||||
exec_module = importlib.import_module("agents.extensions.experimental.codex.exec")
|
||||
thread_module = importlib.import_module("agents.extensions.experimental.codex.thread")
|
||||
output_schema_module = importlib.import_module(
|
||||
"agents.extensions.experimental.codex.output_schema_file"
|
||||
)
|
||||
|
||||
|
||||
class FakeStdin:
|
||||
def __init__(self) -> None:
|
||||
self.buffer = b""
|
||||
self.closed = False
|
||||
|
||||
def write(self, data: bytes) -> None:
|
||||
self.buffer += data
|
||||
|
||||
async def drain(self) -> None:
|
||||
return None
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
class FakeStdout:
|
||||
def __init__(self, lines: list[str]) -> None:
|
||||
self._lines = [line.encode("utf-8") for line in lines]
|
||||
|
||||
async def readline(self) -> bytes:
|
||||
if not self._lines:
|
||||
return b""
|
||||
return self._lines.pop(0)
|
||||
|
||||
|
||||
class FakeStderr:
|
||||
def __init__(self, chunks: list[bytes]) -> None:
|
||||
self._chunks = list(chunks)
|
||||
|
||||
async def read(self, _size: int) -> bytes:
|
||||
if not self._chunks:
|
||||
return b""
|
||||
return self._chunks.pop(0)
|
||||
|
||||
|
||||
class FakeProcess:
|
||||
def __init__(
|
||||
self,
|
||||
stdout_lines: list[str],
|
||||
stderr_chunks: list[bytes] | None = None,
|
||||
*,
|
||||
returncode: int | None = 0,
|
||||
stdin_present: bool = True,
|
||||
stdout_present: bool = True,
|
||||
stderr_present: bool = True,
|
||||
) -> None:
|
||||
self.stdin = FakeStdin() if stdin_present else None
|
||||
self.stdout = FakeStdout(stdout_lines) if stdout_present else None
|
||||
self.stderr = FakeStderr(stderr_chunks or []) if stderr_present else None
|
||||
self.returncode = returncode
|
||||
self.killed = False
|
||||
self.terminated = False
|
||||
|
||||
async def wait(self) -> None:
|
||||
if self.returncode is None:
|
||||
self.returncode = 0
|
||||
|
||||
def kill(self) -> None:
|
||||
self.killed = True
|
||||
|
||||
def terminate(self) -> None:
|
||||
self.terminated = True
|
||||
|
||||
|
||||
class FakeExec:
|
||||
def __init__(self, events: list[Any], delay: float = 0.0) -> None:
|
||||
self.events = events
|
||||
self.delay = delay
|
||||
self.last_args: Any = None
|
||||
|
||||
async def run(self, args: Any):
|
||||
self.last_args = args
|
||||
for event in self.events:
|
||||
if self.delay:
|
||||
await asyncio.sleep(self.delay)
|
||||
payload = event if isinstance(event, str) else json.dumps(event)
|
||||
yield payload
|
||||
|
||||
|
||||
def test_output_schema_file_none_schema() -> None:
|
||||
result = create_output_schema_file(None)
|
||||
assert result.schema_path is None
|
||||
result.cleanup()
|
||||
|
||||
|
||||
def test_output_schema_file_rejects_non_object() -> None:
|
||||
with pytest.raises(UserError, match="output_schema must be a plain JSON object"):
|
||||
create_output_schema_file(cast(Any, ["not", "an", "object"]))
|
||||
|
||||
|
||||
def test_output_schema_file_creates_and_cleans() -> None:
|
||||
schema = {"type": "object", "properties": {"foo": {"type": "string"}}}
|
||||
result = create_output_schema_file(schema)
|
||||
assert result.schema_path is not None
|
||||
with open(result.schema_path, encoding="utf-8") as handle:
|
||||
assert json.load(handle) == schema
|
||||
result.cleanup()
|
||||
assert not os.path.exists(result.schema_path)
|
||||
|
||||
|
||||
def test_output_schema_file_cleanup_swallows_rmtree_errors(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
schema = {"type": "object"}
|
||||
called = False
|
||||
|
||||
def bad_rmtree(_path: str, ignore_errors: bool = True) -> None:
|
||||
nonlocal called
|
||||
called = True
|
||||
raise OSError("boom")
|
||||
|
||||
monkeypatch.setattr(output_schema_module.shutil, "rmtree", bad_rmtree)
|
||||
|
||||
result = create_output_schema_file(schema)
|
||||
result.cleanup()
|
||||
|
||||
assert called is True
|
||||
|
||||
|
||||
def test_output_schema_file_cleanup_on_write_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
schema = {"type": "object"}
|
||||
cleanup_called = False
|
||||
|
||||
def bad_dump(*_args: Any, **_kwargs: Any) -> None:
|
||||
raise RuntimeError("boom")
|
||||
|
||||
def fake_rmtree(_path: str, ignore_errors: bool = True) -> None:
|
||||
nonlocal cleanup_called
|
||||
cleanup_called = True
|
||||
|
||||
monkeypatch.setattr(output_schema_module.json, "dump", bad_dump)
|
||||
monkeypatch.setattr(output_schema_module.shutil, "rmtree", fake_rmtree)
|
||||
|
||||
with pytest.raises(RuntimeError, match="boom"):
|
||||
create_output_schema_file(schema)
|
||||
|
||||
assert cleanup_called is True
|
||||
|
||||
|
||||
def test_normalize_input_merges_text_and_images() -> None:
|
||||
prompt, images = _normalize_input(
|
||||
[
|
||||
{"type": "text", "text": "first"},
|
||||
{"type": "local_image", "path": "/tmp/a.png"},
|
||||
{"type": "text", "text": "second"},
|
||||
{"type": "local_image", "path": ""},
|
||||
]
|
||||
)
|
||||
assert prompt == "first\n\nsecond"
|
||||
assert images == ["/tmp/a.png"]
|
||||
|
||||
|
||||
def test_normalize_env_stringifies_values() -> None:
|
||||
env = _normalize_env(CodexOptions(env=cast(dict[str, str], {"FOO": 1, 2: "bar"})))
|
||||
assert env == {"FOO": "1", "2": "bar"}
|
||||
|
||||
|
||||
def test_coerce_codex_options_rejects_unknown_fields() -> None:
|
||||
with pytest.raises(UserError, match="Unknown CodexOptions field"):
|
||||
coerce_codex_options({"unknown": "value"})
|
||||
|
||||
|
||||
def test_coerce_thread_options_rejects_unknown_fields() -> None:
|
||||
with pytest.raises(UserError, match="Unknown ThreadOptions field"):
|
||||
coerce_thread_options({"unknown": "value"})
|
||||
|
||||
|
||||
def test_coerce_thread_options_rejects_non_mapping() -> None:
|
||||
with pytest.raises(UserError, match="ThreadOptions must be a ThreadOptions or a mapping"):
|
||||
coerce_thread_options(cast(Any, ["model", "gpt"]))
|
||||
|
||||
|
||||
def test_codex_start_and_resume_thread() -> None:
|
||||
codex = Codex(CodexOptions(codex_path_override="/bin/codex"))
|
||||
thread = codex.start_thread({"model": "gpt"})
|
||||
assert thread.id is None
|
||||
resumed = codex.resume_thread("thread-1", {"model": "gpt"})
|
||||
assert resumed.id == "thread-1"
|
||||
|
||||
|
||||
def test_codex_init_accepts_mapping_options() -> None:
|
||||
codex = Codex({"codex_path_override": "/bin/codex"})
|
||||
assert codex._exec._executable_path == "/bin/codex"
|
||||
|
||||
|
||||
def test_codex_init_accepts_kwargs() -> None:
|
||||
codex = Codex(codex_path_override="/bin/codex", base_url="https://example.com")
|
||||
assert codex._exec._executable_path == "/bin/codex"
|
||||
assert codex._options.base_url == "https://example.com"
|
||||
|
||||
|
||||
def test_codex_init_accepts_stream_limit_kwarg() -> None:
|
||||
codex = Codex(codex_path_override="/bin/codex", codex_subprocess_stream_limit_bytes=123456)
|
||||
assert codex._exec._subprocess_stream_limit_bytes == 123456
|
||||
|
||||
|
||||
def test_codex_init_rejects_options_and_kwargs() -> None:
|
||||
with pytest.raises(UserError, match="Codex options must be provided"):
|
||||
Codex( # type: ignore[call-overload]
|
||||
cast(Any, CodexOptions()), codex_path_override="/bin/codex"
|
||||
)
|
||||
|
||||
|
||||
def test_codex_init_kw_matches_codex_options() -> None:
|
||||
signature = inspect.signature(Codex.__init__)
|
||||
kw_only = [
|
||||
param.name
|
||||
for param in signature.parameters.values()
|
||||
if param.kind == inspect.Parameter.KEYWORD_ONLY
|
||||
]
|
||||
option_fields = [field.name for field in fields(CodexOptions)]
|
||||
assert kw_only == option_fields
|
||||
|
||||
|
||||
def test_codex_exec_stream_limit_uses_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv(exec_module._SUBPROCESS_STREAM_LIMIT_ENV_VAR, "131072")
|
||||
exec_client = exec_module.CodexExec(executable_path="/bin/codex")
|
||||
assert exec_client._subprocess_stream_limit_bytes == 131072
|
||||
|
||||
|
||||
def test_codex_exec_stream_limit_explicit_overrides_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv(exec_module._SUBPROCESS_STREAM_LIMIT_ENV_VAR, "262144")
|
||||
exec_client = exec_module.CodexExec(
|
||||
executable_path="/bin/codex",
|
||||
subprocess_stream_limit_bytes=524288,
|
||||
)
|
||||
assert exec_client._subprocess_stream_limit_bytes == 524288
|
||||
|
||||
|
||||
def test_codex_exec_stream_limit_rejects_invalid_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv(exec_module._SUBPROCESS_STREAM_LIMIT_ENV_VAR, "not-a-number")
|
||||
with pytest.raises(UserError, match=exec_module._SUBPROCESS_STREAM_LIMIT_ENV_VAR):
|
||||
_ = exec_module.CodexExec(executable_path="/bin/codex")
|
||||
|
||||
|
||||
def test_codex_exec_stream_limit_rejects_out_of_range_value() -> None:
|
||||
with pytest.raises(UserError, match="must be between"):
|
||||
_ = exec_module.CodexExec(
|
||||
executable_path="/bin/codex",
|
||||
subprocess_stream_limit_bytes=1024,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_exec_run_builds_command_args_and_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
captured: dict[str, Any] = {}
|
||||
process = FakeProcess(stdout_lines=["line-1\n", "line-2\n"])
|
||||
|
||||
async def fake_create_subprocess_exec(*args: Any, **kwargs: Any) -> FakeProcess:
|
||||
captured["args"] = args
|
||||
captured["kwargs"] = kwargs
|
||||
return process
|
||||
|
||||
monkeypatch.setattr(exec_module.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
|
||||
|
||||
exec_client = exec_module.CodexExec(executable_path="/bin/codex", env={"FOO": "bar"})
|
||||
args = exec_module.CodexExecArgs(
|
||||
input="hello",
|
||||
base_url="https://example.com",
|
||||
api_key="api-key",
|
||||
thread_id="thread-123",
|
||||
images=["/tmp/img.png"],
|
||||
model="gpt-4.1-mini",
|
||||
sandbox_mode="read-only",
|
||||
working_directory="/work",
|
||||
additional_directories=["/extra-a", "/extra-b"],
|
||||
skip_git_repo_check=True,
|
||||
output_schema_file="/tmp/schema.json",
|
||||
model_reasoning_effort="high",
|
||||
network_access_enabled=True,
|
||||
web_search_mode="live",
|
||||
approval_policy="on-request",
|
||||
)
|
||||
|
||||
output = [line async for line in exec_client.run(args)]
|
||||
|
||||
assert output == ["line-1", "line-2"]
|
||||
assert process.stdin is not None
|
||||
assert process.stdin.buffer == b"hello"
|
||||
assert process.stdin.closed is True
|
||||
|
||||
assert captured["args"][0] == "/bin/codex"
|
||||
assert list(captured["args"][1:]) == [
|
||||
"exec",
|
||||
"--experimental-json",
|
||||
"--model",
|
||||
"gpt-4.1-mini",
|
||||
"--sandbox",
|
||||
"read-only",
|
||||
"--cd",
|
||||
"/work",
|
||||
"--add-dir",
|
||||
"/extra-a",
|
||||
"--add-dir",
|
||||
"/extra-b",
|
||||
"--skip-git-repo-check",
|
||||
"--output-schema",
|
||||
"/tmp/schema.json",
|
||||
"--config",
|
||||
'model_reasoning_effort="high"',
|
||||
"--config",
|
||||
"sandbox_workspace_write.network_access=true",
|
||||
"--config",
|
||||
'web_search="live"',
|
||||
"--config",
|
||||
'approval_policy="on-request"',
|
||||
"resume",
|
||||
"thread-123",
|
||||
"--image",
|
||||
"/tmp/img.png",
|
||||
"-",
|
||||
]
|
||||
|
||||
env = captured["kwargs"]["env"]
|
||||
assert env["FOO"] == "bar"
|
||||
assert env[exec_module._INTERNAL_ORIGINATOR_ENV] == exec_module._TYPESCRIPT_SDK_ORIGINATOR
|
||||
assert env["OPENAI_BASE_URL"] == "https://example.com"
|
||||
assert env["CODEX_API_KEY"] == "api-key"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_exec_run_handles_large_single_line_events(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
captured: dict[str, Any] = {}
|
||||
large_payload = "x" * (2**16 + 1)
|
||||
|
||||
class StreamReaderProcess:
|
||||
def __init__(self, *, line: str, limit: int) -> None:
|
||||
self.stdin = FakeStdin()
|
||||
self.stdout = asyncio.StreamReader(limit=limit)
|
||||
self.stdout.feed_data(f"{line}\n".encode())
|
||||
self.stdout.feed_eof()
|
||||
self.stderr = FakeStderr([])
|
||||
self.returncode: int | None = 0
|
||||
self.killed = False
|
||||
self.terminated = False
|
||||
|
||||
async def wait(self) -> None:
|
||||
if self.returncode is None:
|
||||
self.returncode = 0
|
||||
|
||||
def kill(self) -> None:
|
||||
self.killed = True
|
||||
|
||||
def terminate(self) -> None:
|
||||
self.terminated = True
|
||||
|
||||
async def fake_create_subprocess_exec(*_args: Any, **kwargs: Any) -> StreamReaderProcess:
|
||||
captured["kwargs"] = kwargs
|
||||
return StreamReaderProcess(line=large_payload, limit=kwargs["limit"])
|
||||
|
||||
monkeypatch.setattr(exec_module.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
|
||||
|
||||
exec_client = exec_module.CodexExec(executable_path="/bin/codex")
|
||||
output = [line async for line in exec_client.run(exec_module.CodexExecArgs(input="hello"))]
|
||||
|
||||
assert output == [large_payload]
|
||||
assert captured["kwargs"]["limit"] == exec_module._DEFAULT_SUBPROCESS_STREAM_LIMIT_BYTES
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("enabled", "expected_config"),
|
||||
[
|
||||
(True, 'web_search="live"'),
|
||||
(False, 'web_search="disabled"'),
|
||||
],
|
||||
)
|
||||
async def test_codex_exec_run_web_search_enabled_flags(
|
||||
monkeypatch: pytest.MonkeyPatch, enabled: bool, expected_config: str
|
||||
) -> None:
|
||||
captured: dict[str, Any] = {}
|
||||
process = FakeProcess(stdout_lines=[])
|
||||
|
||||
async def fake_create_subprocess_exec(*args: Any, **kwargs: Any) -> FakeProcess:
|
||||
captured["args"] = args
|
||||
return process
|
||||
|
||||
monkeypatch.setattr(exec_module.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
|
||||
|
||||
exec_client = exec_module.CodexExec(executable_path="/bin/codex")
|
||||
args = exec_module.CodexExecArgs(input="hello", web_search_enabled=enabled)
|
||||
|
||||
_ = [line async for line in exec_client.run(args)]
|
||||
command_args = list(captured["args"][1:])
|
||||
assert "--config" in command_args
|
||||
assert expected_config in command_args
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_exec_run_raises_on_non_zero_exit(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
process = FakeProcess(stdout_lines=[], stderr_chunks=[b"bad"], returncode=2)
|
||||
|
||||
async def fake_create_subprocess_exec(*args: Any, **kwargs: Any) -> FakeProcess:
|
||||
return process
|
||||
|
||||
monkeypatch.setattr(exec_module.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
|
||||
|
||||
exec_client = exec_module.CodexExec(executable_path="/bin/codex")
|
||||
args = exec_module.CodexExecArgs(input="hello")
|
||||
|
||||
with pytest.raises(RuntimeError, match="exited with code 2"):
|
||||
async for _ in exec_client.run(args):
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_exec_run_raises_without_stdin(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
process = FakeProcess(stdout_lines=[], stdin_present=False)
|
||||
|
||||
async def fake_create_subprocess_exec(*args: Any, **kwargs: Any) -> FakeProcess:
|
||||
return process
|
||||
|
||||
monkeypatch.setattr(exec_module.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
|
||||
|
||||
exec_client = exec_module.CodexExec(executable_path="/bin/codex")
|
||||
args = exec_module.CodexExecArgs(input="hello")
|
||||
|
||||
with pytest.raises(RuntimeError, match="no stdin"):
|
||||
async for _ in exec_client.run(args):
|
||||
pass
|
||||
assert process.killed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_exec_run_raises_without_stdout(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
process = FakeProcess(stdout_lines=[], stdout_present=False)
|
||||
|
||||
async def fake_create_subprocess_exec(*args: Any, **kwargs: Any) -> FakeProcess:
|
||||
return process
|
||||
|
||||
monkeypatch.setattr(exec_module.asyncio, "create_subprocess_exec", fake_create_subprocess_exec)
|
||||
|
||||
exec_client = exec_module.CodexExec(executable_path="/bin/codex")
|
||||
args = exec_module.CodexExecArgs(input="hello")
|
||||
|
||||
with pytest.raises(RuntimeError, match="no stdout"):
|
||||
async for _ in exec_client.run(args):
|
||||
pass
|
||||
assert process.killed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_watch_signal_terminates_process() -> None:
|
||||
signal = asyncio.Event()
|
||||
process = FakeProcess(stdout_lines=[], returncode=None)
|
||||
|
||||
task = asyncio.create_task(exec_module._watch_signal(signal, process))
|
||||
signal.set()
|
||||
await task
|
||||
|
||||
assert process.terminated is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("system", "arch", "expected"),
|
||||
[
|
||||
("linux", "x86_64", "x86_64-unknown-linux-musl"),
|
||||
("linux", "aarch64", "aarch64-unknown-linux-musl"),
|
||||
("darwin", "x86_64", "x86_64-apple-darwin"),
|
||||
("darwin", "arm64", "aarch64-apple-darwin"),
|
||||
("win32", "x86_64", "x86_64-pc-windows-msvc"),
|
||||
("win32", "arm64", "aarch64-pc-windows-msvc"),
|
||||
],
|
||||
)
|
||||
def test_platform_target_triple_mapping(
|
||||
monkeypatch: pytest.MonkeyPatch, system: str, arch: str, expected: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(exec_module.sys, "platform", system)
|
||||
monkeypatch.setattr(exec_module.platform, "machine", lambda: arch)
|
||||
assert exec_module._platform_target_triple() == expected
|
||||
|
||||
|
||||
def test_platform_target_triple_unsupported(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(exec_module.sys, "platform", "solaris")
|
||||
monkeypatch.setattr(exec_module.platform, "machine", lambda: "sparc")
|
||||
with pytest.raises(RuntimeError, match="Unsupported platform"):
|
||||
exec_module._platform_target_triple()
|
||||
|
||||
|
||||
def test_find_codex_path_env_override(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("CODEX_PATH", "/custom/codex")
|
||||
assert exec_module.find_codex_path() == "/custom/codex"
|
||||
|
||||
|
||||
def test_find_codex_path_uses_shutil_which(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("CODEX_PATH", raising=False)
|
||||
monkeypatch.setattr(exec_module.shutil, "which", lambda _name: "/usr/local/bin/codex")
|
||||
assert exec_module.find_codex_path() == "/usr/local/bin/codex"
|
||||
|
||||
|
||||
def test_find_codex_path_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("CODEX_PATH", raising=False)
|
||||
monkeypatch.setattr(exec_module.shutil, "which", lambda _name: None)
|
||||
monkeypatch.setattr(exec_module, "_platform_target_triple", lambda: "dummy-triple")
|
||||
monkeypatch.setattr(exec_module.sys, "platform", "linux")
|
||||
result = exec_module.find_codex_path()
|
||||
expected_root = (
|
||||
Path(cast(str, exec_module.__file__)).resolve().parent.parent.parent
|
||||
/ "vendor"
|
||||
/ "dummy-triple"
|
||||
/ "codex"
|
||||
/ "codex"
|
||||
)
|
||||
assert result == str(expected_root)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thread_run_streamed_passes_options_and_updates_id(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
events = [
|
||||
{"type": "thread.started", "thread_id": "thread-42"},
|
||||
{
|
||||
"type": "turn.completed",
|
||||
"usage": {"input_tokens": 1, "cached_input_tokens": 0, "output_tokens": 1},
|
||||
},
|
||||
]
|
||||
fake_exec = FakeExec(events)
|
||||
options = CodexOptions(base_url="https://example.com", api_key="api-key")
|
||||
thread_options = ThreadOptions(
|
||||
model="gpt-4.1-mini",
|
||||
sandbox_mode="read-only",
|
||||
working_directory="/work",
|
||||
skip_git_repo_check=True,
|
||||
model_reasoning_effort="low",
|
||||
network_access_enabled=False,
|
||||
web_search_mode="cached",
|
||||
approval_policy="on-request",
|
||||
additional_directories=["/extra"],
|
||||
)
|
||||
thread = Thread(
|
||||
exec_client=cast(CodexExec, fake_exec),
|
||||
options=options,
|
||||
thread_options=thread_options,
|
||||
)
|
||||
cleanup_called = False
|
||||
|
||||
def fake_create_output_schema_file(schema: dict[str, Any] | None) -> OutputSchemaFile:
|
||||
nonlocal cleanup_called
|
||||
|
||||
def cleanup() -> None:
|
||||
nonlocal cleanup_called
|
||||
cleanup_called = True
|
||||
|
||||
return OutputSchemaFile(schema_path="/tmp/schema.json", cleanup=cleanup)
|
||||
|
||||
monkeypatch.setattr(thread_module, "create_output_schema_file", fake_create_output_schema_file)
|
||||
|
||||
streamed = await thread.run_streamed(
|
||||
[
|
||||
{"type": "text", "text": "hello"},
|
||||
{"type": "local_image", "path": "/tmp/a.png"},
|
||||
],
|
||||
TurnOptions(output_schema={"type": "object"}),
|
||||
)
|
||||
collected = [event async for event in streamed.events]
|
||||
|
||||
assert collected[0].type == "thread.started"
|
||||
assert thread.id == "thread-42"
|
||||
assert cleanup_called is True
|
||||
|
||||
assert fake_exec.last_args is not None
|
||||
assert fake_exec.last_args.output_schema_file == "/tmp/schema.json"
|
||||
assert fake_exec.last_args.model == "gpt-4.1-mini"
|
||||
assert fake_exec.last_args.sandbox_mode == "read-only"
|
||||
assert fake_exec.last_args.working_directory == "/work"
|
||||
assert fake_exec.last_args.skip_git_repo_check is True
|
||||
assert fake_exec.last_args.model_reasoning_effort == "low"
|
||||
assert fake_exec.last_args.network_access_enabled is False
|
||||
assert fake_exec.last_args.web_search_mode == "cached"
|
||||
assert fake_exec.last_args.approval_policy == "on-request"
|
||||
assert fake_exec.last_args.additional_directories == ["/extra"]
|
||||
assert fake_exec.last_args.images == ["/tmp/a.png"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thread_run_aggregates_items_and_usage() -> None:
|
||||
events = [
|
||||
{"type": "thread.started", "thread_id": "thread-1"},
|
||||
{
|
||||
"type": "item.completed",
|
||||
"item": {"id": "agent-1", "type": "agent_message", "text": "done"},
|
||||
},
|
||||
{
|
||||
"type": "turn.completed",
|
||||
"usage": {"input_tokens": 2, "cached_input_tokens": 1, "output_tokens": 3},
|
||||
},
|
||||
]
|
||||
thread = Thread(
|
||||
exec_client=cast(CodexExec, FakeExec(events)),
|
||||
options=CodexOptions(),
|
||||
thread_options=ThreadOptions(),
|
||||
)
|
||||
result = await thread.run("hello")
|
||||
|
||||
assert result.final_response == "done"
|
||||
assert result.usage == Usage(
|
||||
input_tokens=2,
|
||||
cached_input_tokens=1,
|
||||
output_tokens=3,
|
||||
)
|
||||
assert len(result.items) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thread_run_raises_on_failure() -> None:
|
||||
events = [
|
||||
{"type": "turn.failed", "error": {"message": "boom"}},
|
||||
]
|
||||
thread = Thread(
|
||||
exec_client=cast(CodexExec, FakeExec(events)),
|
||||
options=CodexOptions(),
|
||||
thread_options=ThreadOptions(),
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="boom"):
|
||||
await thread.run("hello")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thread_run_raises_on_stream_error() -> None:
|
||||
events = [
|
||||
{"type": "error", "message": "boom"},
|
||||
]
|
||||
thread = Thread(
|
||||
exec_client=cast(CodexExec, FakeExec(events)),
|
||||
options=CodexOptions(),
|
||||
thread_options=ThreadOptions(),
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="Codex stream error: boom"):
|
||||
await thread.run("hello")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thread_run_streamed_raises_on_parse_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
events = ["not-json"]
|
||||
fake_exec = FakeExec(events)
|
||||
thread = Thread(
|
||||
exec_client=cast(CodexExec, fake_exec),
|
||||
options=CodexOptions(),
|
||||
thread_options=ThreadOptions(),
|
||||
)
|
||||
|
||||
def fake_create_output_schema_file(schema: dict[str, Any] | None) -> OutputSchemaFile:
|
||||
return OutputSchemaFile(schema_path=None, cleanup=lambda: None)
|
||||
|
||||
monkeypatch.setattr(thread_module, "create_output_schema_file", fake_create_output_schema_file)
|
||||
|
||||
streamed = await thread.run_streamed("hello")
|
||||
with pytest.raises(RuntimeError, match="Failed to parse event"):
|
||||
async for _ in streamed.events:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thread_run_streamed_idle_timeout_sets_signal(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
events = [
|
||||
{
|
||||
"type": "turn.completed",
|
||||
"usage": {"input_tokens": 1, "cached_input_tokens": 0, "output_tokens": 1},
|
||||
}
|
||||
]
|
||||
fake_exec = FakeExec(events, delay=0.2)
|
||||
thread = Thread(
|
||||
exec_client=cast(CodexExec, fake_exec),
|
||||
options=CodexOptions(),
|
||||
thread_options=ThreadOptions(),
|
||||
)
|
||||
signal = asyncio.Event()
|
||||
|
||||
def fake_create_output_schema_file(schema: dict[str, Any] | None) -> OutputSchemaFile:
|
||||
return OutputSchemaFile(schema_path=None, cleanup=lambda: None)
|
||||
|
||||
monkeypatch.setattr(thread_module, "create_output_schema_file", fake_create_output_schema_file)
|
||||
|
||||
with pytest.raises(RuntimeError, match="Codex stream idle for"):
|
||||
async for _ in thread._run_streamed_internal(
|
||||
"hello", TurnOptions(signal=signal, idle_timeout_seconds=0.01)
|
||||
):
|
||||
pass
|
||||
|
||||
assert signal.is_set() is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thread_run_streamed_idle_timeout_creates_signal(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
events = [
|
||||
{
|
||||
"type": "turn.completed",
|
||||
"usage": {"input_tokens": 1, "cached_input_tokens": 0, "output_tokens": 1},
|
||||
}
|
||||
]
|
||||
fake_exec = FakeExec(events, delay=0.2)
|
||||
thread = Thread(
|
||||
exec_client=cast(CodexExec, fake_exec),
|
||||
options=CodexOptions(),
|
||||
thread_options=ThreadOptions(),
|
||||
)
|
||||
|
||||
def fake_create_output_schema_file(schema: dict[str, Any] | None) -> OutputSchemaFile:
|
||||
return OutputSchemaFile(schema_path=None, cleanup=lambda: None)
|
||||
|
||||
monkeypatch.setattr(thread_module, "create_output_schema_file", fake_create_output_schema_file)
|
||||
|
||||
with pytest.raises(RuntimeError, match="Codex stream idle for"):
|
||||
async for _ in thread._run_streamed_internal(
|
||||
"hello", TurnOptions(idle_timeout_seconds=0.01)
|
||||
):
|
||||
pass
|
||||
|
||||
assert fake_exec.last_args is not None
|
||||
assert fake_exec.last_args.signal is not None
|
||||
assert fake_exec.last_args.signal.is_set() is True
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,45 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agents.extensions.experimental.codex.items import AgentMessageItem, TodoItem, TodoListItem
|
||||
|
||||
|
||||
def test_dict_like_supports_mapping_access_for_dataclass_fields() -> None:
|
||||
item = AgentMessageItem(id="item-1", text="hello")
|
||||
|
||||
assert item["id"] == "item-1"
|
||||
assert item["text"] == "hello"
|
||||
assert item["type"] == "agent_message"
|
||||
assert item.get("text") == "hello"
|
||||
assert item.get("missing", "fallback") == "fallback"
|
||||
assert "id" in item
|
||||
assert "missing" not in item
|
||||
assert object() not in item
|
||||
assert list(item.keys()) == ["id", "text", "type"]
|
||||
|
||||
|
||||
def test_dict_like_raises_key_error_for_unknown_fields() -> None:
|
||||
item = AgentMessageItem(id="item-1", text="hello")
|
||||
|
||||
with pytest.raises(KeyError, match="missing"):
|
||||
_ = item["missing"]
|
||||
|
||||
|
||||
def test_dict_like_as_dict_recursively_converts_nested_dataclasses() -> None:
|
||||
item = TodoListItem(
|
||||
id="todo-list-1",
|
||||
items=[
|
||||
TodoItem(text="write tests", completed=True),
|
||||
TodoItem(text="run tests", completed=False),
|
||||
],
|
||||
)
|
||||
|
||||
assert item.as_dict() == {
|
||||
"id": "todo-list-1",
|
||||
"items": [
|
||||
{"text": "write tests", "completed": True},
|
||||
{"text": "run tests", "completed": False},
|
||||
],
|
||||
"type": "todo_list",
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from openai import AsyncOpenAI
|
||||
|
||||
from agents import Agent, ModelSettings, RunConfig, Runner, function_tool
|
||||
from agents.extensions.experimental.hosted_multi_agent import (
|
||||
HostedMultiAgentConfig,
|
||||
OpenAIHostedMultiAgentModel,
|
||||
get_hosted_agent_metadata,
|
||||
)
|
||||
from agents.tool_context import ToolContext
|
||||
|
||||
pytestmark = [
|
||||
pytest.mark.allow_call_model_methods,
|
||||
pytest.mark.skipif(
|
||||
os.environ.get("OPENAI_RUN_LIVE_HOSTED_MULTI_AGENT_TESTS") != "1",
|
||||
reason="Set OPENAI_RUN_LIVE_HOSTED_MULTI_AGENT_TESTS=1 to run live beta tests.",
|
||||
),
|
||||
]
|
||||
|
||||
_PROPOSALS = {
|
||||
"alpha": {"estimated_weeks": 6, "risk": "medium"},
|
||||
"beta": {"estimated_weeks": 8, "risk": "low"},
|
||||
}
|
||||
|
||||
|
||||
def _tool_output(arguments: str) -> str:
|
||||
proposal = json.loads(arguments)["proposal"]
|
||||
return json.dumps(_PROPOSALS[proposal], sort_keys=True)
|
||||
|
||||
|
||||
async def _run_direct_baseline(client: AsyncOpenAI) -> tuple[str, set[str], set[str]]:
|
||||
beta = getattr(client, "beta", None)
|
||||
responses = getattr(beta, "responses", None)
|
||||
connect = getattr(responses, "connect", None)
|
||||
if not callable(connect):
|
||||
pytest.fail("The installed openai package does not provide client.beta.responses.connect.")
|
||||
|
||||
tools = [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "get_proposal",
|
||||
"description": "Return deterministic details for one proposal.",
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"proposal": {"type": "string", "enum": ["alpha", "beta"]}},
|
||||
"required": ["proposal"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
"strict": True,
|
||||
}
|
||||
]
|
||||
callers: set[str] = set()
|
||||
call_ids: set[str] = set()
|
||||
completed_response: Any | None = None
|
||||
response_id: str | None = None
|
||||
pending_injections = 0
|
||||
|
||||
async with connect(
|
||||
extra_headers={"OpenAI-Beta": "responses_multi_agent=v1"},
|
||||
max_retries=0,
|
||||
) as connection:
|
||||
await connection.send(
|
||||
{
|
||||
"type": "response.create",
|
||||
"model": "gpt-5.6-sol",
|
||||
"input": [{"role": "user", "content": "Compare proposal alpha and proposal beta."}],
|
||||
"instructions": (
|
||||
"Create two subagents. Assign proposal alpha to one and proposal beta to the "
|
||||
"other. Each subagent must call get_proposal, then the root must synthesize."
|
||||
),
|
||||
"tools": tools,
|
||||
"store": True,
|
||||
"multi_agent": {"enabled": True, "max_concurrent_subagents": 2},
|
||||
}
|
||||
)
|
||||
|
||||
async for event in connection:
|
||||
if event.type == "response.created":
|
||||
response_id = event.response.id
|
||||
elif event.type == "response.output_item.done" and event.item.type == "function_call":
|
||||
if response_id is None:
|
||||
pytest.fail("Direct baseline received a function call before response.created.")
|
||||
call_ids.add(event.item.call_id)
|
||||
agent = getattr(event.item, "agent", None)
|
||||
callers.add(getattr(agent, "agent_name", "/root"))
|
||||
pending_injections += 1
|
||||
await connection.send(
|
||||
{
|
||||
"type": "response.inject",
|
||||
"response_id": response_id,
|
||||
"input": [
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": event.item.call_id,
|
||||
"output": _tool_output(event.item.arguments),
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
elif event.type == "response.inject.created":
|
||||
pending_injections -= 1
|
||||
elif event.type == "response.inject.failed":
|
||||
pytest.fail(f"Direct baseline injection failed: {event.error}")
|
||||
elif event.type == "response.completed":
|
||||
completed_response = event.response
|
||||
elif event.type in {"error", "response.failed", "response.incomplete"}:
|
||||
pytest.fail(f"Direct baseline failed: {event}")
|
||||
|
||||
if completed_response is not None and pending_injections == 0:
|
||||
break
|
||||
|
||||
if completed_response is None:
|
||||
pytest.fail("Direct hosted multi-agent baseline did not complete.")
|
||||
|
||||
root_text: list[str] = []
|
||||
for item in completed_response.output:
|
||||
if (
|
||||
item.type == "message"
|
||||
and getattr(getattr(item, "agent", None), "agent_name", None) == "/root"
|
||||
and getattr(item, "phase", None) == "final_answer"
|
||||
):
|
||||
root_text.extend(part.text for part in item.content if part.type == "output_text")
|
||||
return "".join(root_text), callers, call_ids
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_direct_and_agents_sdk_semantic_parity() -> None:
|
||||
if os.environ.get("OPENAI_API_KEY") in {None, "", "test_key"}:
|
||||
pytest.fail("A real OPENAI_API_KEY is required for the live beta test.")
|
||||
|
||||
client = AsyncOpenAI()
|
||||
direct_text, direct_callers, direct_call_ids = await _run_direct_baseline(client)
|
||||
sdk_callers: set[str] = set()
|
||||
sdk_call_ids: set[str] = set()
|
||||
|
||||
@function_tool
|
||||
def get_proposal(ctx: ToolContext[Any], proposal: str) -> dict[str, object]:
|
||||
metadata = get_hosted_agent_metadata(ctx)
|
||||
sdk_callers.add(metadata.agent_name if metadata else "/root")
|
||||
sdk_call_ids.add(ctx.tool_call_id)
|
||||
return _PROPOSALS[proposal]
|
||||
|
||||
model = OpenAIHostedMultiAgentModel(
|
||||
model="gpt-5.6-sol",
|
||||
openai_client=client,
|
||||
config=HostedMultiAgentConfig(max_concurrent_subagents=2),
|
||||
)
|
||||
agent = Agent(
|
||||
name="Hosted proposal coordinator",
|
||||
instructions=(
|
||||
"Create two subagents. Assign proposal alpha to one and proposal beta to the other. "
|
||||
"Each subagent must call get_proposal, then the root must synthesize."
|
||||
),
|
||||
model=model,
|
||||
model_settings=ModelSettings(
|
||||
store=False,
|
||||
response_include=["reasoning.encrypted_content"],
|
||||
),
|
||||
tools=[get_proposal],
|
||||
)
|
||||
result = await Runner.run(
|
||||
agent,
|
||||
"Compare proposal alpha and proposal beta.",
|
||||
run_config=RunConfig(tracing_disabled=True),
|
||||
max_turns=6,
|
||||
)
|
||||
|
||||
assert direct_text
|
||||
assert result.final_output
|
||||
assert len(direct_call_ids) == 2
|
||||
assert len(sdk_call_ids) == 2
|
||||
assert all(caller != "/root" for caller in direct_callers)
|
||||
assert all(caller != "/root" for caller in sdk_callers)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,410 @@
|
||||
"""Tests for AsyncSQLiteSession functionality."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
from collections.abc import Sequence
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("aiosqlite") # Skip tests if aiosqlite is not installed
|
||||
|
||||
from agents import Agent, Runner, TResponseInputItem
|
||||
from agents.extensions.memory import AsyncSQLiteSession
|
||||
from agents.memory import SessionSettings
|
||||
from tests.fake_model import FakeModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def agent() -> Agent:
|
||||
"""Fixture for a basic agent with a fake model."""
|
||||
return Agent(name="test", model=FakeModel())
|
||||
|
||||
|
||||
def _item_ids(items: Sequence[TResponseInputItem]) -> list[str]:
|
||||
result: list[str] = []
|
||||
for item in items:
|
||||
item_dict = cast(dict[str, Any], item)
|
||||
result.append(cast(str, item_dict["id"]))
|
||||
return result
|
||||
|
||||
|
||||
async def test_async_sqlite_session_basic_flow():
|
||||
"""Test AsyncSQLiteSession add/get/clear behavior."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "async_basic.db"
|
||||
session = AsyncSQLiteSession("async_basic", db_path)
|
||||
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there!"},
|
||||
]
|
||||
|
||||
await session.add_items(items)
|
||||
retrieved = await session.get_items()
|
||||
assert retrieved == items
|
||||
|
||||
await session.clear_session()
|
||||
assert await session.get_items() == []
|
||||
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_pop_item():
|
||||
"""Test AsyncSQLiteSession pop_item behavior."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "async_pop.db"
|
||||
session = AsyncSQLiteSession("async_pop", db_path)
|
||||
|
||||
assert await session.pop_item() is None
|
||||
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "One"},
|
||||
{"role": "assistant", "content": "Two"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
popped = await session.pop_item()
|
||||
assert popped == items[-1]
|
||||
assert await session.get_items() == items[:-1]
|
||||
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_pop_item_skips_corrupt_most_recent():
|
||||
"""pop_item skips corrupt newest rows and returns the next valid item."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "async_pop_corrupt.db"
|
||||
session = AsyncSQLiteSession("async_pop_corrupt", db_path)
|
||||
|
||||
valid_item: TResponseInputItem = {"role": "user", "content": "valid"}
|
||||
await session.add_items([valid_item])
|
||||
|
||||
conn = await session._get_connection()
|
||||
await conn.execute(
|
||||
f"INSERT INTO {session.messages_table} (session_id, message_data) VALUES (?, ?)",
|
||||
(session.session_id, "not valid json {{{"),
|
||||
)
|
||||
await conn.commit()
|
||||
|
||||
assert await session.pop_item() == valid_item
|
||||
assert await session.get_items() == []
|
||||
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_pop_item_returns_none_after_dropping_only_corrupt_rows():
|
||||
"""pop_item removes corrupt rows and returns None when no valid items remain."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "async_pop_only_corrupt.db"
|
||||
session = AsyncSQLiteSession("async_pop_only_corrupt", db_path)
|
||||
|
||||
conn = await session._get_connection()
|
||||
await conn.execute(
|
||||
f"INSERT INTO {session.messages_table} (session_id, message_data) VALUES (?, ?)",
|
||||
(session.session_id, "not valid json {{{"),
|
||||
)
|
||||
await conn.commit()
|
||||
|
||||
assert await session.pop_item() is None
|
||||
assert await session.get_items() == []
|
||||
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_get_items_limit():
|
||||
"""Test AsyncSQLiteSession get_items limit handling."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "async_limit.db"
|
||||
session = AsyncSQLiteSession("async_limit", db_path)
|
||||
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Message 1"},
|
||||
{"role": "assistant", "content": "Response 1"},
|
||||
{"role": "user", "content": "Message 2"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
latest = await session.get_items(limit=2)
|
||||
assert latest == items[-2:]
|
||||
|
||||
none = await session.get_items(limit=0)
|
||||
assert none == []
|
||||
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_session_settings_default():
|
||||
"""Test that session_settings defaults to empty SessionSettings."""
|
||||
session = AsyncSQLiteSession("async_default_settings")
|
||||
|
||||
assert isinstance(session.session_settings, SessionSettings)
|
||||
assert session.session_settings.limit is None
|
||||
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_session_settings_constructor():
|
||||
"""Test passing session_settings via constructor."""
|
||||
session = AsyncSQLiteSession(
|
||||
"async_constructor_settings",
|
||||
session_settings=SessionSettings(limit=5),
|
||||
)
|
||||
|
||||
assert session.session_settings is not None
|
||||
assert session.session_settings.limit == 5
|
||||
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_get_items_uses_session_settings_limit():
|
||||
"""Test that get_items uses session_settings.limit as default."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "async_settings_limit.db"
|
||||
session = AsyncSQLiteSession(
|
||||
"async_settings_limit",
|
||||
db_path,
|
||||
session_settings=SessionSettings(limit=3),
|
||||
)
|
||||
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": f"Message {i}"} for i in range(5)
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
retrieved = await session.get_items()
|
||||
assert retrieved == items[-3:]
|
||||
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_explicit_limit_overrides_session_settings():
|
||||
"""Test that explicit limit parameter overrides session_settings."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "async_settings_override.db"
|
||||
session = AsyncSQLiteSession(
|
||||
"async_settings_override",
|
||||
db_path,
|
||||
session_settings=SessionSettings(limit=5),
|
||||
)
|
||||
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": f"Message {i}"} for i in range(10)
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
retrieved = await session.get_items(limit=2)
|
||||
assert retrieved == items[-2:]
|
||||
|
||||
no_items = await session.get_items(limit=0)
|
||||
assert no_items == []
|
||||
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_unicode_content():
|
||||
"""Test AsyncSQLiteSession stores unicode content."""
|
||||
session = AsyncSQLiteSession("async_unicode")
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "こんにちは"},
|
||||
{"role": "assistant", "content": "Привет"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
retrieved = await session.get_items()
|
||||
assert retrieved == items
|
||||
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_runner_integration(agent: Agent):
|
||||
"""Test that AsyncSQLiteSession works correctly with the agent Runner."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "async_runner_integration.db"
|
||||
session = AsyncSQLiteSession("runner_integration_test", db_path)
|
||||
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
|
||||
agent.model.set_next_output([get_text_message("San Francisco")])
|
||||
result1 = await Runner.run(
|
||||
agent,
|
||||
"What city is the Golden Gate Bridge in?",
|
||||
session=session,
|
||||
)
|
||||
assert result1.final_output == "San Francisco"
|
||||
|
||||
agent.model.set_next_output([get_text_message("California")])
|
||||
result2 = await Runner.run(agent, "What state is it in?", session=session)
|
||||
assert result2.final_output == "California"
|
||||
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
assert isinstance(last_input, list)
|
||||
assert len(last_input) > 1
|
||||
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
|
||||
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_session_isolation(agent: Agent):
|
||||
"""Test that different session IDs result in isolated conversation histories."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "async_isolation.db"
|
||||
session1 = AsyncSQLiteSession("session_1", db_path)
|
||||
session2 = AsyncSQLiteSession("session_2", db_path)
|
||||
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("I like cats.")])
|
||||
await Runner.run(agent, "I like cats.", session=session1)
|
||||
|
||||
agent.model.set_next_output([get_text_message("I like dogs.")])
|
||||
await Runner.run(agent, "I like dogs.", session=session2)
|
||||
|
||||
agent.model.set_next_output([get_text_message("You said you like cats.")])
|
||||
result = await Runner.run(agent, "What animal did I say I like?", session=session1)
|
||||
assert "cats" in result.final_output.lower()
|
||||
assert "dogs" not in result.final_output.lower()
|
||||
|
||||
await session1.close()
|
||||
await session2.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_add_empty_items_list():
|
||||
"""Test that adding an empty list of items is a no-op."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "async_add_empty.db"
|
||||
session = AsyncSQLiteSession("add_empty_test", db_path)
|
||||
|
||||
assert await session.get_items() == []
|
||||
await session.add_items([])
|
||||
assert await session.get_items() == []
|
||||
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_pop_from_empty_session():
|
||||
"""Test that pop_item returns None on an empty session."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "async_pop_empty.db"
|
||||
session = AsyncSQLiteSession("empty_session", db_path)
|
||||
|
||||
popped = await session.pop_item()
|
||||
assert popped is None
|
||||
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_get_items_with_limit_more_than_available():
|
||||
"""Test limit behavior when requesting more items than exist."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "async_limit_more.db"
|
||||
session = AsyncSQLiteSession("limit_more_test", db_path)
|
||||
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "1"},
|
||||
{"role": "assistant", "content": "2"},
|
||||
{"role": "user", "content": "3"},
|
||||
{"role": "assistant", "content": "4"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
retrieved = await session.get_items(limit=10)
|
||||
assert retrieved == items
|
||||
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_get_items_same_timestamp_consistent_order():
|
||||
"""Test that items with identical timestamps keep insertion order."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "async_same_timestamp.db"
|
||||
session = AsyncSQLiteSession("same_timestamp_test", db_path)
|
||||
|
||||
older_item = cast(
|
||||
TResponseInputItem, {"id": "older_same_ts", "role": "user", "content": "old"}
|
||||
)
|
||||
reasoning_item = cast(TResponseInputItem, {"id": "rs_same_ts", "type": "reasoning"})
|
||||
message_item = cast(
|
||||
TResponseInputItem,
|
||||
{"id": "msg_same_ts", "type": "message", "role": "assistant", "content": []},
|
||||
)
|
||||
|
||||
await session.add_items([older_item])
|
||||
await session.add_items([reasoning_item, message_item])
|
||||
|
||||
conn = await session._get_connection()
|
||||
cursor = await conn.execute(
|
||||
f"SELECT id, message_data FROM {session.messages_table} WHERE session_id = ?",
|
||||
(session.session_id,),
|
||||
)
|
||||
rows = await cursor.fetchall()
|
||||
await cursor.close()
|
||||
|
||||
id_map: dict[str, int] = {
|
||||
cast(str, json.loads(message_json)["id"]): cast(int, row_id)
|
||||
for row_id, message_json in rows
|
||||
}
|
||||
|
||||
shared = datetime(2025, 10, 15, 17, 26, 39, 132483)
|
||||
shared_str = shared.strftime("%Y-%m-%d %H:%M:%S.%f")
|
||||
await conn.execute(
|
||||
f"""
|
||||
UPDATE {session.messages_table}
|
||||
SET created_at = ?
|
||||
WHERE id IN (?, ?, ?)
|
||||
""",
|
||||
(
|
||||
shared_str,
|
||||
id_map["older_same_ts"],
|
||||
id_map["rs_same_ts"],
|
||||
id_map["msg_same_ts"],
|
||||
),
|
||||
)
|
||||
await conn.commit()
|
||||
|
||||
retrieved = await session.get_items()
|
||||
assert _item_ids(retrieved) == ["older_same_ts", "rs_same_ts", "msg_same_ts"]
|
||||
|
||||
latest_two = await session.get_items(limit=2)
|
||||
assert _item_ids(latest_two) == ["rs_same_ts", "msg_same_ts"]
|
||||
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_async_sqlite_session_pop_item_same_timestamp_returns_latest():
|
||||
"""Test that pop_item returns the newest item when timestamps tie."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "async_same_timestamp_pop.db"
|
||||
session = AsyncSQLiteSession("same_timestamp_pop_test", db_path)
|
||||
|
||||
reasoning_item = cast(TResponseInputItem, {"id": "rs_pop_same_ts", "type": "reasoning"})
|
||||
message_item = cast(
|
||||
TResponseInputItem,
|
||||
{"id": "msg_pop_same_ts", "type": "message", "role": "assistant", "content": []},
|
||||
)
|
||||
|
||||
await session.add_items([reasoning_item, message_item])
|
||||
|
||||
conn = await session._get_connection()
|
||||
shared = datetime(2025, 10, 15, 17, 26, 39, 132483)
|
||||
shared_str = shared.strftime("%Y-%m-%d %H:%M:%S.%f")
|
||||
await conn.execute(
|
||||
f"UPDATE {session.messages_table} SET created_at = ? WHERE session_id = ?",
|
||||
(shared_str, session.session_id),
|
||||
)
|
||||
await conn.commit()
|
||||
|
||||
popped = await session.pop_item()
|
||||
assert popped is not None
|
||||
assert cast(dict[str, Any], popped)["id"] == "msg_pop_same_ts"
|
||||
|
||||
remaining = await session.get_items()
|
||||
assert _item_ids(remaining) == ["rs_pop_same_ts"]
|
||||
|
||||
await session.close()
|
||||
@@ -0,0 +1,594 @@
|
||||
"""
|
||||
Integration tests for DaprSession with real Dapr sidecar and Redis using testcontainers.
|
||||
|
||||
These tests use Docker containers for both Redis and Dapr, with proper networking.
|
||||
Tests are automatically skipped if dependencies (dapr, testcontainers, docker) are not available.
|
||||
|
||||
Run with: pytest tests/extensions/memory/test_dapr_redis_integration.py -v
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
import urllib.request
|
||||
|
||||
import docker # type: ignore[import-untyped]
|
||||
import pytest
|
||||
from docker.errors import DockerException # type: ignore[import-untyped]
|
||||
|
||||
# Skip tests if dependencies are not available
|
||||
pytest.importorskip("dapr") # Skip tests if Dapr is not installed
|
||||
pytest.importorskip("testcontainers") # Skip if testcontainers is not installed
|
||||
if sys.platform == "win32":
|
||||
pytest.skip(
|
||||
"Dapr Docker integration tests are not supported on Windows",
|
||||
allow_module_level=True,
|
||||
)
|
||||
if shutil.which("docker") is None:
|
||||
pytest.skip(
|
||||
"Docker executable is not available; skipping Dapr integration tests",
|
||||
allow_module_level=True,
|
||||
)
|
||||
try:
|
||||
client = docker.from_env()
|
||||
client.ping()
|
||||
except DockerException:
|
||||
pytest.skip(
|
||||
"Docker daemon is not available; skipping Dapr integration tests", allow_module_level=True
|
||||
)
|
||||
else:
|
||||
client.close()
|
||||
|
||||
from testcontainers.core.container import DockerContainer # type: ignore[import-untyped]
|
||||
from testcontainers.core.network import Network # type: ignore[import-untyped]
|
||||
from testcontainers.core.waiting_utils import wait_for_logs # type: ignore[import-untyped]
|
||||
|
||||
from agents import Agent, Runner, TResponseInputItem
|
||||
from agents.extensions.memory import (
|
||||
DAPR_CONSISTENCY_EVENTUAL,
|
||||
DAPR_CONSISTENCY_STRONG,
|
||||
DaprSession,
|
||||
)
|
||||
from tests.fake_model import FakeModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
# Docker-backed integration tests should stay on the serial test path.
|
||||
pytestmark = [pytest.mark.asyncio, pytest.mark.serial]
|
||||
|
||||
|
||||
def wait_for_dapr_health(host: str, port: int, timeout: int = 60) -> bool:
|
||||
"""
|
||||
Wait for Dapr sidecar to become healthy by checking the HTTP health endpoint.
|
||||
|
||||
Args:
|
||||
host: The host where Dapr is running
|
||||
port: The HTTP port (typically 3500)
|
||||
timeout: Maximum time to wait in seconds
|
||||
|
||||
Returns:
|
||||
True if Dapr becomes healthy, False otherwise
|
||||
"""
|
||||
health_url = f"http://{host}:{port}/v1.0/healthz/outbound"
|
||||
start_time = time.time()
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
try:
|
||||
with urllib.request.urlopen(health_url, timeout=5) as response:
|
||||
if 200 <= response.status < 300:
|
||||
print(f"✓ Dapr health check passed on {health_url}")
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
print(f"✗ Dapr health check timed out after {timeout}s on {health_url}")
|
||||
return False
|
||||
|
||||
|
||||
def wait_for_dapr_component(host: str, port: int, component_name: str, timeout: int = 60) -> bool:
|
||||
"""Wait for a named component to appear in the Dapr metadata endpoint."""
|
||||
metadata_url = f"http://{host}:{port}/v1.0/metadata"
|
||||
start_time = time.time()
|
||||
|
||||
while time.time() - start_time < timeout:
|
||||
try:
|
||||
with urllib.request.urlopen(metadata_url, timeout=5) as response:
|
||||
if 200 <= response.status < 300:
|
||||
payload = json.load(response)
|
||||
components = payload.get("components", [])
|
||||
if any(component.get("name") == component_name for component in components):
|
||||
print(f"✓ Dapr component {component_name} loaded via {metadata_url}")
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
print(f"✗ Dapr component {component_name} did not load after {timeout}s")
|
||||
return False
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def docker_network():
|
||||
"""Create a Docker network for container-to-container communication."""
|
||||
with Network() as network:
|
||||
yield network
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def redis_container(docker_network):
|
||||
"""Start Redis container on the shared network."""
|
||||
container = (
|
||||
DockerContainer("redis:7-alpine")
|
||||
.with_network(docker_network)
|
||||
.with_network_aliases("redis")
|
||||
.with_exposed_ports(6379)
|
||||
)
|
||||
container.start()
|
||||
wait_for_logs(container, "Ready to accept connections", timeout=30)
|
||||
try:
|
||||
yield container
|
||||
finally:
|
||||
container.stop()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def dapr_container(redis_container, docker_network):
|
||||
"""Start Dapr sidecar container with Redis state store configuration."""
|
||||
# Create temporary components directory
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
os.chmod(temp_dir, 0o755)
|
||||
components_path = os.path.join(temp_dir, "components")
|
||||
os.makedirs(components_path, exist_ok=True)
|
||||
os.chmod(components_path, 0o755)
|
||||
|
||||
# Write Redis state store component configuration
|
||||
# KEY: Use 'redis:6379' (network alias), NOT localhost!
|
||||
state_store_config = """
|
||||
apiVersion: dapr.io/v1alpha1
|
||||
kind: Component
|
||||
metadata:
|
||||
name: statestore
|
||||
spec:
|
||||
type: state.redis
|
||||
version: v1
|
||||
metadata:
|
||||
- name: redisHost
|
||||
value: redis:6379
|
||||
- name: redisPassword
|
||||
value: ""
|
||||
- name: actorStateStore
|
||||
value: "false"
|
||||
"""
|
||||
state_store_path = os.path.join(components_path, "statestore.yaml")
|
||||
with open(state_store_path, "w") as f:
|
||||
f.write(state_store_config)
|
||||
os.chmod(state_store_path, 0o644)
|
||||
|
||||
# Create Dapr container
|
||||
container = DockerContainer("daprio/daprd:latest")
|
||||
container = container.with_network(docker_network) # Join the same network
|
||||
container = container.with_volume_mapping(components_path, "/components", mode="ro")
|
||||
container = container.with_command(
|
||||
[
|
||||
"./daprd",
|
||||
"-app-id",
|
||||
"test-app",
|
||||
"-dapr-http-port",
|
||||
"3500", # HTTP API port for health checks
|
||||
"-dapr-grpc-port",
|
||||
"50001",
|
||||
"-resources-path",
|
||||
"/components",
|
||||
"-log-level",
|
||||
"info",
|
||||
]
|
||||
)
|
||||
container = container.with_exposed_ports(3500, 50001) # Expose both ports
|
||||
|
||||
container.start()
|
||||
|
||||
# Get the exposed HTTP port and host
|
||||
http_host = container.get_container_host_ip()
|
||||
http_port = container.get_exposed_port(3500)
|
||||
|
||||
# Wait for Dapr to become healthy
|
||||
if not wait_for_dapr_health(http_host, http_port, timeout=60):
|
||||
container.stop()
|
||||
pytest.fail("Dapr container failed to become healthy")
|
||||
|
||||
if not wait_for_dapr_component(http_host, http_port, "statestore", timeout=60):
|
||||
logs = container.get_wrapped_container().logs().decode("utf-8", errors="replace")
|
||||
container.stop()
|
||||
pytest.fail(f"Dapr state store component failed to load.\nContainer logs:\n{logs}")
|
||||
|
||||
# Set environment variables for Dapr SDK health checks
|
||||
# The Dapr SDK checks these when creating a client
|
||||
os.environ["DAPR_HTTP_PORT"] = str(http_port)
|
||||
os.environ["DAPR_RUNTIME_HOST"] = http_host
|
||||
|
||||
yield container
|
||||
|
||||
# Cleanup environment variables
|
||||
os.environ.pop("DAPR_HTTP_PORT", None)
|
||||
os.environ.pop("DAPR_RUNTIME_HOST", None)
|
||||
|
||||
container.stop()
|
||||
|
||||
# Cleanup
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(temp_dir, ignore_errors=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def agent() -> Agent:
|
||||
"""Fixture for a basic agent with a fake model."""
|
||||
return Agent(name="test", model=FakeModel())
|
||||
|
||||
|
||||
async def test_dapr_redis_integration(dapr_container, monkeypatch):
|
||||
"""Test DaprSession with real Dapr sidecar and Redis backend."""
|
||||
# Get Dapr gRPC address (exposed to host)
|
||||
dapr_host = dapr_container.get_container_host_ip()
|
||||
dapr_port = dapr_container.get_exposed_port(50001)
|
||||
dapr_address = f"{dapr_host}:{dapr_port}"
|
||||
|
||||
# Monkeypatch the Dapr health check since we already verified it in the fixture
|
||||
from dapr.clients.health import DaprHealth
|
||||
|
||||
monkeypatch.setattr(DaprHealth, "wait_until_ready", lambda: None)
|
||||
|
||||
# Create session using from_address
|
||||
session = DaprSession.from_address(
|
||||
session_id="integration_test_session",
|
||||
state_store_name="statestore",
|
||||
dapr_address=dapr_address,
|
||||
)
|
||||
|
||||
try:
|
||||
# Test connectivity
|
||||
is_connected = await session.ping()
|
||||
assert is_connected is True
|
||||
|
||||
# Clear any existing data
|
||||
await session.clear_session()
|
||||
|
||||
# Test add_items
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Hello from integration test"},
|
||||
{"role": "assistant", "content": "Hi there!"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
# Test get_items
|
||||
retrieved = await session.get_items()
|
||||
assert len(retrieved) == 2
|
||||
assert retrieved[0].get("content") == "Hello from integration test"
|
||||
assert retrieved[1].get("content") == "Hi there!"
|
||||
|
||||
# Test get_items with limit
|
||||
latest_1 = await session.get_items(limit=1)
|
||||
assert len(latest_1) == 1
|
||||
assert latest_1[0].get("content") == "Hi there!"
|
||||
|
||||
# Test pop_item
|
||||
popped = await session.pop_item()
|
||||
assert popped is not None
|
||||
assert popped.get("content") == "Hi there!"
|
||||
|
||||
remaining = await session.get_items()
|
||||
assert len(remaining) == 1
|
||||
assert remaining[0].get("content") == "Hello from integration test"
|
||||
|
||||
# Test clear_session
|
||||
await session.clear_session()
|
||||
cleared = await session.get_items()
|
||||
assert len(cleared) == 0
|
||||
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_dapr_runner_integration(agent: Agent, dapr_container, monkeypatch):
|
||||
"""Test DaprSession with agent Runner using real Dapr sidecar."""
|
||||
from dapr.clients.health import DaprHealth
|
||||
|
||||
monkeypatch.setattr(DaprHealth, "wait_until_ready", lambda: None)
|
||||
|
||||
dapr_host = dapr_container.get_container_host_ip()
|
||||
dapr_port = dapr_container.get_exposed_port(50001)
|
||||
dapr_address = f"{dapr_host}:{dapr_port}"
|
||||
|
||||
session = DaprSession.from_address(
|
||||
session_id="runner_integration_test",
|
||||
state_store_name="statestore",
|
||||
dapr_address=dapr_address,
|
||||
)
|
||||
|
||||
try:
|
||||
await session.clear_session()
|
||||
|
||||
# First turn
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("San Francisco")])
|
||||
result1 = await Runner.run(
|
||||
agent,
|
||||
"What city is the Golden Gate Bridge in?",
|
||||
session=session,
|
||||
)
|
||||
assert result1.final_output == "San Francisco"
|
||||
|
||||
# Second turn - should remember context
|
||||
agent.model.set_next_output([get_text_message("California")])
|
||||
result2 = await Runner.run(agent, "What state is it in?", session=session)
|
||||
assert result2.final_output == "California"
|
||||
|
||||
# Verify history
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
assert len(last_input) > 1
|
||||
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
|
||||
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_dapr_session_isolation(dapr_container, monkeypatch):
|
||||
"""Test that different session IDs are isolated with real Dapr."""
|
||||
from dapr.clients.health import DaprHealth
|
||||
|
||||
monkeypatch.setattr(DaprHealth, "wait_until_ready", lambda: None)
|
||||
|
||||
dapr_host = dapr_container.get_container_host_ip()
|
||||
dapr_port = dapr_container.get_exposed_port(50001)
|
||||
dapr_address = f"{dapr_host}:{dapr_port}"
|
||||
|
||||
session1 = DaprSession.from_address(
|
||||
session_id="isolated_session_1",
|
||||
state_store_name="statestore",
|
||||
dapr_address=dapr_address,
|
||||
)
|
||||
session2 = DaprSession.from_address(
|
||||
session_id="isolated_session_2",
|
||||
state_store_name="statestore",
|
||||
dapr_address=dapr_address,
|
||||
)
|
||||
|
||||
try:
|
||||
# Clear both sessions
|
||||
await session1.clear_session()
|
||||
await session2.clear_session()
|
||||
|
||||
# Add different data to each session
|
||||
await session1.add_items([{"role": "user", "content": "session 1 data"}])
|
||||
await session2.add_items([{"role": "user", "content": "session 2 data"}])
|
||||
|
||||
# Verify isolation
|
||||
items1 = await session1.get_items()
|
||||
items2 = await session2.get_items()
|
||||
|
||||
assert len(items1) == 1
|
||||
assert len(items2) == 1
|
||||
assert items1[0].get("content") == "session 1 data"
|
||||
assert items2[0].get("content") == "session 2 data"
|
||||
|
||||
finally:
|
||||
await session1.clear_session()
|
||||
await session2.clear_session()
|
||||
await session1.close()
|
||||
await session2.close()
|
||||
|
||||
|
||||
async def test_dapr_ttl_functionality(dapr_container, monkeypatch):
|
||||
"""Test TTL functionality with real Dapr and Redis (if supported by state store)."""
|
||||
from dapr.clients.health import DaprHealth
|
||||
|
||||
monkeypatch.setattr(DaprHealth, "wait_until_ready", lambda: None)
|
||||
|
||||
dapr_host = dapr_container.get_container_host_ip()
|
||||
dapr_port = dapr_container.get_exposed_port(50001)
|
||||
dapr_address = f"{dapr_host}:{dapr_port}"
|
||||
|
||||
# Create session with short TTL
|
||||
session = DaprSession.from_address(
|
||||
session_id="ttl_test_session",
|
||||
state_store_name="statestore",
|
||||
dapr_address=dapr_address,
|
||||
ttl=2, # 2 seconds TTL
|
||||
)
|
||||
|
||||
try:
|
||||
await session.clear_session()
|
||||
|
||||
# Add items with TTL
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "This should expire soon"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
# Verify items exist immediately
|
||||
retrieved = await session.get_items()
|
||||
assert len(retrieved) == 1
|
||||
|
||||
# Note: Actual expiration testing depends on state store TTL support
|
||||
# Redis state store supports TTL via ttlInSeconds metadata
|
||||
|
||||
finally:
|
||||
await session.clear_session()
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_dapr_consistency_levels(dapr_container, monkeypatch):
|
||||
"""Test different consistency levels with real Dapr."""
|
||||
from dapr.clients.health import DaprHealth
|
||||
|
||||
monkeypatch.setattr(DaprHealth, "wait_until_ready", lambda: None)
|
||||
|
||||
dapr_host = dapr_container.get_container_host_ip()
|
||||
dapr_port = dapr_container.get_exposed_port(50001)
|
||||
dapr_address = f"{dapr_host}:{dapr_port}"
|
||||
|
||||
# Test eventual consistency
|
||||
session_eventual = DaprSession.from_address(
|
||||
session_id="eventual_consistency_test",
|
||||
state_store_name="statestore",
|
||||
dapr_address=dapr_address,
|
||||
consistency=DAPR_CONSISTENCY_EVENTUAL,
|
||||
)
|
||||
|
||||
# Test strong consistency
|
||||
session_strong = DaprSession.from_address(
|
||||
session_id="strong_consistency_test",
|
||||
state_store_name="statestore",
|
||||
dapr_address=dapr_address,
|
||||
consistency=DAPR_CONSISTENCY_STRONG,
|
||||
)
|
||||
|
||||
try:
|
||||
await session_eventual.clear_session()
|
||||
await session_strong.clear_session()
|
||||
|
||||
# Both should work correctly
|
||||
items: list[TResponseInputItem] = [{"role": "user", "content": "Consistency test"}]
|
||||
|
||||
await session_eventual.add_items(items)
|
||||
retrieved_eventual = await session_eventual.get_items()
|
||||
assert len(retrieved_eventual) == 1
|
||||
|
||||
await session_strong.add_items(items)
|
||||
retrieved_strong = await session_strong.get_items()
|
||||
assert len(retrieved_strong) == 1
|
||||
|
||||
finally:
|
||||
await session_eventual.clear_session()
|
||||
await session_strong.clear_session()
|
||||
await session_eventual.close()
|
||||
await session_strong.close()
|
||||
|
||||
|
||||
async def test_dapr_unicode_and_special_chars(dapr_container, monkeypatch):
|
||||
"""Test unicode and special characters with real Dapr and Redis."""
|
||||
from dapr.clients.health import DaprHealth
|
||||
|
||||
monkeypatch.setattr(DaprHealth, "wait_until_ready", lambda: None)
|
||||
|
||||
dapr_host = dapr_container.get_container_host_ip()
|
||||
dapr_port = dapr_container.get_exposed_port(50001)
|
||||
dapr_address = f"{dapr_host}:{dapr_port}"
|
||||
|
||||
session = DaprSession.from_address(
|
||||
session_id="unicode_test_session",
|
||||
state_store_name="statestore",
|
||||
dapr_address=dapr_address,
|
||||
)
|
||||
|
||||
try:
|
||||
await session.clear_session()
|
||||
|
||||
# Test unicode content
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "こんにちは"},
|
||||
{"role": "assistant", "content": "😊👍"},
|
||||
{"role": "user", "content": "Привет"},
|
||||
{"role": "assistant", "content": '{"nested": "json"}'},
|
||||
{"role": "user", "content": "Line1\nLine2\tTabbed"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
# Retrieve and verify
|
||||
retrieved = await session.get_items()
|
||||
assert len(retrieved) == 5
|
||||
assert retrieved[0].get("content") == "こんにちは"
|
||||
assert retrieved[1].get("content") == "😊👍"
|
||||
assert retrieved[2].get("content") == "Привет"
|
||||
assert retrieved[3].get("content") == '{"nested": "json"}'
|
||||
assert retrieved[4].get("content") == "Line1\nLine2\tTabbed"
|
||||
|
||||
finally:
|
||||
await session.clear_session()
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_dapr_concurrent_writes_resolution(dapr_container, monkeypatch):
|
||||
"""
|
||||
Concurrent writes from multiple session instances should resolve via
|
||||
optimistic concurrency.
|
||||
"""
|
||||
from dapr.clients.health import DaprHealth
|
||||
|
||||
monkeypatch.setattr(DaprHealth, "wait_until_ready", lambda: None)
|
||||
|
||||
dapr_host = dapr_container.get_container_host_ip()
|
||||
dapr_port = dapr_container.get_exposed_port(50001)
|
||||
dapr_address = f"{dapr_host}:{dapr_port}"
|
||||
|
||||
# Use two different session objects pointing to the same logical session_id
|
||||
# to create real contention.
|
||||
session_id = "concurrent_integration_session"
|
||||
s1 = DaprSession.from_address(
|
||||
session_id=session_id,
|
||||
state_store_name="statestore",
|
||||
dapr_address=dapr_address,
|
||||
)
|
||||
s2 = DaprSession.from_address(
|
||||
session_id=session_id,
|
||||
state_store_name="statestore",
|
||||
dapr_address=dapr_address,
|
||||
)
|
||||
|
||||
try:
|
||||
# Clean slate.
|
||||
await s1.clear_session()
|
||||
|
||||
# Fire multiple parallel add_items calls from two different session instances.
|
||||
tasks: list[asyncio.Task[None]] = []
|
||||
for i in range(10):
|
||||
tasks.append(
|
||||
asyncio.create_task(
|
||||
s1.add_items(
|
||||
[
|
||||
{"role": "user", "content": f"A-{i}"},
|
||||
]
|
||||
)
|
||||
)
|
||||
)
|
||||
tasks.append(
|
||||
asyncio.create_task(
|
||||
s2.add_items(
|
||||
[
|
||||
{"role": "assistant", "content": f"B-{i}"},
|
||||
]
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
# Validate all messages were persisted.
|
||||
# Use a fresh session object for readback to avoid any local caching
|
||||
# (none expected, but explicit).
|
||||
s_read = DaprSession.from_address(
|
||||
session_id=session_id,
|
||||
state_store_name="statestore",
|
||||
dapr_address=dapr_address,
|
||||
)
|
||||
try:
|
||||
items = await s_read.get_items()
|
||||
contents = [item.get("content") for item in items]
|
||||
# We expect 20 total messages: A-0..9 and B-0..9 (order unspecified).
|
||||
assert len(contents) == 20
|
||||
for i in range(10):
|
||||
assert f"A-{i}" in contents
|
||||
assert f"B-{i}" in contents
|
||||
finally:
|
||||
await s_read.close()
|
||||
finally:
|
||||
await s1.close()
|
||||
await s2.close()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,576 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("cryptography") # Skip tests if cryptography is not installed
|
||||
|
||||
from cryptography.fernet import Fernet
|
||||
|
||||
from agents import Agent, Runner, SessionSettings, SQLiteSession, TResponseInputItem
|
||||
from agents.extensions.memory.encrypt_session import EncryptedSession
|
||||
from tests.fake_model import FakeModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
# Mark all tests in this file as asyncio
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
def _invalid_encrypted_envelope() -> TResponseInputItem:
|
||||
return cast(
|
||||
TResponseInputItem,
|
||||
{"__enc__": 1, "v": 1, "kid": "hkdf-v1", "payload": "not-a-valid-token"},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def agent() -> Agent:
|
||||
"""Fixture for a basic agent with a fake model."""
|
||||
return Agent(name="test", model=FakeModel())
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def encryption_key() -> str:
|
||||
"""Fixture for a valid Fernet encryption key."""
|
||||
return str(Fernet.generate_key().decode("utf-8"))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def set_fernet_time(monkeypatch):
|
||||
"""Freeze Fernet TTL checks so expiration tests avoid real waiting."""
|
||||
current_time = 1_000
|
||||
|
||||
def _set_time(value: int) -> None:
|
||||
nonlocal current_time
|
||||
current_time = value
|
||||
|
||||
monkeypatch.setattr("cryptography.fernet.time.time", lambda: current_time)
|
||||
return _set_time
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def underlying_session():
|
||||
"""Fixture for an underlying SQLite session."""
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
db_path = Path(temp_dir) / "test_encrypt.db"
|
||||
return SQLiteSession("test_session", db_path)
|
||||
|
||||
|
||||
async def test_encrypted_session_basic_functionality(
|
||||
agent: Agent, encryption_key: str, underlying_session: SQLiteSession
|
||||
):
|
||||
"""Test basic encryption/decryption functionality."""
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying_session,
|
||||
encryption_key=encryption_key,
|
||||
ttl=600,
|
||||
)
|
||||
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there!"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
retrieved = await session.get_items()
|
||||
assert len(retrieved) == 2
|
||||
assert retrieved[0].get("content") == "Hello"
|
||||
assert retrieved[1].get("content") == "Hi there!"
|
||||
|
||||
encrypted_items = await underlying_session.get_items()
|
||||
assert encrypted_items[0].get("__enc__") == 1
|
||||
assert "payload" in encrypted_items[0]
|
||||
assert encrypted_items[0].get("content") != "Hello"
|
||||
|
||||
underlying_session.close()
|
||||
|
||||
|
||||
async def test_encrypted_session_with_runner(
|
||||
agent: Agent, encryption_key: str, underlying_session: SQLiteSession
|
||||
):
|
||||
"""Test that EncryptedSession works with Runner."""
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying_session,
|
||||
encryption_key=encryption_key,
|
||||
)
|
||||
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("San Francisco")])
|
||||
result1 = await Runner.run(
|
||||
agent,
|
||||
"What city is the Golden Gate Bridge in?",
|
||||
session=session,
|
||||
)
|
||||
assert result1.final_output == "San Francisco"
|
||||
|
||||
agent.model.set_next_output([get_text_message("California")])
|
||||
result2 = await Runner.run(agent, "What state is it in?", session=session)
|
||||
assert result2.final_output == "California"
|
||||
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
assert len(last_input) > 1
|
||||
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
|
||||
|
||||
underlying_session.close()
|
||||
|
||||
|
||||
async def test_encrypted_session_pop_item(encryption_key: str, underlying_session: SQLiteSession):
|
||||
"""Test pop_item functionality."""
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying_session,
|
||||
encryption_key=encryption_key,
|
||||
)
|
||||
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "First"},
|
||||
{"role": "assistant", "content": "Second"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
popped = await session.pop_item()
|
||||
assert popped is not None
|
||||
assert popped.get("content") == "Second"
|
||||
|
||||
remaining = await session.get_items()
|
||||
assert len(remaining) == 1
|
||||
assert remaining[0].get("content") == "First"
|
||||
|
||||
underlying_session.close()
|
||||
|
||||
|
||||
async def test_encrypted_session_clear(encryption_key: str, underlying_session: SQLiteSession):
|
||||
"""Test clear_session functionality."""
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying_session,
|
||||
encryption_key=encryption_key,
|
||||
)
|
||||
|
||||
await session.add_items([{"role": "user", "content": "Test"}])
|
||||
await session.clear_session()
|
||||
|
||||
items = await session.get_items()
|
||||
assert len(items) == 0
|
||||
|
||||
underlying_session.close()
|
||||
|
||||
|
||||
async def test_encrypted_session_ttl_expiration(
|
||||
encryption_key: str, underlying_session: SQLiteSession, set_fernet_time
|
||||
):
|
||||
"""Test TTL expiration - expired items are silently skipped."""
|
||||
set_fernet_time(1_000)
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying_session,
|
||||
encryption_key=encryption_key,
|
||||
ttl=1, # 1 second TTL
|
||||
)
|
||||
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
set_fernet_time(1_002)
|
||||
|
||||
retrieved = await session.get_items()
|
||||
assert len(retrieved) == 0
|
||||
|
||||
underlying_items = await underlying_session.get_items()
|
||||
assert len(underlying_items) == 2
|
||||
|
||||
underlying_session.close()
|
||||
|
||||
|
||||
async def test_encrypted_session_pop_expired(
|
||||
encryption_key: str, underlying_session: SQLiteSession, set_fernet_time
|
||||
):
|
||||
"""Test pop_item with expired data."""
|
||||
set_fernet_time(1_000)
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying_session,
|
||||
encryption_key=encryption_key,
|
||||
ttl=1,
|
||||
)
|
||||
|
||||
await session.add_items([{"role": "user", "content": "Test"}])
|
||||
set_fernet_time(1_002)
|
||||
|
||||
popped = await session.pop_item()
|
||||
assert popped is None
|
||||
|
||||
underlying_session.close()
|
||||
|
||||
|
||||
async def test_encrypted_session_pop_mixed_expired_valid(
|
||||
encryption_key: str, underlying_session: SQLiteSession, set_fernet_time
|
||||
):
|
||||
"""Test pop_item auto-retry with mixed expired and valid items."""
|
||||
set_fernet_time(1_000)
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying_session,
|
||||
encryption_key=encryption_key,
|
||||
ttl=2, # 2 second TTL
|
||||
)
|
||||
|
||||
await session.add_items(
|
||||
[
|
||||
{"role": "user", "content": "Old message 1"},
|
||||
{"role": "assistant", "content": "Old response 1"},
|
||||
]
|
||||
)
|
||||
|
||||
set_fernet_time(1_003)
|
||||
|
||||
await session.add_items(
|
||||
[
|
||||
{"role": "user", "content": "New message"},
|
||||
{"role": "assistant", "content": "New response"},
|
||||
]
|
||||
)
|
||||
|
||||
popped = await session.pop_item()
|
||||
assert popped is not None
|
||||
assert popped.get("content") == "New response"
|
||||
|
||||
popped2 = await session.pop_item()
|
||||
assert popped2 is not None
|
||||
assert popped2.get("content") == "New message"
|
||||
|
||||
popped3 = await session.pop_item()
|
||||
assert popped3 is None
|
||||
|
||||
underlying_session.close()
|
||||
|
||||
|
||||
async def test_encrypted_session_raw_string_key(underlying_session: SQLiteSession):
|
||||
"""Test using raw string as encryption key (not base64)."""
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying_session,
|
||||
encryption_key="my-secret-password", # Raw string, not Fernet key
|
||||
)
|
||||
|
||||
await session.add_items([{"role": "user", "content": "Test"}])
|
||||
items = await session.get_items()
|
||||
assert len(items) == 1
|
||||
assert items[0].get("content") == "Test"
|
||||
|
||||
underlying_session.close()
|
||||
|
||||
|
||||
async def test_encrypted_session_get_items_limit(
|
||||
encryption_key: str, underlying_session: SQLiteSession
|
||||
):
|
||||
"""Test get_items with limit parameter."""
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying_session,
|
||||
encryption_key=encryption_key,
|
||||
)
|
||||
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": f"Message {i}"} for i in range(5)
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
limited = await session.get_items(limit=2)
|
||||
assert len(limited) == 2
|
||||
assert limited[0].get("content") == "Message 3" # Latest 2
|
||||
assert limited[1].get("content") == "Message 4"
|
||||
|
||||
underlying_session.close()
|
||||
|
||||
|
||||
async def test_encrypted_session_get_items_limit_skips_invalid_latest_envelope(
|
||||
encryption_key: str, underlying_session: SQLiteSession
|
||||
):
|
||||
"""Test that limit counts valid decrypted items, not encrypted envelopes."""
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying_session,
|
||||
encryption_key=encryption_key,
|
||||
)
|
||||
|
||||
await session.add_items([{"role": "user", "content": "older valid"}])
|
||||
await underlying_session.add_items([_invalid_encrypted_envelope()])
|
||||
|
||||
all_items = await session.get_items()
|
||||
assert [item.get("content") for item in all_items] == ["older valid"]
|
||||
|
||||
limited = await session.get_items(limit=1)
|
||||
assert [item.get("content") for item in limited] == ["older valid"]
|
||||
|
||||
underlying_session.close()
|
||||
|
||||
|
||||
async def test_encrypted_session_get_items_limit_returns_latest_valid_items_after_invalids(
|
||||
encryption_key: str, underlying_session: SQLiteSession
|
||||
):
|
||||
"""Test that invalid envelopes do not hide earlier valid items from limit."""
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying_session,
|
||||
encryption_key=encryption_key,
|
||||
)
|
||||
|
||||
await session.add_items(
|
||||
[
|
||||
{"role": "user", "content": "valid 0"},
|
||||
{"role": "assistant", "content": "valid 1"},
|
||||
]
|
||||
)
|
||||
await underlying_session.add_items([_invalid_encrypted_envelope()])
|
||||
await session.add_items([{"role": "user", "content": "valid 2"}])
|
||||
|
||||
limited = await session.get_items(limit=2)
|
||||
assert [item.get("content") for item in limited] == ["valid 1", "valid 2"]
|
||||
|
||||
underlying_session.close()
|
||||
|
||||
|
||||
async def test_encrypted_session_get_items_session_settings_limit_skips_invalid_envelopes(
|
||||
encryption_key: str, underlying_session: SQLiteSession
|
||||
):
|
||||
"""Test that session settings limit counts valid decrypted items."""
|
||||
underlying_session.session_settings = SessionSettings(limit=3)
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying_session,
|
||||
encryption_key=encryption_key,
|
||||
)
|
||||
|
||||
await session.add_items(
|
||||
[
|
||||
{"role": "user", "content": "valid 0"},
|
||||
{"role": "assistant", "content": "valid 1"},
|
||||
{"role": "user", "content": "valid 2"},
|
||||
]
|
||||
)
|
||||
await underlying_session.add_items([_invalid_encrypted_envelope()])
|
||||
|
||||
items = await session.get_items()
|
||||
assert [item.get("content") for item in items] == ["valid 0", "valid 1", "valid 2"]
|
||||
|
||||
underlying_session.close()
|
||||
|
||||
|
||||
async def test_encrypted_session_unicode_content(
|
||||
encryption_key: str, underlying_session: SQLiteSession
|
||||
):
|
||||
"""Test encryption of international text content."""
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying_session,
|
||||
encryption_key=encryption_key,
|
||||
)
|
||||
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Hello world"},
|
||||
{"role": "assistant", "content": "Special chars: áéíóú"},
|
||||
{"role": "user", "content": "Numbers and symbols: 123!@#"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
retrieved = await session.get_items()
|
||||
assert retrieved[0].get("content") == "Hello world"
|
||||
assert retrieved[1].get("content") == "Special chars: áéíóú"
|
||||
assert retrieved[2].get("content") == "Numbers and symbols: 123!@#"
|
||||
|
||||
underlying_session.close()
|
||||
|
||||
|
||||
class CustomSession(SQLiteSession):
|
||||
"""Mock custom session with additional methods for testing delegation."""
|
||||
|
||||
def get_stats(self) -> dict[str, int]:
|
||||
"""Custom method that should be accessible through delegation."""
|
||||
return {"custom_method_calls": 42, "test_value": 123}
|
||||
|
||||
async def custom_async_method(self) -> str:
|
||||
"""Custom async method for testing delegation."""
|
||||
return "custom_async_result"
|
||||
|
||||
|
||||
async def test_encrypted_session_delegation():
|
||||
"""Test that custom methods on underlying session are accessible through delegation."""
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
db_path = Path(temp_dir) / "test_delegation.db"
|
||||
underlying_session = CustomSession("test_session", db_path)
|
||||
|
||||
encryption_key = str(Fernet.generate_key().decode("utf-8"))
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying_session,
|
||||
encryption_key=encryption_key,
|
||||
)
|
||||
|
||||
stats = session.get_stats()
|
||||
assert stats == {"custom_method_calls": 42, "test_value": 123}
|
||||
|
||||
result = await session.custom_async_method()
|
||||
assert result == "custom_async_result"
|
||||
|
||||
await session.add_items([{"role": "user", "content": "Test delegation"}])
|
||||
items = await session.get_items()
|
||||
assert len(items) == 1
|
||||
assert items[0].get("content") == "Test delegation"
|
||||
|
||||
underlying_session.close()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# SessionSettings Tests
|
||||
# ============================================================================
|
||||
|
||||
|
||||
async def test_session_settings_delegated_to_underlying(encryption_key: str):
|
||||
"""Test that session_settings is correctly delegated to underlying session."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
db_path = Path(temp_dir) / "test_settings.db"
|
||||
underlying = SQLiteSession("test_session", db_path, session_settings=SessionSettings(limit=5))
|
||||
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying,
|
||||
encryption_key=encryption_key,
|
||||
)
|
||||
|
||||
# session_settings should be accessible through EncryptedSession
|
||||
assert session.session_settings is not None
|
||||
assert session.session_settings.limit == 5
|
||||
|
||||
underlying.close()
|
||||
|
||||
|
||||
async def test_session_settings_get_items_uses_underlying_limit(encryption_key: str):
|
||||
"""Test that get_items uses underlying session's session_settings.limit."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
db_path = Path(temp_dir) / "test_settings_limit.db"
|
||||
underlying = SQLiteSession("test_session", db_path, session_settings=SessionSettings(limit=3))
|
||||
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying,
|
||||
encryption_key=encryption_key,
|
||||
)
|
||||
|
||||
# Add 5 items
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": f"Message {i}"} for i in range(5)
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
# get_items() with no limit should use underlying session_settings.limit=3
|
||||
retrieved = await session.get_items()
|
||||
assert len(retrieved) == 3
|
||||
# Should get the last 3 items
|
||||
assert retrieved[0].get("content") == "Message 2"
|
||||
assert retrieved[1].get("content") == "Message 3"
|
||||
assert retrieved[2].get("content") == "Message 4"
|
||||
|
||||
underlying.close()
|
||||
|
||||
|
||||
async def test_session_settings_explicit_limit_overrides_settings(encryption_key: str):
|
||||
"""Test that explicit limit parameter overrides session_settings."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
db_path = Path(temp_dir) / "test_override.db"
|
||||
underlying = SQLiteSession("test_session", db_path, session_settings=SessionSettings(limit=5))
|
||||
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying,
|
||||
encryption_key=encryption_key,
|
||||
)
|
||||
|
||||
# Add 10 items
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": f"Message {i}"} for i in range(10)
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
# Explicit limit=2 should override session_settings.limit=5
|
||||
retrieved = await session.get_items(limit=2)
|
||||
assert len(retrieved) == 2
|
||||
assert retrieved[0].get("content") == "Message 8"
|
||||
assert retrieved[1].get("content") == "Message 9"
|
||||
|
||||
underlying.close()
|
||||
|
||||
|
||||
async def test_session_settings_resolve():
|
||||
"""Test SessionSettings.resolve() method."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
base = SessionSettings(limit=100)
|
||||
override = SessionSettings(limit=50)
|
||||
|
||||
final = base.resolve(override)
|
||||
|
||||
assert final.limit == 50 # Override wins
|
||||
assert base.limit == 100 # Original unchanged
|
||||
|
||||
# Resolving with None returns self
|
||||
final_none = base.resolve(None)
|
||||
assert final_none.limit == 100
|
||||
|
||||
|
||||
async def test_runner_with_session_settings_override(encryption_key: str):
|
||||
"""Test that RunConfig can override session's default settings."""
|
||||
from agents import Agent, RunConfig, Runner
|
||||
from agents.memory import SessionSettings
|
||||
from tests.fake_model import FakeModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
db_path = Path(temp_dir) / "test_runner_override.db"
|
||||
underlying = SQLiteSession("test_session", db_path, session_settings=SessionSettings(limit=100))
|
||||
|
||||
session = EncryptedSession(
|
||||
session_id="test_session",
|
||||
underlying_session=underlying,
|
||||
encryption_key=encryption_key,
|
||||
)
|
||||
|
||||
# Add some history
|
||||
items: list[TResponseInputItem] = [{"role": "user", "content": f"Turn {i}"} for i in range(10)]
|
||||
await session.add_items(items)
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
model.set_next_output([get_text_message("Got it")])
|
||||
|
||||
await Runner.run(
|
||||
agent,
|
||||
"New question",
|
||||
session=session,
|
||||
run_config=RunConfig(
|
||||
session_settings=SessionSettings(limit=2) # Override to 2
|
||||
),
|
||||
)
|
||||
|
||||
# Verify the agent received only the last 2 history items + new question
|
||||
last_input = model.last_turn_args["input"]
|
||||
# Filter out the new "New question" input
|
||||
history_items = [item for item in last_input if item.get("content") != "New question"]
|
||||
# Should have 2 history items (last two from the 10 we added)
|
||||
assert len(history_items) == 2
|
||||
|
||||
underlying.close()
|
||||
@@ -0,0 +1,154 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.abc
|
||||
import sys
|
||||
from types import ModuleType
|
||||
|
||||
import pytest
|
||||
|
||||
_PACKAGE_EXPORTS: tuple[tuple[str, str, str, str, str], ...] = (
|
||||
(
|
||||
"EncryptedSession",
|
||||
"agents.extensions.memory.encrypt_session",
|
||||
"agents.extensions.memory.encrypt_session",
|
||||
"cryptography",
|
||||
"encrypt",
|
||||
),
|
||||
("RedisSession", "agents.extensions.memory.redis_session", "redis.asyncio", "redis", "redis"),
|
||||
(
|
||||
"SQLAlchemySession",
|
||||
"agents.extensions.memory.sqlalchemy_session",
|
||||
"agents.extensions.memory.sqlalchemy_session",
|
||||
"sqlalchemy",
|
||||
"sqlalchemy",
|
||||
),
|
||||
("DaprSession", "agents.extensions.memory.dapr_session", "dapr.aio.clients", "dapr", "dapr"),
|
||||
(
|
||||
"DAPR_CONSISTENCY_EVENTUAL",
|
||||
"agents.extensions.memory.dapr_session",
|
||||
"dapr.aio.clients",
|
||||
"dapr",
|
||||
"dapr",
|
||||
),
|
||||
(
|
||||
"DAPR_CONSISTENCY_STRONG",
|
||||
"agents.extensions.memory.dapr_session",
|
||||
"dapr.aio.clients",
|
||||
"dapr",
|
||||
"dapr",
|
||||
),
|
||||
(
|
||||
"MongoDBSession",
|
||||
"agents.extensions.memory.mongodb_session",
|
||||
"pymongo.asynchronous.collection",
|
||||
"mongodb",
|
||||
"mongodb",
|
||||
),
|
||||
)
|
||||
|
||||
_DIRECT_MODULE_IMPORTS: tuple[tuple[str, str, str, str], ...] = (
|
||||
("agents.extensions.memory.redis_session", "redis.asyncio", "redis", "redis"),
|
||||
("agents.extensions.memory.dapr_session", "dapr.aio.clients", "dapr", "dapr"),
|
||||
(
|
||||
"agents.extensions.memory.mongodb_session",
|
||||
"pymongo.asynchronous.collection",
|
||||
"mongodb",
|
||||
"mongodb",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class _BrokenImportFinder(importlib.abc.MetaPathFinder):
|
||||
def __init__(self, broken_module: str, error_cls: type[ImportError]) -> None:
|
||||
self._broken_module = broken_module
|
||||
self._error_cls = error_cls
|
||||
|
||||
def find_spec(
|
||||
self,
|
||||
fullname: str,
|
||||
path: object | None,
|
||||
target: ModuleType | None = None,
|
||||
) -> None:
|
||||
if fullname == self._broken_module:
|
||||
raise self._error_cls("simulated dependency import failure")
|
||||
return None
|
||||
|
||||
|
||||
def _reset_package_imports(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
memory_module: ModuleType,
|
||||
symbol: str,
|
||||
module_name: str,
|
||||
broken_module: str,
|
||||
) -> None:
|
||||
monkeypatch.delitem(memory_module.__dict__, symbol, raising=False)
|
||||
_reset_loaded_module(monkeypatch, module_name)
|
||||
_reset_loaded_module(monkeypatch, broken_module)
|
||||
|
||||
|
||||
def _reset_loaded_module(monkeypatch: pytest.MonkeyPatch, module_name: str) -> None:
|
||||
monkeypatch.delitem(sys.modules, module_name, raising=False)
|
||||
parent_name, short_name = module_name.rsplit(".", 1)
|
||||
parent_module = sys.modules.get(parent_name)
|
||||
if parent_module is not None:
|
||||
monkeypatch.delitem(parent_module.__dict__, short_name, raising=False)
|
||||
|
||||
|
||||
def _reset_module_imports(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
module_name: str,
|
||||
broken_module: str,
|
||||
) -> None:
|
||||
_reset_loaded_module(monkeypatch, module_name)
|
||||
_reset_loaded_module(monkeypatch, broken_module)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("symbol", "module_name", "broken_module", "dependency_name", "extra_name"),
|
||||
_PACKAGE_EXPORTS,
|
||||
)
|
||||
def test_memory_package_imports_point_to_optional_extra(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
symbol: str,
|
||||
module_name: str,
|
||||
broken_module: str,
|
||||
dependency_name: str,
|
||||
extra_name: str,
|
||||
) -> None:
|
||||
import agents.extensions.memory as memory_module
|
||||
|
||||
_reset_package_imports(monkeypatch, memory_module, symbol, module_name, broken_module)
|
||||
finder = _BrokenImportFinder(broken_module, ModuleNotFoundError)
|
||||
monkeypatch.setattr(sys, "meta_path", [finder, *sys.meta_path])
|
||||
|
||||
with pytest.raises(ImportError) as exc_info:
|
||||
getattr(memory_module, symbol)
|
||||
|
||||
assert f"requires the '{dependency_name}' extra" in str(exc_info.value)
|
||||
assert f"openai-agents[{extra_name}]" in str(exc_info.value)
|
||||
assert isinstance(exc_info.value.__cause__, ImportError)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("module_name", "broken_module", "dependency_name", "extra_name"),
|
||||
_DIRECT_MODULE_IMPORTS,
|
||||
)
|
||||
@pytest.mark.parametrize("error_cls", [ImportError, ModuleNotFoundError])
|
||||
def test_memory_direct_module_imports_point_to_optional_extra(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
module_name: str,
|
||||
broken_module: str,
|
||||
dependency_name: str,
|
||||
extra_name: str,
|
||||
error_cls: type[ImportError],
|
||||
) -> None:
|
||||
_reset_module_imports(monkeypatch, module_name, broken_module)
|
||||
finder = _BrokenImportFinder(broken_module, error_cls)
|
||||
monkeypatch.setattr(sys, "meta_path", [finder, *sys.meta_path])
|
||||
|
||||
with pytest.raises(ImportError) as exc_info:
|
||||
__import__(module_name)
|
||||
|
||||
assert f"requires the '{dependency_name}' extra" in str(exc_info.value)
|
||||
assert f"openai-agents[{extra_name}]" in str(exc_info.value)
|
||||
assert isinstance(exc_info.value.__cause__, ImportError)
|
||||
@@ -0,0 +1,831 @@
|
||||
"""Tests for MongoDBSession using in-process mock objects.
|
||||
|
||||
All tests run without a real MongoDB server — or even the ``pymongo``
|
||||
package — by injecting lightweight fake classes into ``sys.modules``
|
||||
before the module under test is imported. This keeps the suite fast and
|
||||
dependency-free while exercising the full session logic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import types
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agents import Agent, Runner, TResponseInputItem
|
||||
from agents.memory.session_settings import SessionSettings
|
||||
from tests.fake_model import FakeModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# In-memory fake pymongo async types
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class FakeObjectId:
|
||||
"""Minimal ObjectId stand-in with a monotonic counter for sort order."""
|
||||
|
||||
_counter = 0
|
||||
|
||||
def __init__(self) -> None:
|
||||
FakeObjectId._counter += 1
|
||||
self._value = FakeObjectId._counter
|
||||
|
||||
def __lt__(self, other: FakeObjectId) -> bool:
|
||||
return self._value < other._value
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"FakeObjectId({self._value})"
|
||||
|
||||
|
||||
class FakeCursor:
|
||||
"""Minimal async cursor returned by ``find()``."""
|
||||
|
||||
def __init__(self, docs: list[dict[str, Any]]) -> None:
|
||||
self._docs = docs
|
||||
|
||||
def sort(
|
||||
self,
|
||||
key: str | list[tuple[str, int]],
|
||||
direction: int | None = None,
|
||||
) -> FakeCursor:
|
||||
if isinstance(key, list):
|
||||
pairs = key
|
||||
else:
|
||||
direction = direction if direction is not None else 1
|
||||
pairs = [(key, direction)]
|
||||
|
||||
docs = list(self._docs)
|
||||
for field, dir_ in reversed(pairs):
|
||||
docs.sort(key=lambda d: d.get(field, 0), reverse=(dir_ == -1))
|
||||
self._docs = docs
|
||||
return self
|
||||
|
||||
def limit(self, n: int) -> FakeCursor:
|
||||
self._docs = self._docs[:n]
|
||||
return self
|
||||
|
||||
async def to_list(self) -> list[dict[str, Any]]:
|
||||
return list(self._docs)
|
||||
|
||||
|
||||
class FakeAsyncCollection:
|
||||
"""In-memory substitute for pymongo AsyncCollection."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._docs: dict[Any, dict[str, Any]] = {}
|
||||
|
||||
async def create_index(self, keys: Any, **kwargs: Any) -> str:
|
||||
return "fake_index"
|
||||
|
||||
def find(self, query: dict[str, Any] | None = None) -> FakeCursor:
|
||||
query = query or {}
|
||||
results = [doc for doc in self._docs.values() if self._matches(doc, query)]
|
||||
return FakeCursor(results)
|
||||
|
||||
async def find_one_and_delete(
|
||||
self,
|
||||
query: dict[str, Any],
|
||||
sort: list[tuple[str, int]] | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
matches = [doc for doc in self._docs.values() if self._matches(doc, query)]
|
||||
if not matches:
|
||||
return None
|
||||
if sort:
|
||||
field, dir_ = sort[0]
|
||||
matches.sort(key=lambda d: d.get(field, 0), reverse=(dir_ == -1))
|
||||
doc = matches[0]
|
||||
self._docs.pop(id(doc["_id"]))
|
||||
return doc
|
||||
|
||||
async def insert_many(
|
||||
self,
|
||||
documents: list[dict[str, Any]],
|
||||
ordered: bool = True,
|
||||
) -> Any:
|
||||
for doc in documents:
|
||||
if "_id" not in doc:
|
||||
doc["_id"] = FakeObjectId()
|
||||
self._docs[id(doc["_id"])] = dict(doc)
|
||||
|
||||
async def find_one_and_update(
|
||||
self,
|
||||
query: dict[str, Any],
|
||||
update: dict[str, Any],
|
||||
upsert: bool = False,
|
||||
return_document: bool = False,
|
||||
) -> dict[str, Any] | None:
|
||||
for doc in self._docs.values():
|
||||
if self._matches(doc, query):
|
||||
# Apply $inc fields.
|
||||
for field, delta in update.get("$inc", {}).items():
|
||||
doc[field] = doc.get(field, 0) + delta
|
||||
for field, value in update.get("$set", {}).items():
|
||||
doc[field] = value
|
||||
return dict(doc) if return_document else None
|
||||
if upsert:
|
||||
new_doc: dict[str, Any] = {"_id": FakeObjectId()}
|
||||
new_doc.update(update.get("$setOnInsert", {}))
|
||||
new_doc.update(update.get("$set", {}))
|
||||
for field, delta in update.get("$inc", {}).items():
|
||||
new_doc[field] = new_doc.get(field, 0) + delta
|
||||
self._docs[id(new_doc["_id"])] = new_doc
|
||||
return dict(new_doc) if return_document else None
|
||||
return None
|
||||
|
||||
async def update_one(
|
||||
self,
|
||||
query: dict[str, Any],
|
||||
update: dict[str, Any],
|
||||
upsert: bool = False,
|
||||
) -> None:
|
||||
for doc in self._docs.values():
|
||||
if self._matches(doc, query):
|
||||
return # Exists — $setOnInsert is a no-op on existing docs.
|
||||
if upsert:
|
||||
new_doc2: dict[str, Any] = {"_id": FakeObjectId()}
|
||||
new_doc2.update(update.get("$setOnInsert", {}))
|
||||
self._docs[id(new_doc2["_id"])] = new_doc2
|
||||
|
||||
async def delete_many(self, query: dict[str, Any]) -> None:
|
||||
to_remove = [k for k, d in self._docs.items() if self._matches(d, query)]
|
||||
for key in to_remove:
|
||||
del self._docs[key]
|
||||
|
||||
async def delete_one(self, query: dict[str, Any]) -> None:
|
||||
for key, doc in list(self._docs.items()):
|
||||
if self._matches(doc, query):
|
||||
del self._docs[key]
|
||||
return
|
||||
|
||||
@staticmethod
|
||||
def _matches(doc: dict[str, Any], query: dict[str, Any]) -> bool:
|
||||
return all(doc.get(k) == v for k, v in query.items())
|
||||
|
||||
|
||||
class FakeAsyncDatabase:
|
||||
"""In-memory substitute for a pymongo async Database."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._collections: dict[str, FakeAsyncCollection] = defaultdict(FakeAsyncCollection)
|
||||
|
||||
def __getitem__(self, name: str) -> FakeAsyncCollection:
|
||||
return self._collections[name]
|
||||
|
||||
|
||||
class FakeAdminDatabase:
|
||||
"""Minimal admin database used by ping()."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._closed = False
|
||||
|
||||
async def command(self, cmd: str) -> dict[str, Any]:
|
||||
if self._closed:
|
||||
raise ConnectionError("Client is closed.")
|
||||
return {"ok": 1}
|
||||
|
||||
|
||||
class FakeDriverInfo:
|
||||
"""Minimal stand-in for pymongo.driver_info.DriverInfo."""
|
||||
|
||||
def __init__(self, name: str, version: str | None = None) -> None:
|
||||
self.name = name
|
||||
self.version = version
|
||||
|
||||
|
||||
class FakeAsyncMongoClient:
|
||||
"""In-memory substitute for pymongo AsyncMongoClient."""
|
||||
|
||||
def __init__(self, *args: Any, **kwargs: Any) -> None:
|
||||
self._databases: dict[str, FakeAsyncDatabase] = defaultdict(FakeAsyncDatabase)
|
||||
self._closed = False
|
||||
self.admin = FakeAdminDatabase()
|
||||
self._metadata_calls: list[FakeDriverInfo] = []
|
||||
|
||||
def __getitem__(self, name: str) -> FakeAsyncDatabase:
|
||||
return self._databases[name]
|
||||
|
||||
def append_metadata(self, driver_info: FakeDriverInfo) -> None:
|
||||
"""Record append_metadata calls for test assertions."""
|
||||
self._metadata_calls.append(driver_info)
|
||||
|
||||
async def close(self) -> None:
|
||||
"""Async close — matches PyMongo's AsyncMongoClient.close() signature."""
|
||||
self._closed = True
|
||||
self.admin._closed = True
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Inject fake pymongo into sys.modules before importing the module under test
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_fake_pymongo_modules() -> None:
|
||||
"""Populate sys.modules with stub pymongo async modules."""
|
||||
pymongo_mod = sys.modules.get("pymongo") or types.ModuleType("pymongo")
|
||||
|
||||
async_pkg = types.ModuleType("pymongo.asynchronous")
|
||||
collection_mod = types.ModuleType("pymongo.asynchronous.collection")
|
||||
client_mod = types.ModuleType("pymongo.asynchronous.mongo_client")
|
||||
driver_info_mod = types.ModuleType("pymongo.driver_info")
|
||||
|
||||
collection_mod.AsyncCollection = FakeAsyncCollection # type: ignore[attr-defined]
|
||||
client_mod.AsyncMongoClient = FakeAsyncMongoClient # type: ignore[attr-defined]
|
||||
driver_info_mod.DriverInfo = FakeDriverInfo # type: ignore[attr-defined]
|
||||
|
||||
sys.modules["pymongo"] = pymongo_mod
|
||||
sys.modules["pymongo.asynchronous"] = async_pkg
|
||||
sys.modules["pymongo.asynchronous.collection"] = collection_mod
|
||||
sys.modules["pymongo.asynchronous.mongo_client"] = client_mod
|
||||
sys.modules["pymongo.driver_info"] = driver_info_mod
|
||||
|
||||
|
||||
_make_fake_pymongo_modules()
|
||||
|
||||
# Now it's safe to import the module under test.
|
||||
from agents.extensions.memory.mongodb_session import MongoDBSession # noqa: E402
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers / fixtures
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _make_session(session_id: str = "test-session", **kwargs: Any) -> MongoDBSession:
|
||||
"""Create a MongoDBSession backed by a FakeAsyncMongoClient."""
|
||||
client = FakeAsyncMongoClient()
|
||||
MongoDBSession._init_state.clear()
|
||||
return MongoDBSession(
|
||||
session_id,
|
||||
client=client, # type: ignore[arg-type]
|
||||
database="agents_test",
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def session() -> MongoDBSession:
|
||||
return _make_session()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def agent() -> Agent:
|
||||
return Agent(name="test", model=FakeModel())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Core CRUD tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_add_and_get_items(session: MongoDBSession) -> None:
|
||||
"""Items added to the session are retrievable in insertion order."""
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there!"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
retrieved = await session.get_items()
|
||||
assert len(retrieved) == 2
|
||||
assert retrieved[0].get("content") == "Hello"
|
||||
assert retrieved[1].get("content") == "Hi there!"
|
||||
|
||||
|
||||
async def test_add_empty_list_is_noop(session: MongoDBSession) -> None:
|
||||
"""Adding an empty list must not create any documents."""
|
||||
await session.add_items([])
|
||||
assert await session.get_items() == []
|
||||
|
||||
|
||||
async def test_get_items_empty_session(session: MongoDBSession) -> None:
|
||||
"""Retrieving items from a brand-new session returns an empty list."""
|
||||
assert await session.get_items() == []
|
||||
|
||||
|
||||
async def test_pop_item_returns_last(session: MongoDBSession) -> None:
|
||||
"""pop_item must return and remove the most recently added item."""
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "first"},
|
||||
{"role": "assistant", "content": "second"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
popped = await session.pop_item()
|
||||
assert popped is not None
|
||||
assert popped.get("content") == "second"
|
||||
|
||||
remaining = await session.get_items()
|
||||
assert len(remaining) == 1
|
||||
assert remaining[0].get("content") == "first"
|
||||
|
||||
|
||||
async def test_pop_item_empty_session(session: MongoDBSession) -> None:
|
||||
"""pop_item on an empty session must return None."""
|
||||
assert await session.pop_item() is None
|
||||
|
||||
|
||||
async def test_clear_session(session: MongoDBSession) -> None:
|
||||
"""clear_session must remove all items and session metadata."""
|
||||
await session.add_items([{"role": "user", "content": "x"}])
|
||||
await session.clear_session()
|
||||
assert await session.get_items() == []
|
||||
|
||||
|
||||
async def test_multiple_add_calls_accumulate(session: MongoDBSession) -> None:
|
||||
"""Items from separate add_items calls all appear in get_items."""
|
||||
await session.add_items([{"role": "user", "content": "a"}])
|
||||
await session.add_items([{"role": "assistant", "content": "b"}])
|
||||
await session.add_items([{"role": "user", "content": "c"}])
|
||||
|
||||
items = await session.get_items()
|
||||
assert [i.get("content") for i in items] == ["a", "b", "c"]
|
||||
|
||||
|
||||
async def test_session_metadata_timestamps_are_written(session: MongoDBSession) -> None:
|
||||
"""Session metadata records creation time and last update time."""
|
||||
created_at = datetime(2026, 1, 1, tzinfo=timezone.utc)
|
||||
updated_at = datetime(2026, 1, 2, tzinfo=timezone.utc)
|
||||
|
||||
with patch("agents.extensions.memory.mongodb_session.datetime") as mocked_datetime:
|
||||
mocked_datetime.now.side_effect = [created_at, updated_at]
|
||||
|
||||
await session.add_items([{"role": "user", "content": "first"}])
|
||||
session_doc: dict[str, Any] = next(iter(session._sessions._docs.values()))
|
||||
assert session_doc["session_id"] == session.session_id
|
||||
assert session_doc["created_at"] == created_at
|
||||
assert session_doc["updated_at"] == created_at
|
||||
|
||||
await session.add_items([{"role": "assistant", "content": "second"}])
|
||||
assert session_doc["created_at"] == created_at
|
||||
assert session_doc["updated_at"] == updated_at
|
||||
assert session_doc["_seq"] == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Limit / SessionSettings tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_get_items_with_explicit_limit(session: MongoDBSession) -> None:
|
||||
"""Explicit limit returns the N most recent items in chronological order."""
|
||||
await session.add_items([{"role": "user", "content": str(i)} for i in range(6)])
|
||||
|
||||
result = await session.get_items(limit=3)
|
||||
assert len(result) == 3
|
||||
assert [r.get("content") for r in result] == ["3", "4", "5"]
|
||||
|
||||
|
||||
async def test_get_items_limit_zero(session: MongoDBSession) -> None:
|
||||
"""A limit of 0 must return an empty list immediately."""
|
||||
await session.add_items([{"role": "user", "content": "x"}])
|
||||
assert await session.get_items(limit=0) == []
|
||||
|
||||
|
||||
async def test_get_items_limit_exceeds_count(session: MongoDBSession) -> None:
|
||||
"""Requesting more items than exist returns all items without error."""
|
||||
await session.add_items([{"role": "user", "content": "only"}])
|
||||
result = await session.get_items(limit=100)
|
||||
assert len(result) == 1
|
||||
|
||||
|
||||
async def test_session_settings_limit_used_as_default() -> None:
|
||||
"""session_settings.limit is applied when no explicit limit is given."""
|
||||
MongoDBSession._init_state.clear()
|
||||
s = MongoDBSession(
|
||||
"ls-test",
|
||||
client=FakeAsyncMongoClient(), # type: ignore[arg-type]
|
||||
database="agents_test",
|
||||
session_settings=SessionSettings(limit=2),
|
||||
)
|
||||
await s.add_items([{"role": "user", "content": str(i)} for i in range(5)])
|
||||
|
||||
result = await s.get_items()
|
||||
assert len(result) == 2
|
||||
assert result[0].get("content") == "3"
|
||||
assert result[1].get("content") == "4"
|
||||
|
||||
|
||||
async def test_explicit_limit_overrides_session_settings() -> None:
|
||||
"""An explicit limit passed to get_items must override session_settings.limit."""
|
||||
MongoDBSession._init_state.clear()
|
||||
s = MongoDBSession(
|
||||
"override-test",
|
||||
client=FakeAsyncMongoClient(), # type: ignore[arg-type]
|
||||
database="agents_test",
|
||||
session_settings=SessionSettings(limit=10),
|
||||
)
|
||||
await s.add_items([{"role": "user", "content": str(i)} for i in range(8)])
|
||||
|
||||
result = await s.get_items(limit=2)
|
||||
assert len(result) == 2
|
||||
assert result[0].get("content") == "6"
|
||||
assert result[1].get("content") == "7"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Session isolation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_sessions_are_isolated() -> None:
|
||||
"""Two sessions with different IDs must not share data."""
|
||||
MongoDBSession._init_state.clear()
|
||||
client = FakeAsyncMongoClient()
|
||||
s1 = MongoDBSession("alice", client=client, database="agents_test") # type: ignore[arg-type]
|
||||
s2 = MongoDBSession("bob", client=client, database="agents_test") # type: ignore[arg-type]
|
||||
|
||||
await s1.add_items([{"role": "user", "content": "alice msg"}])
|
||||
await s2.add_items([{"role": "user", "content": "bob msg"}])
|
||||
|
||||
assert [i.get("content") for i in await s1.get_items()] == ["alice msg"]
|
||||
assert [i.get("content") for i in await s2.get_items()] == ["bob msg"]
|
||||
|
||||
|
||||
async def test_clear_does_not_affect_other_sessions() -> None:
|
||||
"""Clearing one session must leave sibling sessions untouched."""
|
||||
MongoDBSession._init_state.clear()
|
||||
client = FakeAsyncMongoClient()
|
||||
s1 = MongoDBSession("s1", client=client, database="agents_test") # type: ignore[arg-type]
|
||||
s2 = MongoDBSession("s2", client=client, database="agents_test") # type: ignore[arg-type]
|
||||
|
||||
await s1.add_items([{"role": "user", "content": "keep"}])
|
||||
await s2.add_items([{"role": "user", "content": "delete"}])
|
||||
|
||||
await s2.clear_session()
|
||||
|
||||
assert len(await s1.get_items()) == 1
|
||||
assert await s2.get_items() == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Serialisation / unicode safety
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_unicode_content_roundtrip(session: MongoDBSession) -> None:
|
||||
"""Unicode and emoji content must survive the serialisation round-trip."""
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "こんにちは"},
|
||||
{"role": "assistant", "content": "😊👍"},
|
||||
{"role": "user", "content": "Привет"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
result = await session.get_items()
|
||||
assert result[0].get("content") == "こんにちは"
|
||||
assert result[1].get("content") == "😊👍"
|
||||
assert result[2].get("content") == "Привет"
|
||||
|
||||
|
||||
async def test_json_special_characters(session: MongoDBSession) -> None:
|
||||
"""Items containing JSON-special strings must be stored without corruption."""
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": '{"nested": "value"}'},
|
||||
{"role": "assistant", "content": 'Quote: "Hello"'},
|
||||
{"role": "user", "content": "Line1\nLine2\tTabbed"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
result = await session.get_items()
|
||||
assert result[0].get("content") == '{"nested": "value"}'
|
||||
assert result[1].get("content") == 'Quote: "Hello"'
|
||||
assert result[2].get("content") == "Line1\nLine2\tTabbed"
|
||||
|
||||
|
||||
async def test_corrupted_document_is_skipped(session: MongoDBSession) -> None:
|
||||
"""Documents with invalid JSON in message_data are silently skipped."""
|
||||
await session.add_items([{"role": "user", "content": "valid"}])
|
||||
|
||||
# Inject a corrupted document directly into the fake collection.
|
||||
bad_doc = {
|
||||
"_id": FakeObjectId(),
|
||||
"session_id": session.session_id,
|
||||
"message_data": "not valid json {{{",
|
||||
}
|
||||
session._messages._docs[id(bad_doc["_id"])] = bad_doc
|
||||
|
||||
items = await session.get_items()
|
||||
assert len(items) == 1
|
||||
assert items[0].get("content") == "valid"
|
||||
|
||||
|
||||
async def test_missing_message_data_field_is_skipped(session: MongoDBSession) -> None:
|
||||
"""Documents without a message_data field are silently skipped."""
|
||||
await session.add_items([{"role": "user", "content": "valid"}])
|
||||
|
||||
bad_doc = {"_id": FakeObjectId(), "session_id": session.session_id}
|
||||
session._messages._docs[id(bad_doc["_id"])] = bad_doc
|
||||
|
||||
items = await session.get_items()
|
||||
assert len(items) == 1
|
||||
|
||||
|
||||
async def test_non_string_message_data_is_skipped(session: MongoDBSession) -> None:
|
||||
"""Documents whose message_data is a non-string BSON type are silently skipped."""
|
||||
await session.add_items([{"role": "user", "content": "valid"}])
|
||||
|
||||
# Inject a document where message_data is an integer — json.loads raises TypeError.
|
||||
bad_doc = {"_id": FakeObjectId(), "session_id": session.session_id, "message_data": 42}
|
||||
session._messages._docs[id(bad_doc["_id"])] = bad_doc
|
||||
|
||||
items = await session.get_items()
|
||||
assert len(items) == 1
|
||||
assert items[0].get("content") == "valid"
|
||||
|
||||
|
||||
async def test_pop_item_skips_corrupt_most_recent(session: MongoDBSession) -> None:
|
||||
"""pop_item must skip a corrupt most-recent document and return the next valid one."""
|
||||
await session.add_items([{"role": "user", "content": "valid"}])
|
||||
|
||||
# Inject a corrupt document with a higher seq so it sorts as "most recent".
|
||||
bad_doc = {
|
||||
"_id": FakeObjectId(),
|
||||
"session_id": session.session_id,
|
||||
"seq": 999,
|
||||
"message_data": "not valid json {{{",
|
||||
}
|
||||
session._messages._docs[id(bad_doc["_id"])] = bad_doc
|
||||
|
||||
popped = await session.pop_item()
|
||||
assert popped is not None
|
||||
assert popped.get("content") == "valid"
|
||||
|
||||
# Both the corrupt doc and the valid one are now gone.
|
||||
assert await session.get_items() == []
|
||||
|
||||
|
||||
async def test_pop_item_returns_none_when_only_corrupt_docs_remain(
|
||||
session: MongoDBSession,
|
||||
) -> None:
|
||||
"""pop_item must drop every corrupt doc and return None when nothing valid remains."""
|
||||
bad1 = {
|
||||
"_id": FakeObjectId(),
|
||||
"session_id": session.session_id,
|
||||
"seq": 1,
|
||||
"message_data": "garbage",
|
||||
}
|
||||
bad2 = {
|
||||
"_id": FakeObjectId(),
|
||||
"session_id": session.session_id,
|
||||
"seq": 2,
|
||||
"message_data": 42, # non-string — TypeError
|
||||
}
|
||||
session._messages._docs[id(bad1["_id"])] = bad1
|
||||
session._messages._docs[id(bad2["_id"])] = bad2
|
||||
|
||||
assert await session.pop_item() is None
|
||||
# Both corrupt docs must have been removed in the process.
|
||||
assert session._messages._docs == {}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Index initialisation (idempotency)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_index_creation_runs_only_once(session: MongoDBSession) -> None:
|
||||
"""_ensure_indexes must call create_index only on the very first call."""
|
||||
call_count = 0
|
||||
original_messages = session._messages.create_index
|
||||
original_sessions = session._sessions.create_index
|
||||
|
||||
async def counting(*args: Any, **kwargs: Any) -> str:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return "fake_index"
|
||||
|
||||
session._messages.create_index = counting # type: ignore[method-assign]
|
||||
session._sessions.create_index = counting # type: ignore[method-assign]
|
||||
|
||||
await session._ensure_indexes()
|
||||
await session._ensure_indexes() # Second call must be a no-op.
|
||||
|
||||
# Exactly one call per collection (sessions + messages).
|
||||
assert call_count == 2
|
||||
|
||||
session._messages.create_index = original_messages # type: ignore[method-assign]
|
||||
session._sessions.create_index = original_sessions # type: ignore[method-assign]
|
||||
|
||||
|
||||
async def test_different_clients_each_run_index_init() -> None:
|
||||
"""Each distinct AsyncMongoClient gets its own index-creation pass."""
|
||||
MongoDBSession._init_state.clear()
|
||||
|
||||
client_a = FakeAsyncMongoClient()
|
||||
client_b = FakeAsyncMongoClient()
|
||||
|
||||
call_counts: dict[str, int] = {"a": 0, "b": 0}
|
||||
|
||||
async def counting_a(*args: Any, **kwargs: Any) -> str:
|
||||
call_counts["a"] += 1
|
||||
return "fake_index"
|
||||
|
||||
async def counting_b(*args: Any, **kwargs: Any) -> str:
|
||||
call_counts["b"] += 1
|
||||
return "fake_index"
|
||||
|
||||
s_a = MongoDBSession("x", client=client_a, database="agents_test") # type: ignore[arg-type]
|
||||
s_b = MongoDBSession("x", client=client_b, database="agents_test") # type: ignore[arg-type]
|
||||
|
||||
s_a._messages.create_index = counting_a # type: ignore[method-assign]
|
||||
s_a._sessions.create_index = counting_a # type: ignore[method-assign]
|
||||
s_b._messages.create_index = counting_b # type: ignore[method-assign]
|
||||
s_b._sessions.create_index = counting_b # type: ignore[method-assign]
|
||||
|
||||
await s_a._ensure_indexes()
|
||||
await s_b._ensure_indexes()
|
||||
|
||||
# Each client must trigger its own index creation (2 calls = sessions + messages).
|
||||
assert call_counts["a"] == 2
|
||||
assert call_counts["b"] == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Connectivity and lifecycle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_ping_success(session: MongoDBSession) -> None:
|
||||
"""ping() must return True when the client responds normally."""
|
||||
assert await session.ping() is True
|
||||
|
||||
|
||||
async def test_ping_failure(session: MongoDBSession) -> None:
|
||||
"""ping() must return False when the server raises an exception."""
|
||||
original = session._client.admin.command
|
||||
|
||||
async def _fail(*args: Any, **kwargs: Any) -> dict[str, Any]:
|
||||
raise ConnectionError("unreachable")
|
||||
|
||||
session._client.admin.command = _fail # type: ignore[method-assign, assignment]
|
||||
assert await session.ping() is False
|
||||
session._client.admin.command = original # type: ignore[method-assign]
|
||||
|
||||
|
||||
async def test_close_external_client_not_closed() -> None:
|
||||
"""close() must NOT close a client that was injected externally."""
|
||||
MongoDBSession._init_state.clear()
|
||||
client = FakeAsyncMongoClient()
|
||||
s = MongoDBSession("x", client=client, database="agents_test") # type: ignore[arg-type]
|
||||
assert s._owns_client is False
|
||||
|
||||
await s.close()
|
||||
assert not client._closed
|
||||
|
||||
|
||||
async def test_close_owned_client_is_closed() -> None:
|
||||
"""close() must close a client created by from_uri."""
|
||||
MongoDBSession._init_state.clear()
|
||||
fake_client = FakeAsyncMongoClient()
|
||||
with patch(
|
||||
"agents.extensions.memory.mongodb_session.AsyncMongoClient",
|
||||
return_value=fake_client,
|
||||
):
|
||||
s = MongoDBSession.from_uri("owned", uri="mongodb://localhost:27017", database="t")
|
||||
assert s._owns_client is True
|
||||
|
||||
await s.close()
|
||||
assert fake_client._closed
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Runner integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_runner_integration(agent: Agent) -> None:
|
||||
"""MongoDBSession must supply conversation history to the Runner."""
|
||||
session = _make_session("runner-test")
|
||||
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("San Francisco")])
|
||||
result1 = await Runner.run(agent, "Where is the Golden Gate Bridge?", session=session)
|
||||
assert result1.final_output == "San Francisco"
|
||||
|
||||
agent.model.set_next_output([get_text_message("California")])
|
||||
result2 = await Runner.run(agent, "What state is it in?", session=session)
|
||||
assert result2.final_output == "California"
|
||||
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
assert len(last_input) > 1
|
||||
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
|
||||
|
||||
|
||||
async def test_runner_session_isolation(agent: Agent) -> None:
|
||||
"""Two independent sessions must not bleed history into each other."""
|
||||
MongoDBSession._init_state.clear()
|
||||
client = FakeAsyncMongoClient()
|
||||
s1 = MongoDBSession("user-a", client=client, database="agents_test") # type: ignore[arg-type]
|
||||
s2 = MongoDBSession("user-b", client=client, database="agents_test") # type: ignore[arg-type]
|
||||
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("I like cats.")])
|
||||
await Runner.run(agent, "I like cats.", session=s1)
|
||||
|
||||
agent.model.set_next_output([get_text_message("I like dogs.")])
|
||||
await Runner.run(agent, "I like dogs.", session=s2)
|
||||
|
||||
agent.model.set_next_output([get_text_message("You said you like cats.")])
|
||||
result = await Runner.run(agent, "What animal did I mention?", session=s1)
|
||||
assert "cats" in result.final_output.lower()
|
||||
assert "dogs" not in result.final_output.lower()
|
||||
|
||||
|
||||
async def test_runner_with_session_settings_limit(agent: Agent) -> None:
|
||||
"""RunConfig.session_settings.limit must cap the history sent to the model."""
|
||||
from agents import RunConfig
|
||||
|
||||
MongoDBSession._init_state.clear()
|
||||
session = MongoDBSession(
|
||||
"limit-test",
|
||||
client=FakeAsyncMongoClient(), # type: ignore[arg-type]
|
||||
database="agents_test",
|
||||
session_settings=SessionSettings(limit=100),
|
||||
)
|
||||
|
||||
history: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": f"Turn {i}"} for i in range(10)
|
||||
]
|
||||
await session.add_items(history)
|
||||
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("Got it")])
|
||||
await Runner.run(
|
||||
agent,
|
||||
"New question",
|
||||
session=session,
|
||||
run_config=RunConfig(session_settings=SessionSettings(limit=2)),
|
||||
)
|
||||
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
history_items = [i for i in last_input if i.get("content") != "New question"]
|
||||
assert len(history_items) == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Client metadata (driver handshake)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_injected_client_receives_append_metadata() -> None:
|
||||
"""Append_metadata is called on a caller-supplied client."""
|
||||
MongoDBSession._init_state.clear()
|
||||
client = FakeAsyncMongoClient()
|
||||
|
||||
MongoDBSession("meta-test", client=client, database="agents_test") # type: ignore[arg-type]
|
||||
|
||||
assert len(client._metadata_calls) == 1
|
||||
info = client._metadata_calls[0]
|
||||
assert info.name == "openai-agents"
|
||||
|
||||
|
||||
async def test_from_uri_passes_driver_info_to_constructor() -> None:
|
||||
"""driver=_DRIVER_INFO is forwarded to AsyncMongoClient via from_uri."""
|
||||
MongoDBSession._init_state.clear()
|
||||
|
||||
captured_kwargs: dict[str, Any] = {}
|
||||
|
||||
def _fake_client(uri: str, **kwargs: Any) -> FakeAsyncMongoClient:
|
||||
captured_kwargs.update(kwargs)
|
||||
return FakeAsyncMongoClient()
|
||||
|
||||
with patch(
|
||||
"agents.extensions.memory.mongodb_session.AsyncMongoClient",
|
||||
side_effect=_fake_client,
|
||||
):
|
||||
MongoDBSession.from_uri("uri-test", uri="mongodb://localhost:27017", database="t")
|
||||
|
||||
assert "driver" in captured_kwargs
|
||||
assert captured_kwargs["driver"].name == "openai-agents"
|
||||
|
||||
|
||||
async def test_caller_supplied_driver_info_is_not_overwritten() -> None:
|
||||
"""A caller-supplied driver kwarg must not be silently replaced."""
|
||||
MongoDBSession._init_state.clear()
|
||||
|
||||
captured_kwargs: dict[str, Any] = {}
|
||||
custom_info = FakeDriverInfo(name="MyApp")
|
||||
|
||||
def _fake_client(uri: str, **kwargs: Any) -> FakeAsyncMongoClient:
|
||||
captured_kwargs.update(kwargs)
|
||||
return FakeAsyncMongoClient()
|
||||
|
||||
with patch(
|
||||
"agents.extensions.memory.mongodb_session.AsyncMongoClient",
|
||||
side_effect=_fake_client,
|
||||
):
|
||||
MongoDBSession.from_uri(
|
||||
"uri-test",
|
||||
uri="mongodb://localhost:27017",
|
||||
database="t",
|
||||
client_kwargs={"driver": custom_info},
|
||||
)
|
||||
|
||||
# The caller's value must be preserved — setdefault must not overwrite it.
|
||||
assert captured_kwargs["driver"] is custom_info
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,956 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import threading
|
||||
from collections.abc import Iterable, Sequence
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
from openai.types.responses.response_output_message_param import ResponseOutputMessageParam
|
||||
from openai.types.responses.response_output_text_param import ResponseOutputTextParam
|
||||
from openai.types.responses.response_reasoning_item_param import (
|
||||
ResponseReasoningItemParam,
|
||||
Summary,
|
||||
)
|
||||
from sqlalchemy import insert, select, text, update
|
||||
from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine
|
||||
from sqlalchemy.sql import Select
|
||||
|
||||
pytest.importorskip("sqlalchemy") # Skip tests if SQLAlchemy is not installed
|
||||
|
||||
from agents import Agent, Runner, TResponseInputItem
|
||||
from agents.extensions.memory.sqlalchemy_session import SQLAlchemySession
|
||||
from tests.fake_model import FakeModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
# Mark all tests in this file as asyncio
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
# Use in-memory SQLite for tests
|
||||
DB_URL = "sqlite+aiosqlite:///:memory:"
|
||||
|
||||
|
||||
def _make_message_item(item_id: str, text_value: str) -> TResponseInputItem:
|
||||
content: ResponseOutputTextParam = {
|
||||
"type": "output_text",
|
||||
"text": text_value,
|
||||
"annotations": [],
|
||||
"logprobs": [],
|
||||
}
|
||||
message: ResponseOutputMessageParam = {
|
||||
"id": item_id,
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [content],
|
||||
}
|
||||
return cast(TResponseInputItem, message)
|
||||
|
||||
|
||||
def _make_reasoning_item(item_id: str, summary_text: str) -> TResponseInputItem:
|
||||
summary: Summary = {"type": "summary_text", "text": summary_text}
|
||||
reasoning: ResponseReasoningItemParam = {
|
||||
"id": item_id,
|
||||
"type": "reasoning",
|
||||
"summary": [summary],
|
||||
}
|
||||
return cast(TResponseInputItem, reasoning)
|
||||
|
||||
|
||||
def _item_ids(items: Sequence[TResponseInputItem]) -> list[str]:
|
||||
result: list[str] = []
|
||||
for item in items:
|
||||
item_dict = cast(dict[str, Any], item)
|
||||
result.append(cast(str, item_dict["id"]))
|
||||
return result
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def agent() -> Agent:
|
||||
"""Fixture for a basic agent with a fake model."""
|
||||
return Agent(name="test", model=FakeModel())
|
||||
|
||||
|
||||
async def test_sqlalchemy_session_direct_ops(agent: Agent):
|
||||
"""Test direct database operations of SQLAlchemySession."""
|
||||
session_id = "direct_ops_test"
|
||||
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
|
||||
|
||||
# 1. Add items
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there!"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
# 2. Get items and verify
|
||||
retrieved = await session.get_items()
|
||||
assert len(retrieved) == 2
|
||||
assert retrieved[0].get("content") == "Hello"
|
||||
assert retrieved[1].get("content") == "Hi there!"
|
||||
|
||||
# 3. Pop item
|
||||
popped = await session.pop_item()
|
||||
assert popped is not None
|
||||
assert popped.get("content") == "Hi there!"
|
||||
retrieved_after_pop = await session.get_items()
|
||||
assert len(retrieved_after_pop) == 1
|
||||
assert retrieved_after_pop[0].get("content") == "Hello"
|
||||
|
||||
# 4. Clear session
|
||||
await session.clear_session()
|
||||
retrieved_after_clear = await session.get_items()
|
||||
assert len(retrieved_after_clear) == 0
|
||||
|
||||
|
||||
async def test_sqlalchemy_session_defaults_to_escaped_non_ascii_storage():
|
||||
"""Default storage keeps the historical escaped non-ASCII JSON representation."""
|
||||
session = SQLAlchemySession.from_url("default_ascii_storage", url=DB_URL, create_tables=True)
|
||||
item: TResponseInputItem = {"role": "user", "content": "café"}
|
||||
|
||||
await session.add_items([item])
|
||||
|
||||
async with session._session_factory() as sess:
|
||||
rows = await sess.execute(
|
||||
select(session._messages.c.message_data).where(
|
||||
session._messages.c.session_id == session.session_id
|
||||
)
|
||||
)
|
||||
stored = rows.scalar_one()
|
||||
|
||||
assert "\\u00e9" in stored
|
||||
assert "café" not in stored
|
||||
assert await session.get_items() == [item]
|
||||
|
||||
|
||||
async def test_sqlalchemy_session_can_store_non_ascii_without_escaping():
|
||||
"""ensure_ascii=False stores multilingual content readably while preserving round-trip data."""
|
||||
session = SQLAlchemySession.from_url(
|
||||
"non_ascii_storage",
|
||||
url=DB_URL,
|
||||
create_tables=True,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
item: TResponseInputItem = {"role": "user", "content": "café"}
|
||||
|
||||
await session.add_items([item])
|
||||
|
||||
async with session._session_factory() as sess:
|
||||
rows = await sess.execute(
|
||||
select(session._messages.c.message_data).where(
|
||||
session._messages.c.session_id == session.session_id
|
||||
)
|
||||
)
|
||||
stored = rows.scalar_one()
|
||||
|
||||
assert "café" in stored
|
||||
assert "\\u00e9" not in stored
|
||||
assert await session.get_items() == [item]
|
||||
|
||||
|
||||
async def test_runner_integration(agent: Agent):
|
||||
"""Test that SQLAlchemySession works correctly with the agent Runner."""
|
||||
session_id = "runner_integration_test"
|
||||
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
|
||||
|
||||
# First turn
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("San Francisco")])
|
||||
result1 = await Runner.run(
|
||||
agent,
|
||||
"What city is the Golden Gate Bridge in?",
|
||||
session=session,
|
||||
)
|
||||
assert result1.final_output == "San Francisco"
|
||||
|
||||
# Second turn
|
||||
agent.model.set_next_output([get_text_message("California")])
|
||||
result2 = await Runner.run(agent, "What state is it in?", session=session)
|
||||
assert result2.final_output == "California"
|
||||
|
||||
# Verify history was passed to the model on the second turn
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
assert len(last_input) > 1
|
||||
assert any("Golden Gate Bridge" in str(item.get("content", "")) for item in last_input)
|
||||
|
||||
|
||||
async def test_session_isolation(agent: Agent):
|
||||
"""Test that different session IDs result in isolated conversation histories."""
|
||||
session_id_1 = "session_1"
|
||||
session1 = SQLAlchemySession.from_url(session_id_1, url=DB_URL, create_tables=True)
|
||||
|
||||
session_id_2 = "session_2"
|
||||
session2 = SQLAlchemySession.from_url(session_id_2, url=DB_URL, create_tables=True)
|
||||
|
||||
# Interact with session 1
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("I like cats.")])
|
||||
await Runner.run(agent, "I like cats.", session=session1)
|
||||
|
||||
# Interact with session 2
|
||||
agent.model.set_next_output([get_text_message("I like dogs.")])
|
||||
await Runner.run(agent, "I like dogs.", session=session2)
|
||||
|
||||
# Go back to session 1 and check its memory
|
||||
agent.model.set_next_output([get_text_message("You said you like cats.")])
|
||||
result = await Runner.run(agent, "What animal did I say I like?", session=session1)
|
||||
assert "cats" in result.final_output.lower()
|
||||
assert "dogs" not in result.final_output.lower()
|
||||
|
||||
|
||||
async def test_get_items_with_limit(agent: Agent):
|
||||
"""Test the limit parameter in get_items."""
|
||||
session_id = "limit_test"
|
||||
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
|
||||
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "1"},
|
||||
{"role": "assistant", "content": "2"},
|
||||
{"role": "user", "content": "3"},
|
||||
{"role": "assistant", "content": "4"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
# Get last 2 items
|
||||
latest_2 = await session.get_items(limit=2)
|
||||
assert len(latest_2) == 2
|
||||
assert latest_2[0].get("content") == "3"
|
||||
assert latest_2[1].get("content") == "4"
|
||||
|
||||
# Get all items
|
||||
all_items = await session.get_items()
|
||||
assert len(all_items) == 4
|
||||
|
||||
# Get more than available
|
||||
more_than_all = await session.get_items(limit=10)
|
||||
assert len(more_than_all) == 4
|
||||
|
||||
|
||||
async def test_pop_from_empty_session():
|
||||
"""Test that pop_item returns None on an empty session."""
|
||||
session = SQLAlchemySession.from_url("empty_session", url=DB_URL, create_tables=True)
|
||||
popped = await session.pop_item()
|
||||
assert popped is None
|
||||
|
||||
|
||||
async def test_pop_item_skips_corrupt_most_recent():
|
||||
"""pop_item skips corrupt newest rows and returns the next valid item."""
|
||||
session = SQLAlchemySession.from_url("pop_corrupt", url=DB_URL, create_tables=True)
|
||||
|
||||
valid_item: TResponseInputItem = {"role": "user", "content": "valid"}
|
||||
await session.add_items([valid_item])
|
||||
|
||||
await session._ensure_tables()
|
||||
async with session._session_factory() as sess:
|
||||
async with sess.begin():
|
||||
await sess.execute(
|
||||
insert(session._messages).values(
|
||||
{"session_id": session.session_id, "message_data": "not valid json {{{"}
|
||||
)
|
||||
)
|
||||
|
||||
assert await session.pop_item() == valid_item
|
||||
assert await session.get_items() == []
|
||||
|
||||
|
||||
async def test_pop_item_returns_none_after_dropping_only_corrupt_rows():
|
||||
"""pop_item removes corrupt rows and returns None when no valid items remain."""
|
||||
session = SQLAlchemySession.from_url("pop_only_corrupt", url=DB_URL, create_tables=True)
|
||||
|
||||
await session._ensure_tables()
|
||||
async with session._session_factory() as sess:
|
||||
async with sess.begin():
|
||||
await sess.execute(
|
||||
insert(session._messages).values(
|
||||
{"session_id": session.session_id, "message_data": "not valid json {{{"}
|
||||
)
|
||||
)
|
||||
|
||||
assert await session.pop_item() is None
|
||||
assert await session.get_items() == []
|
||||
|
||||
|
||||
async def test_add_empty_items_list():
|
||||
"""Test that adding an empty list of items is a no-op."""
|
||||
session_id = "add_empty_test"
|
||||
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
|
||||
|
||||
initial_items = await session.get_items()
|
||||
assert len(initial_items) == 0
|
||||
|
||||
await session.add_items([])
|
||||
|
||||
items_after_add = await session.get_items()
|
||||
assert len(items_after_add) == 0
|
||||
|
||||
|
||||
async def test_add_items_concurrent_first_access_with_create_tables(tmp_path):
|
||||
"""Concurrent first writes should not race table creation or drop items."""
|
||||
db_url = f"sqlite+aiosqlite:///{tmp_path / 'concurrent_first_access.db'}"
|
||||
session = SQLAlchemySession.from_url(
|
||||
"concurrent_first_access",
|
||||
url=db_url,
|
||||
create_tables=True,
|
||||
)
|
||||
submitted = [f"msg-{i}" for i in range(25)]
|
||||
|
||||
async def worker(content: str) -> None:
|
||||
await session.add_items([{"role": "user", "content": content}])
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(worker(content) for content in submitted),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
assert [result for result in results if isinstance(result, Exception)] == []
|
||||
|
||||
stored = await session.get_items()
|
||||
assert len(stored) == len(submitted)
|
||||
stored_contents: list[str] = []
|
||||
for item in stored:
|
||||
content = item.get("content")
|
||||
assert isinstance(content, str)
|
||||
stored_contents.append(content)
|
||||
assert sorted(stored_contents) == sorted(submitted)
|
||||
|
||||
|
||||
async def test_add_items_concurrent_first_write_after_tables_exist(tmp_path):
|
||||
"""Concurrent first writes should not race parent session creation."""
|
||||
db_url = f"sqlite+aiosqlite:///{tmp_path / 'concurrent_first_write.db'}"
|
||||
setup_session = SQLAlchemySession.from_url(
|
||||
"concurrent_first_write",
|
||||
url=db_url,
|
||||
create_tables=True,
|
||||
)
|
||||
await setup_session.get_items()
|
||||
|
||||
session = SQLAlchemySession.from_url(
|
||||
"concurrent_first_write",
|
||||
url=db_url,
|
||||
create_tables=False,
|
||||
)
|
||||
submitted = [f"msg-{i}" for i in range(25)]
|
||||
|
||||
async def worker(content: str) -> None:
|
||||
await session.add_items([{"role": "user", "content": content}])
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(worker(content) for content in submitted),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
assert [result for result in results if isinstance(result, Exception)] == []
|
||||
|
||||
stored = await session.get_items()
|
||||
assert len(stored) == len(submitted)
|
||||
stored_contents: list[str] = []
|
||||
for item in stored:
|
||||
content = item.get("content")
|
||||
assert isinstance(content, str)
|
||||
stored_contents.append(content)
|
||||
assert sorted(stored_contents) == sorted(submitted)
|
||||
|
||||
|
||||
async def test_add_items_waits_for_transient_sqlite_write_lock(tmp_path):
|
||||
"""SQLite writes should wait briefly for a transient lock instead of failing."""
|
||||
db_url = f"sqlite+aiosqlite:///{tmp_path / 'sqlite_write_lock_retry.db'}"
|
||||
session = SQLAlchemySession.from_url(
|
||||
"sqlite_write_lock_retry",
|
||||
url=db_url,
|
||||
create_tables=True,
|
||||
)
|
||||
await session.get_items()
|
||||
|
||||
async with session.engine.connect() as conn:
|
||||
await conn.execute(text("BEGIN IMMEDIATE"))
|
||||
blocked_write = asyncio.create_task(
|
||||
session.add_items([{"role": "user", "content": "after-lock"}])
|
||||
)
|
||||
await asyncio.sleep(0.1)
|
||||
await conn.rollback()
|
||||
|
||||
await asyncio.wait_for(blocked_write, timeout=5)
|
||||
|
||||
stored = await session.get_items()
|
||||
assert len(stored) == 1
|
||||
assert stored[0].get("content") == "after-lock"
|
||||
|
||||
|
||||
async def test_add_items_concurrent_first_access_across_sessions_with_shared_engine(tmp_path):
|
||||
"""Concurrent first writes should not race table creation across session instances."""
|
||||
db_url = f"sqlite+aiosqlite:///{tmp_path / 'concurrent_shared_engine.db'}"
|
||||
engine = create_async_engine(db_url)
|
||||
try:
|
||||
session_a = SQLAlchemySession("shared_engine_a", engine=engine, create_tables=True)
|
||||
session_b = SQLAlchemySession("shared_engine_b", engine=engine, create_tables=True)
|
||||
|
||||
results = await asyncio.gather(
|
||||
session_a.add_items([{"role": "user", "content": "one"}]),
|
||||
session_b.add_items([{"role": "user", "content": "two"}]),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
assert [result for result in results if isinstance(result, Exception)] == []
|
||||
|
||||
stored_a = await session_a.get_items()
|
||||
assert len(stored_a) == 1
|
||||
assert stored_a[0].get("content") == "one"
|
||||
|
||||
stored_b = await session_b.get_items()
|
||||
assert len(stored_b) == 1
|
||||
assert stored_b[0].get("content") == "two"
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def test_add_items_concurrent_first_access_across_from_url_sessions(tmp_path):
|
||||
"""Concurrent first writes should not race table creation across from_url sessions."""
|
||||
db_url = f"sqlite+aiosqlite:///{tmp_path / 'concurrent_from_url.db'}"
|
||||
session_a = SQLAlchemySession.from_url("from_url_a", url=db_url, create_tables=True)
|
||||
session_b = SQLAlchemySession.from_url("from_url_b", url=db_url, create_tables=True)
|
||||
try:
|
||||
results = await asyncio.gather(
|
||||
session_a.add_items([{"role": "user", "content": "one"}]),
|
||||
session_b.add_items([{"role": "user", "content": "two"}]),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
assert [result for result in results if isinstance(result, Exception)] == []
|
||||
|
||||
stored_a = await session_a.get_items()
|
||||
assert len(stored_a) == 1
|
||||
assert stored_a[0].get("content") == "one"
|
||||
|
||||
stored_b = await session_b.get_items()
|
||||
assert len(stored_b) == 1
|
||||
assert stored_b[0].get("content") == "two"
|
||||
finally:
|
||||
await session_a.engine.dispose()
|
||||
await session_b.engine.dispose()
|
||||
|
||||
|
||||
async def test_add_items_concurrent_first_access_across_from_url_sessions_cross_loop(tmp_path):
|
||||
"""Concurrent first writes should not race or hang across event loops."""
|
||||
db_url = f"sqlite+aiosqlite:///{tmp_path / 'concurrent_from_url_cross_loop.db'}"
|
||||
barrier = threading.Barrier(2)
|
||||
results: list[tuple[str, str, Any]] = []
|
||||
results_lock = threading.Lock()
|
||||
|
||||
def worker(session_id: str, content: str) -> None:
|
||||
async def run() -> tuple[str, Any]:
|
||||
session = SQLAlchemySession.from_url(session_id, url=db_url, create_tables=True)
|
||||
barrier.wait()
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
session.add_items([{"role": "user", "content": content}]),
|
||||
timeout=5,
|
||||
)
|
||||
stored = await session.get_items()
|
||||
return ("ok", stored)
|
||||
finally:
|
||||
await session.engine.dispose()
|
||||
|
||||
try:
|
||||
status, payload = asyncio.run(run())
|
||||
except Exception as exc:
|
||||
status, payload = type(exc).__name__, str(exc)
|
||||
|
||||
with results_lock:
|
||||
results.append((session_id, status, payload))
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=worker, args=("from_url_cross_loop_a", "one")),
|
||||
threading.Thread(target=worker, args=("from_url_cross_loop_b", "two")),
|
||||
]
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
await asyncio.to_thread(thread.join)
|
||||
|
||||
assert len(results) == 2
|
||||
assert [status for _, status, _ in results] == ["ok", "ok"]
|
||||
|
||||
stored_by_session = {
|
||||
session_id: cast(list[TResponseInputItem], payload) for session_id, _, payload in results
|
||||
}
|
||||
assert stored_by_session["from_url_cross_loop_a"][0].get("content") == "one"
|
||||
assert stored_by_session["from_url_cross_loop_b"][0].get("content") == "two"
|
||||
|
||||
|
||||
async def test_add_items_concurrent_first_access_with_shared_session_cross_loop(tmp_path):
|
||||
"""A shared session instance should not hang when used from two event loops."""
|
||||
db_url = f"sqlite+aiosqlite:///{tmp_path / 'shared_session_cross_loop.db'}"
|
||||
session = SQLAlchemySession.from_url(
|
||||
"shared_session_cross_loop",
|
||||
url=db_url,
|
||||
create_tables=True,
|
||||
)
|
||||
barrier = threading.Barrier(2)
|
||||
results: list[tuple[str, str]] = []
|
||||
results_lock = threading.Lock()
|
||||
|
||||
def worker(content: str) -> None:
|
||||
async def run() -> None:
|
||||
barrier.wait()
|
||||
await asyncio.wait_for(
|
||||
session.add_items([{"role": "user", "content": content}]),
|
||||
timeout=5,
|
||||
)
|
||||
|
||||
try:
|
||||
asyncio.run(run())
|
||||
status = "ok"
|
||||
except Exception as exc:
|
||||
status = type(exc).__name__
|
||||
|
||||
with results_lock:
|
||||
results.append((content, status))
|
||||
|
||||
threads = [
|
||||
threading.Thread(target=worker, args=("one",)),
|
||||
threading.Thread(target=worker, args=("two",)),
|
||||
]
|
||||
try:
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
for thread in threads:
|
||||
await asyncio.to_thread(thread.join)
|
||||
|
||||
assert sorted(results) == [("one", "ok"), ("two", "ok")]
|
||||
|
||||
stored = await session.get_items()
|
||||
stored_contents: list[str] = []
|
||||
for item in stored:
|
||||
content = item.get("content")
|
||||
assert isinstance(content, str)
|
||||
stored_contents.append(content)
|
||||
assert sorted(stored_contents) == ["one", "two"]
|
||||
finally:
|
||||
await session.engine.dispose()
|
||||
|
||||
|
||||
async def test_add_items_cancelled_waiter_does_not_strand_table_init_lock(tmp_path):
|
||||
"""Cancelling a waiting initializer must not leave the shared init lock acquired."""
|
||||
db_url = f"sqlite+aiosqlite:///{tmp_path / 'cancelled_table_init_waiter.db'}"
|
||||
holder = SQLAlchemySession.from_url("holder", url=db_url, create_tables=True)
|
||||
waiter = SQLAlchemySession.from_url("waiter", url=db_url, create_tables=True)
|
||||
follower = SQLAlchemySession.from_url("follower", url=db_url, create_tables=True)
|
||||
|
||||
assert holder._init_lock is waiter._init_lock
|
||||
assert waiter._init_lock is follower._init_lock
|
||||
assert holder._init_lock is not None
|
||||
|
||||
acquired = holder._init_lock.acquire(blocking=False)
|
||||
assert acquired
|
||||
|
||||
try:
|
||||
blocked = asyncio.create_task(waiter.add_items([{"role": "user", "content": "waiter"}]))
|
||||
await asyncio.sleep(0.05)
|
||||
blocked.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await blocked
|
||||
finally:
|
||||
holder._init_lock.release()
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
follower.add_items([{"role": "user", "content": "follower"}]),
|
||||
timeout=2,
|
||||
)
|
||||
stored = await follower.get_items()
|
||||
assert len(stored) == 1
|
||||
assert stored[0].get("content") == "follower"
|
||||
finally:
|
||||
await holder.engine.dispose()
|
||||
await waiter.engine.dispose()
|
||||
await follower.engine.dispose()
|
||||
|
||||
|
||||
async def test_create_tables_false_does_not_allocate_shared_init_lock(tmp_path):
|
||||
"""Sessions that skip auto-create should not populate the shared lock map."""
|
||||
db_url = f"sqlite+aiosqlite:///{tmp_path / 'no_create_tables_lock.db'}"
|
||||
before = len(SQLAlchemySession._table_init_locks)
|
||||
session = SQLAlchemySession.from_url("no_create_tables_lock", url=db_url, create_tables=False)
|
||||
try:
|
||||
assert session._init_lock is None
|
||||
assert len(SQLAlchemySession._table_init_locks) == before
|
||||
finally:
|
||||
await session.engine.dispose()
|
||||
|
||||
|
||||
async def test_get_items_same_timestamp_consistent_order():
|
||||
"""Test that items with identical timestamps keep insertion order."""
|
||||
session_id = "same_timestamp_test"
|
||||
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
|
||||
|
||||
older_item = _make_message_item("older_same_ts", "old")
|
||||
reasoning_item = _make_reasoning_item("rs_same_ts", "...")
|
||||
message_item = _make_message_item("msg_same_ts", "...")
|
||||
await session.add_items([older_item])
|
||||
await session.add_items([reasoning_item, message_item])
|
||||
|
||||
async with session._session_factory() as sess:
|
||||
rows = await sess.execute(
|
||||
select(session._messages.c.id, session._messages.c.message_data).where(
|
||||
session._messages.c.session_id == session.session_id
|
||||
)
|
||||
)
|
||||
id_map = {
|
||||
json.loads(message_json)["id"]: row_id for row_id, message_json in rows.fetchall()
|
||||
}
|
||||
shared = datetime(2025, 10, 15, 17, 26, 39, 132483)
|
||||
older = shared - timedelta(milliseconds=1)
|
||||
await sess.execute(
|
||||
update(session._messages)
|
||||
.where(
|
||||
session._messages.c.id.in_(
|
||||
[
|
||||
id_map["rs_same_ts"],
|
||||
id_map["msg_same_ts"],
|
||||
]
|
||||
)
|
||||
)
|
||||
.values(created_at=shared)
|
||||
)
|
||||
await sess.execute(
|
||||
update(session._messages)
|
||||
.where(session._messages.c.id == id_map["older_same_ts"])
|
||||
.values(created_at=older)
|
||||
)
|
||||
await sess.commit()
|
||||
|
||||
real_factory = session._session_factory
|
||||
|
||||
class FakeResult:
|
||||
def __init__(self, rows: Iterable[Any]):
|
||||
self._rows = list(rows)
|
||||
|
||||
def all(self) -> list[Any]:
|
||||
return list(self._rows)
|
||||
|
||||
def needs_shuffle(statement: Any) -> bool:
|
||||
if not isinstance(statement, Select):
|
||||
return False
|
||||
orderings = list(statement._order_by_clause)
|
||||
if not orderings:
|
||||
return False
|
||||
id_asc = session._messages.c.id.asc()
|
||||
id_desc = session._messages.c.id.desc()
|
||||
|
||||
def references_id(clause) -> bool:
|
||||
try:
|
||||
return bool(clause.compare(id_asc) or clause.compare(id_desc))
|
||||
except AttributeError:
|
||||
return False
|
||||
|
||||
if any(references_id(clause) for clause in orderings):
|
||||
return False
|
||||
# Only shuffle queries that target the messages table.
|
||||
target_tables: set[str] = set()
|
||||
for from_clause in statement.get_final_froms():
|
||||
name_attr = getattr(from_clause, "name", None)
|
||||
if isinstance(name_attr, str):
|
||||
target_tables.add(name_attr)
|
||||
table_name_obj = getattr(session._messages, "name", "")
|
||||
table_name = table_name_obj if isinstance(table_name_obj, str) else ""
|
||||
return bool(table_name in target_tables)
|
||||
|
||||
@asynccontextmanager
|
||||
async def shuffled_session():
|
||||
async with real_factory() as inner:
|
||||
original_execute = inner.execute
|
||||
|
||||
async def execute_with_shuffle(statement: Any, *args: Any, **kwargs: Any) -> Any:
|
||||
result = await original_execute(statement, *args, **kwargs)
|
||||
if needs_shuffle(statement):
|
||||
rows = result.all()
|
||||
shuffled = list(rows)
|
||||
shuffled.reverse()
|
||||
return FakeResult(shuffled)
|
||||
return result
|
||||
|
||||
cast(Any, inner).execute = execute_with_shuffle
|
||||
try:
|
||||
yield inner
|
||||
finally:
|
||||
cast(Any, inner).execute = original_execute
|
||||
|
||||
session._session_factory = cast(Any, shuffled_session)
|
||||
try:
|
||||
retrieved = await session.get_items()
|
||||
assert _item_ids(retrieved) == ["older_same_ts", "rs_same_ts", "msg_same_ts"]
|
||||
|
||||
latest_two = await session.get_items(limit=2)
|
||||
assert _item_ids(latest_two) == ["rs_same_ts", "msg_same_ts"]
|
||||
finally:
|
||||
session._session_factory = real_factory
|
||||
|
||||
|
||||
async def test_pop_item_same_timestamp_returns_latest():
|
||||
"""Test that pop_item returns the newest item when timestamps tie."""
|
||||
session_id = "same_timestamp_pop_test"
|
||||
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
|
||||
|
||||
reasoning_item = _make_reasoning_item("rs_pop_same_ts", "...")
|
||||
message_item = _make_message_item("msg_pop_same_ts", "...")
|
||||
await session.add_items([reasoning_item, message_item])
|
||||
|
||||
async with session._session_factory() as sess:
|
||||
await sess.execute(
|
||||
text(
|
||||
"UPDATE agent_messages SET created_at = :created_at WHERE session_id = :session_id"
|
||||
),
|
||||
{
|
||||
"created_at": "2025-10-15 17:26:39.132483",
|
||||
"session_id": session.session_id,
|
||||
},
|
||||
)
|
||||
await sess.commit()
|
||||
|
||||
popped = await session.pop_item()
|
||||
assert popped is not None
|
||||
assert cast(dict[str, Any], popped)["id"] == "msg_pop_same_ts"
|
||||
|
||||
remaining = await session.get_items()
|
||||
assert _item_ids(remaining) == ["rs_pop_same_ts"]
|
||||
|
||||
|
||||
async def test_get_items_orders_by_id_for_ties():
|
||||
"""Test that get_items adds id ordering to break timestamp ties."""
|
||||
session_id = "order_by_id_test"
|
||||
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
|
||||
|
||||
await session.add_items(
|
||||
[
|
||||
_make_reasoning_item("rs_first", "..."),
|
||||
_make_message_item("msg_second", "..."),
|
||||
]
|
||||
)
|
||||
|
||||
real_factory = session._session_factory
|
||||
recorded: list[Any] = []
|
||||
|
||||
@asynccontextmanager
|
||||
async def wrapped_session():
|
||||
async with real_factory() as inner:
|
||||
original_execute = inner.execute
|
||||
|
||||
async def recording_execute(statement: Any, *args: Any, **kwargs: Any) -> Any:
|
||||
recorded.append(statement)
|
||||
return await original_execute(statement, *args, **kwargs)
|
||||
|
||||
cast(Any, inner).execute = recording_execute
|
||||
try:
|
||||
yield inner
|
||||
finally:
|
||||
cast(Any, inner).execute = original_execute
|
||||
|
||||
session._session_factory = cast(Any, wrapped_session)
|
||||
try:
|
||||
retrieved_full = await session.get_items()
|
||||
retrieved_limited = await session.get_items(limit=2)
|
||||
finally:
|
||||
session._session_factory = real_factory
|
||||
|
||||
assert len(recorded) >= 2
|
||||
orderings_full = [str(clause) for clause in recorded[0]._order_by_clause]
|
||||
assert orderings_full == [
|
||||
"agent_messages.created_at ASC",
|
||||
"agent_messages.id ASC",
|
||||
]
|
||||
|
||||
orderings_limited = [str(clause) for clause in recorded[1]._order_by_clause]
|
||||
assert orderings_limited == [
|
||||
"agent_messages.created_at DESC",
|
||||
"agent_messages.id DESC",
|
||||
]
|
||||
|
||||
assert _item_ids(retrieved_full) == ["rs_first", "msg_second"]
|
||||
assert _item_ids(retrieved_limited) == ["rs_first", "msg_second"]
|
||||
|
||||
|
||||
async def test_engine_property_from_url():
|
||||
"""Test that the engine property returns the AsyncEngine from from_url."""
|
||||
session_id = "engine_property_test"
|
||||
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
|
||||
|
||||
# Verify engine property returns an AsyncEngine instance
|
||||
assert isinstance(session.engine, AsyncEngine)
|
||||
|
||||
# Verify we can use the engine for advanced operations
|
||||
# For example, check pool status
|
||||
assert session.engine.pool is not None
|
||||
|
||||
# Verify we can manually dispose the engine
|
||||
await session.engine.dispose()
|
||||
|
||||
|
||||
async def test_engine_property_from_external_engine():
|
||||
"""Test that the engine property returns the external engine."""
|
||||
session_id = "external_engine_test"
|
||||
|
||||
# Create engine externally
|
||||
external_engine = create_async_engine(DB_URL)
|
||||
|
||||
# Create session with external engine
|
||||
session = SQLAlchemySession(session_id, engine=external_engine, create_tables=True)
|
||||
|
||||
# Verify engine property returns the same engine instance
|
||||
assert session.engine is external_engine
|
||||
|
||||
# Verify we can use the engine
|
||||
assert isinstance(session.engine, AsyncEngine)
|
||||
|
||||
# Clean up - user is responsible for disposing external engine
|
||||
await external_engine.dispose()
|
||||
|
||||
|
||||
async def test_engine_property_is_read_only():
|
||||
"""Test that the engine property cannot be modified."""
|
||||
session_id = "readonly_engine_test"
|
||||
session = SQLAlchemySession.from_url(session_id, url=DB_URL, create_tables=True)
|
||||
|
||||
# Verify engine property exists
|
||||
assert hasattr(session, "engine")
|
||||
|
||||
# Verify it's a property (read-only, cannot be set)
|
||||
# Type ignore needed because mypy correctly detects this is read-only
|
||||
with pytest.raises(AttributeError):
|
||||
session.engine = create_async_engine(DB_URL) # type: ignore[misc]
|
||||
|
||||
# Clean up
|
||||
await session.engine.dispose()
|
||||
|
||||
|
||||
async def test_session_settings_default():
|
||||
"""Test that session_settings defaults to empty SessionSettings."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
session = SQLAlchemySession.from_url("default_settings_test", url=DB_URL, create_tables=True)
|
||||
|
||||
# Should have default SessionSettings
|
||||
assert isinstance(session.session_settings, SessionSettings)
|
||||
assert session.session_settings.limit is None
|
||||
|
||||
|
||||
async def test_session_settings_from_url():
|
||||
"""Test passing session_settings via from_url."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
session = SQLAlchemySession.from_url(
|
||||
"from_url_settings_test",
|
||||
url=DB_URL,
|
||||
create_tables=True,
|
||||
session_settings=SessionSettings(limit=5),
|
||||
)
|
||||
|
||||
assert session.session_settings is not None
|
||||
assert session.session_settings.limit == 5
|
||||
|
||||
|
||||
async def test_get_items_uses_session_settings_limit():
|
||||
"""Test that get_items uses session_settings.limit as default."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
session = SQLAlchemySession.from_url(
|
||||
"uses_settings_limit_test",
|
||||
url=DB_URL,
|
||||
create_tables=True,
|
||||
session_settings=SessionSettings(limit=3),
|
||||
)
|
||||
|
||||
# Add 5 items
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": f"Message {i}"} for i in range(5)
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
# get_items() with no limit should use session_settings.limit=3
|
||||
retrieved = await session.get_items()
|
||||
assert len(retrieved) == 3
|
||||
# Should get the last 3 items
|
||||
assert retrieved[0].get("content") == "Message 2"
|
||||
assert retrieved[1].get("content") == "Message 3"
|
||||
assert retrieved[2].get("content") == "Message 4"
|
||||
|
||||
|
||||
async def test_get_items_explicit_limit_overrides_session_settings():
|
||||
"""Test that explicit limit parameter overrides session_settings."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
session = SQLAlchemySession.from_url(
|
||||
"explicit_override_test",
|
||||
url=DB_URL,
|
||||
create_tables=True,
|
||||
session_settings=SessionSettings(limit=5),
|
||||
)
|
||||
|
||||
# Add 10 items
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": f"Message {i}"} for i in range(10)
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
# Explicit limit=2 should override session_settings.limit=5
|
||||
retrieved = await session.get_items(limit=2)
|
||||
assert len(retrieved) == 2
|
||||
assert retrieved[0].get("content") == "Message 8"
|
||||
assert retrieved[1].get("content") == "Message 9"
|
||||
|
||||
|
||||
async def test_session_settings_resolve():
|
||||
"""Test SessionSettings.resolve() method."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
base = SessionSettings(limit=100)
|
||||
override = SessionSettings(limit=50)
|
||||
|
||||
final = base.resolve(override)
|
||||
|
||||
assert final.limit == 50 # Override wins
|
||||
assert base.limit == 100 # Original unchanged
|
||||
|
||||
# Resolving with None returns self
|
||||
final_none = base.resolve(None)
|
||||
assert final_none.limit == 100
|
||||
|
||||
|
||||
async def test_runner_with_session_settings_override(agent: Agent):
|
||||
"""Test that RunConfig can override session's default settings."""
|
||||
from agents import RunConfig
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
# Session with default limit=100
|
||||
session = SQLAlchemySession.from_url(
|
||||
"runner_override_test",
|
||||
url=DB_URL,
|
||||
create_tables=True,
|
||||
session_settings=SessionSettings(limit=100),
|
||||
)
|
||||
|
||||
# Add some history
|
||||
items: list[TResponseInputItem] = [{"role": "user", "content": f"Turn {i}"} for i in range(10)]
|
||||
await session.add_items(items)
|
||||
|
||||
# Use RunConfig to override limit to 2
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("Got it")])
|
||||
|
||||
await Runner.run(
|
||||
agent,
|
||||
"New question",
|
||||
session=session,
|
||||
run_config=RunConfig(
|
||||
session_settings=SessionSettings(limit=2) # Override to 2
|
||||
),
|
||||
)
|
||||
|
||||
# Verify the agent received only the last 2 history items + new question
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
# Filter out the new "New question" input
|
||||
history_items = [item for item in last_input if item.get("content") != "New question"]
|
||||
# Should have 2 history items (last two from the 10 we added)
|
||||
assert len(history_items) == 2
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,345 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
def _load_example_module() -> Any:
|
||||
path = (
|
||||
Path(__file__).resolve().parents[3]
|
||||
/ "examples"
|
||||
/ "sandbox"
|
||||
/ "extensions"
|
||||
/ "runloop"
|
||||
/ "capabilities.py"
|
||||
)
|
||||
module_name = "tests.extensions.sandbox.runloop_capabilities_example"
|
||||
spec = importlib.util.spec_from_file_location(module_name, path)
|
||||
assert spec is not None
|
||||
assert spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[module_name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class _FakeNotFoundError(Exception):
|
||||
def __init__(self) -> None:
|
||||
self.status_code = 404
|
||||
self.response = types.SimpleNamespace(status_code=404)
|
||||
|
||||
|
||||
class _FakeConflictError(Exception):
|
||||
def __init__(self, message: str) -> None:
|
||||
self.status_code = 400
|
||||
self.response = types.SimpleNamespace(status_code=400)
|
||||
self.body = {"message": message}
|
||||
|
||||
|
||||
class _FakeSecret:
|
||||
def __init__(self, name: str, secret_id: str) -> None:
|
||||
self.id = secret_id
|
||||
self.name = name
|
||||
|
||||
|
||||
class _FakeSecretsClient:
|
||||
def __init__(self) -> None:
|
||||
self.secrets: dict[str, _FakeSecret] = {}
|
||||
self.create_calls: list[tuple[str, str]] = []
|
||||
self.delete_calls: list[str] = []
|
||||
self._counter = 0
|
||||
|
||||
def add(self, name: str) -> _FakeSecret:
|
||||
self._counter += 1
|
||||
secret = _FakeSecret(name=name, secret_id=f"secret-{self._counter}")
|
||||
self.secrets[name] = secret
|
||||
return secret
|
||||
|
||||
async def get(self, name: str) -> _FakeSecret:
|
||||
if name not in self.secrets:
|
||||
raise _FakeNotFoundError()
|
||||
return self.secrets[name]
|
||||
|
||||
async def create(self, *, name: str, value: str) -> _FakeSecret:
|
||||
self.create_calls.append((name, value))
|
||||
return self.add(name)
|
||||
|
||||
|
||||
class _FakePolicy:
|
||||
def __init__(self, policy_id: str, name: str, description: str | None = None) -> None:
|
||||
self.id = policy_id
|
||||
self.name = name
|
||||
self.description = description
|
||||
|
||||
|
||||
class _FakePolicyRef:
|
||||
def __init__(self, policy: _FakePolicy) -> None:
|
||||
self._policy = policy
|
||||
|
||||
async def get_info(self) -> object:
|
||||
return types.SimpleNamespace(
|
||||
id=self._policy.id,
|
||||
name=self._policy.name,
|
||||
description=self._policy.description,
|
||||
)
|
||||
|
||||
|
||||
class _FakeNetworkPoliciesClient:
|
||||
def __init__(self) -> None:
|
||||
self.policies: dict[str, _FakePolicy] = {}
|
||||
self.create_calls: list[dict[str, object]] = []
|
||||
self.delete_calls: list[str] = []
|
||||
self._counter = 0
|
||||
|
||||
def add(self, name: str, description: str | None = None) -> _FakePolicy:
|
||||
self._counter += 1
|
||||
policy = _FakePolicy(
|
||||
policy_id=f"np-{self._counter}",
|
||||
name=name,
|
||||
description=description,
|
||||
)
|
||||
self.policies[policy.id] = policy
|
||||
return policy
|
||||
|
||||
async def list(self, **params: object) -> list[_FakePolicy]:
|
||||
name = params.get("name")
|
||||
policies = list(self.policies.values())
|
||||
if isinstance(name, str):
|
||||
return [policy for policy in policies if policy.name == name]
|
||||
return policies
|
||||
|
||||
async def create(self, **params: object) -> _FakePolicy:
|
||||
self.create_calls.append(dict(params))
|
||||
name = str(params["name"])
|
||||
if any(policy.name == name for policy in self.policies.values()):
|
||||
raise _FakeConflictError(f"NetworkPolicy with name '{name}' already exists")
|
||||
description = cast(
|
||||
str | None,
|
||||
params.get("description") if isinstance(params.get("description"), str) else None,
|
||||
)
|
||||
return self.add(
|
||||
name=name,
|
||||
description=description,
|
||||
)
|
||||
|
||||
def get(self, policy_id: str) -> _FakePolicyRef:
|
||||
return _FakePolicyRef(self.policies[policy_id])
|
||||
|
||||
|
||||
class _FakePlatformClient:
|
||||
def __init__(self) -> None:
|
||||
self.secrets = _FakeSecretsClient()
|
||||
self.network_policies = _FakeNetworkPoliciesClient()
|
||||
|
||||
|
||||
class _FakeRunloopClient:
|
||||
def __init__(self) -> None:
|
||||
self.platform = _FakePlatformClient()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_runloop_secret_returns_non_sensitive_metadata() -> None:
|
||||
module = _load_example_module()
|
||||
client = _FakeRunloopClient()
|
||||
secret = client.platform.secrets.add("RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN")
|
||||
|
||||
result = await module._query_runloop_secret( # noqa: SLF001
|
||||
client,
|
||||
name=secret.name,
|
||||
)
|
||||
|
||||
assert result.found is True
|
||||
assert result.id == secret.id
|
||||
assert "value" not in result.model_dump(mode="json")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_runloop_secret_reports_missing_before_create() -> None:
|
||||
module = _load_example_module()
|
||||
client = _FakeRunloopClient()
|
||||
|
||||
result = await module._query_runloop_secret( # noqa: SLF001
|
||||
client,
|
||||
name="RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN",
|
||||
)
|
||||
|
||||
assert result.found is False
|
||||
assert result.id is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_runloop_network_policy_reports_existing_resource() -> None:
|
||||
module = _load_example_module()
|
||||
client = _FakeRunloopClient()
|
||||
policy = client.platform.network_policies.add(
|
||||
"runloop-capabilities-example-policy",
|
||||
description="Persistent example policy.",
|
||||
)
|
||||
|
||||
result = await module._query_runloop_network_policy( # noqa: SLF001
|
||||
client,
|
||||
name=policy.name,
|
||||
)
|
||||
|
||||
assert result.found is True
|
||||
assert result.id == policy.id
|
||||
assert result.description == "Persistent example policy."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bootstrap_persistent_resources_reuses_existing_resources_without_cleanup() -> None:
|
||||
module = _load_example_module()
|
||||
client = _FakeRunloopClient()
|
||||
secret = client.platform.secrets.add("RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN")
|
||||
policy = client.platform.network_policies.add("runloop-capabilities-example-policy")
|
||||
query_results = {
|
||||
"secret": module.RunloopResourceQueryResult(
|
||||
resource_type="secret",
|
||||
name=secret.name,
|
||||
found=True,
|
||||
id=secret.id,
|
||||
),
|
||||
"network_policy": module.RunloopResourceQueryResult(
|
||||
resource_type="network_policy",
|
||||
name=policy.name,
|
||||
found=True,
|
||||
id=policy.id,
|
||||
),
|
||||
}
|
||||
|
||||
bootstrap = await module._bootstrap_persistent_resources( # noqa: SLF001
|
||||
client,
|
||||
managed_secret_name=secret.name,
|
||||
managed_secret_value="runloop-capabilities-example-token",
|
||||
network_policy_name=policy.name,
|
||||
network_policy_id_override=None,
|
||||
query_results=query_results,
|
||||
axon_name=None,
|
||||
)
|
||||
|
||||
secret_bootstrap = bootstrap["secret"]
|
||||
network_policy_bootstrap = bootstrap["network_policy"]
|
||||
assert secret_bootstrap.action == "reused"
|
||||
assert network_policy_bootstrap.action == "reused"
|
||||
assert client.platform.secrets.create_calls == []
|
||||
assert client.platform.network_policies.create_calls == []
|
||||
assert client.platform.secrets.delete_calls == []
|
||||
assert client.platform.network_policies.delete_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bootstrap_persistent_resources_creates_missing_resources() -> None:
|
||||
module = _load_example_module()
|
||||
client = _FakeRunloopClient()
|
||||
query_results = {
|
||||
"secret": module.RunloopResourceQueryResult(
|
||||
resource_type="secret",
|
||||
name="RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN",
|
||||
found=False,
|
||||
),
|
||||
"network_policy": module.RunloopResourceQueryResult(
|
||||
resource_type="network_policy",
|
||||
name="runloop-capabilities-example-policy",
|
||||
found=False,
|
||||
),
|
||||
}
|
||||
|
||||
bootstrap = await module._bootstrap_persistent_resources( # noqa: SLF001
|
||||
client,
|
||||
managed_secret_name="RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN",
|
||||
managed_secret_value="runloop-capabilities-example-token",
|
||||
network_policy_name="runloop-capabilities-example-policy",
|
||||
network_policy_id_override=None,
|
||||
query_results=query_results,
|
||||
axon_name=None,
|
||||
)
|
||||
|
||||
secret_bootstrap = bootstrap["secret"]
|
||||
network_policy_bootstrap = bootstrap["network_policy"]
|
||||
assert secret_bootstrap.action == "created"
|
||||
assert network_policy_bootstrap.action == "created"
|
||||
assert client.platform.secrets.create_calls == [
|
||||
("RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN", "runloop-capabilities-example-token")
|
||||
]
|
||||
assert client.platform.network_policies.create_calls == [
|
||||
{
|
||||
"name": "runloop-capabilities-example-policy",
|
||||
"allow_all": True,
|
||||
"description": "Persistent network policy for the Runloop capabilities example.",
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bootstrap_persistent_resources_respects_policy_override() -> None:
|
||||
module = _load_example_module()
|
||||
client = _FakeRunloopClient()
|
||||
query_results = {
|
||||
"secret": module.RunloopResourceQueryResult(
|
||||
resource_type="secret",
|
||||
name="RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN",
|
||||
found=False,
|
||||
),
|
||||
"network_policy": module.RunloopResourceQueryResult(
|
||||
resource_type="network_policy",
|
||||
name="runloop-capabilities-example-policy",
|
||||
found=False,
|
||||
),
|
||||
}
|
||||
|
||||
bootstrap = await module._bootstrap_persistent_resources( # noqa: SLF001
|
||||
client,
|
||||
managed_secret_name="RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN",
|
||||
managed_secret_value="runloop-capabilities-example-token",
|
||||
network_policy_name="runloop-capabilities-example-policy",
|
||||
network_policy_id_override="np-override",
|
||||
query_results=query_results,
|
||||
axon_name=None,
|
||||
)
|
||||
|
||||
network_policy_bootstrap = bootstrap["network_policy"]
|
||||
assert network_policy_bootstrap.action == "override"
|
||||
assert network_policy_bootstrap.id == "np-override"
|
||||
assert client.platform.network_policies.create_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bootstrap_persistent_resources_recovers_from_existing_policy_conflict() -> None:
|
||||
module = _load_example_module()
|
||||
client = _FakeRunloopClient()
|
||||
policy = client.platform.network_policies.add(
|
||||
"runloop-capabilities-example-policy",
|
||||
description="Persistent example policy.",
|
||||
)
|
||||
query_results = {
|
||||
"secret": module.RunloopResourceQueryResult(
|
||||
resource_type="secret",
|
||||
name="RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN",
|
||||
found=False,
|
||||
),
|
||||
"network_policy": module.RunloopResourceQueryResult(
|
||||
resource_type="network_policy",
|
||||
name=policy.name,
|
||||
found=False,
|
||||
),
|
||||
}
|
||||
|
||||
bootstrap = await module._bootstrap_persistent_resources( # noqa: SLF001
|
||||
client,
|
||||
managed_secret_name="RUNLOOP_CAPABILITIES_EXAMPLE_TOKEN",
|
||||
managed_secret_value="runloop-capabilities-example-token",
|
||||
network_policy_name=policy.name,
|
||||
network_policy_id_override=None,
|
||||
query_results=query_results,
|
||||
axon_name=None,
|
||||
)
|
||||
|
||||
network_policy_bootstrap = bootstrap["network_policy"]
|
||||
assert network_policy_bootstrap.action == "reused"
|
||||
assert network_policy_bootstrap.found_before_bootstrap is True
|
||||
assert network_policy_bootstrap.id == policy.id
|
||||
@@ -0,0 +1,224 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import types
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from agents.sandbox import Manifest
|
||||
from agents.sandbox.entries import RcloneMountPattern, S3Mount
|
||||
from agents.sandbox.errors import MountConfigError
|
||||
from agents.sandbox.session.base_sandbox_session import BaseSandboxSession
|
||||
from agents.sandbox.types import ExecResult
|
||||
|
||||
|
||||
class _FakeRunloopMountSession(BaseSandboxSession):
|
||||
def __init__(self, results: list[ExecResult] | None = None) -> None:
|
||||
self.state = cast(
|
||||
Any,
|
||||
types.SimpleNamespace(
|
||||
session_id=uuid.uuid4(),
|
||||
manifest=Manifest(root="/workspace"),
|
||||
),
|
||||
)
|
||||
self._results = list(results or [])
|
||||
self.exec_calls: list[str] = []
|
||||
|
||||
async def _exec_internal(
|
||||
self,
|
||||
*command: str | Path,
|
||||
timeout: float | None = None,
|
||||
) -> ExecResult:
|
||||
_ = timeout
|
||||
cmd_str = " ".join(str(c) for c in command)
|
||||
self.exec_calls.append(cmd_str)
|
||||
if self._results:
|
||||
return self._results.pop(0)
|
||||
return ExecResult(stdout=b"", stderr=b"", exit_code=0)
|
||||
|
||||
async def read(self, path: Path, *, user: object = None) -> io.IOBase:
|
||||
_ = (path, user)
|
||||
return io.BytesIO(b"")
|
||||
|
||||
async def write(self, path: Path, data: io.IOBase, *, user: object = None) -> None:
|
||||
_ = (path, data, user)
|
||||
|
||||
async def persist_workspace(self) -> io.IOBase:
|
||||
raise AssertionError("not expected")
|
||||
|
||||
async def hydrate_workspace(self, data: io.IOBase) -> None:
|
||||
_ = data
|
||||
raise AssertionError("not expected")
|
||||
|
||||
async def running(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
_FakeRunloopMountSession.__name__ = "RunloopSandboxSession"
|
||||
|
||||
|
||||
def _exec_ok(stdout: bytes = b"") -> ExecResult:
|
||||
return ExecResult(stdout=stdout, stderr=b"", exit_code=0)
|
||||
|
||||
|
||||
def _exec_fail() -> ExecResult:
|
||||
return ExecResult(stdout=b"", stderr=b"", exit_code=1)
|
||||
|
||||
|
||||
def test_runloop_package_re_exports_cloud_bucket_strategy() -> None:
|
||||
package_module = __import__(
|
||||
"agents.extensions.sandbox.runloop",
|
||||
fromlist=["RunloopCloudBucketMountStrategy"],
|
||||
)
|
||||
|
||||
assert hasattr(package_module, "RunloopCloudBucketMountStrategy")
|
||||
|
||||
|
||||
def test_runloop_extension_re_exports_cloud_bucket_strategy() -> None:
|
||||
package_module = __import__(
|
||||
"agents.extensions.sandbox",
|
||||
fromlist=["RunloopCloudBucketMountStrategy"],
|
||||
)
|
||||
|
||||
assert hasattr(package_module, "RunloopCloudBucketMountStrategy")
|
||||
|
||||
|
||||
def test_runloop_mount_strategy_type_and_default_pattern() -> None:
|
||||
from agents.extensions.sandbox.runloop.mounts import RunloopCloudBucketMountStrategy
|
||||
|
||||
strategy = RunloopCloudBucketMountStrategy()
|
||||
|
||||
assert strategy.type == "runloop_cloud_bucket"
|
||||
assert isinstance(strategy.pattern, RcloneMountPattern)
|
||||
assert strategy.pattern.mode == "fuse"
|
||||
|
||||
|
||||
def test_runloop_mount_strategy_round_trips_through_manifest() -> None:
|
||||
from agents.extensions.sandbox.runloop.mounts import RunloopCloudBucketMountStrategy
|
||||
|
||||
manifest = Manifest.model_validate(
|
||||
{
|
||||
"root": "/workspace",
|
||||
"entries": {
|
||||
"bucket": {
|
||||
"type": "s3_mount",
|
||||
"bucket": "my-bucket",
|
||||
"mount_strategy": {"type": "runloop_cloud_bucket"},
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
mount = manifest.entries["bucket"]
|
||||
assert isinstance(mount, S3Mount)
|
||||
assert isinstance(mount.mount_strategy, RunloopCloudBucketMountStrategy)
|
||||
|
||||
|
||||
def test_runloop_session_guard_rejects_wrong_type() -> None:
|
||||
from agents.extensions.sandbox.runloop.mounts import _assert_runloop_session
|
||||
|
||||
class _WrongSession:
|
||||
pass
|
||||
|
||||
with pytest.raises(MountConfigError, match="RunloopSandboxSession"):
|
||||
_assert_runloop_session(_WrongSession()) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_runloop_session_guard_accepts_correct_type() -> None:
|
||||
from agents.extensions.sandbox.runloop.mounts import _assert_runloop_session
|
||||
|
||||
_assert_runloop_session(_FakeRunloopMountSession())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runloop_ensure_rclone_installs_with_root_apt() -> None:
|
||||
from agents.extensions.sandbox._rclone import ensure_rclone
|
||||
|
||||
session = _FakeRunloopMountSession(
|
||||
[
|
||||
_exec_fail(),
|
||||
_exec_ok(),
|
||||
_exec_ok(),
|
||||
_exec_ok(),
|
||||
_exec_ok(),
|
||||
]
|
||||
)
|
||||
|
||||
await ensure_rclone(session)
|
||||
|
||||
assert session.exec_calls[:2] == [
|
||||
"sh -lc command -v rclone >/dev/null 2>&1 || test -x /usr/local/bin/rclone",
|
||||
"sh -lc command -v apt-get >/dev/null 2>&1",
|
||||
]
|
||||
assert session.exec_calls[2] == (
|
||||
"sudo -u root -- sh -lc DEBIAN_FRONTEND=noninteractive "
|
||||
"DEBCONF_NOWARNINGS=yes apt-get -o Dpkg::Use-Pty=0 update -qq"
|
||||
)
|
||||
assert session.exec_calls[3] == (
|
||||
"sudo -u root -- sh -lc DEBIAN_FRONTEND=noninteractive "
|
||||
"DEBCONF_NOWARNINGS=yes apt-get -o Dpkg::Use-Pty=0 install -y -qq "
|
||||
"curl unzip ca-certificates"
|
||||
)
|
||||
assert (
|
||||
session.exec_calls[4]
|
||||
== "sudo -u root -- sh -lc curl -fsSL https://rclone.org/install.sh | bash"
|
||||
)
|
||||
assert session.exec_calls[5] == (
|
||||
"sh -lc command -v rclone >/dev/null 2>&1 || test -x /usr/local/bin/rclone"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runloop_ensure_fuse_installs_missing_fusermount() -> None:
|
||||
from agents.extensions.sandbox.runloop.mounts import _ensure_fuse_support
|
||||
|
||||
session = _FakeRunloopMountSession(
|
||||
[
|
||||
_exec_ok(),
|
||||
_exec_ok(),
|
||||
_exec_fail(),
|
||||
_exec_ok(),
|
||||
_exec_ok(),
|
||||
_exec_ok(),
|
||||
_exec_ok(),
|
||||
_exec_ok(),
|
||||
]
|
||||
)
|
||||
|
||||
await _ensure_fuse_support(session)
|
||||
|
||||
assert session.exec_calls == [
|
||||
"sh -lc test -c /dev/fuse",
|
||||
"sh -lc grep -qw fuse /proc/filesystems",
|
||||
"sh -lc command -v fusermount3 >/dev/null 2>&1 || command -v fusermount >/dev/null 2>&1",
|
||||
"sh -lc command -v apt-get >/dev/null 2>&1",
|
||||
(
|
||||
"sudo -u root -- sh -lc DEBIAN_FRONTEND=noninteractive "
|
||||
"DEBCONF_NOWARNINGS=yes apt-get -o Dpkg::Use-Pty=0 update -qq"
|
||||
),
|
||||
(
|
||||
"sudo -u root -- sh -lc DEBIAN_FRONTEND=noninteractive "
|
||||
"DEBCONF_NOWARNINGS=yes apt-get -o Dpkg::Use-Pty=0 install -y -qq fuse3"
|
||||
),
|
||||
"sh -lc command -v fusermount3 >/dev/null 2>&1 || command -v fusermount >/dev/null 2>&1",
|
||||
(
|
||||
"sudo -u root -- sh -lc chmod a+rw /dev/fuse && "
|
||||
"touch /etc/fuse.conf && "
|
||||
"(grep -qxF user_allow_other /etc/fuse.conf || "
|
||||
"printf '\\nuser_allow_other\\n' >> /etc/fuse.conf)"
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runloop_rclone_pattern_adds_fuse_access_args() -> None:
|
||||
from agents.extensions.sandbox._rclone import rclone_pattern_for_session
|
||||
|
||||
session = _FakeRunloopMountSession([_exec_ok(stdout=b"1000\n1000\n")])
|
||||
|
||||
pattern = await rclone_pattern_for_session(session, RcloneMountPattern(mode="fuse"))
|
||||
|
||||
assert pattern.extra_args == ["--allow-other", "--uid", "1000", "--gid", "1000"]
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,635 @@
|
||||
"""Tests for ToolOutputTrimmer — the built-in call_model_input_filter for trimming
|
||||
large tool outputs from older conversation turns.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
from typing import Any, cast
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from agents.extensions.tool_output_trimmer import ToolOutputTrimmer
|
||||
from agents.run_config import CallModelData, ModelInputData
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _user(text: str = "hello") -> dict[str, Any]:
|
||||
return {"role": "user", "content": text}
|
||||
|
||||
|
||||
def _assistant(text: str = "response") -> dict[str, Any]:
|
||||
return {"role": "assistant", "content": text}
|
||||
|
||||
|
||||
def _func_call(call_id: str, name: str, *, namespace: str | None = None) -> dict[str, Any]:
|
||||
item = {"type": "function_call", "call_id": call_id, "name": name, "arguments": "{}"}
|
||||
if namespace is not None:
|
||||
item["namespace"] = namespace
|
||||
return item
|
||||
|
||||
|
||||
def _func_output(call_id: str, output: str) -> dict[str, Any]:
|
||||
return {"type": "function_call_output", "call_id": call_id, "output": output}
|
||||
|
||||
|
||||
def _make_data(items: list[Any]) -> CallModelData[Any]:
|
||||
model_data = ModelInputData(input=items, instructions="You are helpful.")
|
||||
return CallModelData(model_data=model_data, agent=MagicMock(), context=None)
|
||||
|
||||
|
||||
def _output(result: ModelInputData, idx: int) -> Any:
|
||||
"""Extract the ``output`` field from a result item (untyped for test convenience)."""
|
||||
item: Any = result.input[idx]
|
||||
return item["output"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Defaults
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestDefaults:
|
||||
def test_default_values(self) -> None:
|
||||
trimmer = ToolOutputTrimmer()
|
||||
assert trimmer.recent_turns == 2
|
||||
assert trimmer.max_output_chars == 500
|
||||
assert trimmer.preview_chars == 200
|
||||
assert trimmer.trimmable_tools is None
|
||||
|
||||
def test_trimmable_tools_coerced_to_frozenset(self) -> None:
|
||||
trimmer = ToolOutputTrimmer(trimmable_tools=frozenset({"a", "b"}))
|
||||
assert isinstance(trimmer.trimmable_tools, frozenset)
|
||||
assert trimmer.trimmable_tools == frozenset({"a", "b"})
|
||||
|
||||
def test_trimmable_tools_from_list(self) -> None:
|
||||
trimmer = ToolOutputTrimmer(trimmable_tools=["search", "run_code"])
|
||||
assert isinstance(trimmer.trimmable_tools, frozenset)
|
||||
assert "search" in trimmer.trimmable_tools
|
||||
assert "run_code" in trimmer.trimmable_tools
|
||||
|
||||
def test_trimmable_tools_from_string(self) -> None:
|
||||
trimmer = ToolOutputTrimmer(trimmable_tools="search")
|
||||
assert isinstance(trimmer.trimmable_tools, frozenset)
|
||||
assert trimmer.trimmable_tools == frozenset({"search"})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Input validation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestValidation:
|
||||
def test_recent_turns_zero_raises(self) -> None:
|
||||
with pytest.raises(ValueError, match="recent_turns must be >= 1"):
|
||||
ToolOutputTrimmer(recent_turns=0)
|
||||
|
||||
def test_recent_turns_negative_raises(self) -> None:
|
||||
with pytest.raises(ValueError, match="recent_turns must be >= 1"):
|
||||
ToolOutputTrimmer(recent_turns=-1)
|
||||
|
||||
def test_max_output_chars_zero_raises(self) -> None:
|
||||
with pytest.raises(ValueError, match="max_output_chars must be >= 1"):
|
||||
ToolOutputTrimmer(max_output_chars=0)
|
||||
|
||||
def test_preview_chars_negative_raises(self) -> None:
|
||||
with pytest.raises(ValueError, match="preview_chars must be >= 0"):
|
||||
ToolOutputTrimmer(preview_chars=-1)
|
||||
|
||||
def test_preview_chars_zero_allowed(self) -> None:
|
||||
trimmer = ToolOutputTrimmer(preview_chars=0)
|
||||
assert trimmer.preview_chars == 0
|
||||
|
||||
def test_trimmable_tools_bytes_raises(self) -> None:
|
||||
with pytest.raises(ValueError, match="trimmable_tools must be a string or iterable"):
|
||||
ToolOutputTrimmer(trimmable_tools=b"search") # type: ignore[arg-type]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Boundary detection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRecentBoundary:
|
||||
def test_empty_items(self) -> None:
|
||||
trimmer = ToolOutputTrimmer()
|
||||
assert trimmer._find_recent_boundary([]) == 0
|
||||
|
||||
def test_single_user_message(self) -> None:
|
||||
trimmer = ToolOutputTrimmer()
|
||||
assert trimmer._find_recent_boundary([_user()]) == 0
|
||||
|
||||
def test_two_user_messages_boundary_at_first(self) -> None:
|
||||
items = [_user("q1"), _assistant("a1"), _user("q2"), _assistant("a2")]
|
||||
trimmer = ToolOutputTrimmer(recent_turns=2)
|
||||
assert trimmer._find_recent_boundary(items) == 0
|
||||
|
||||
def test_three_user_messages(self) -> None:
|
||||
items = [
|
||||
_user("q1"),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer(recent_turns=2)
|
||||
assert trimmer._find_recent_boundary(items) == 2
|
||||
|
||||
def test_custom_recent_turns(self) -> None:
|
||||
items = [
|
||||
_user("q1"),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
_user("q4"),
|
||||
_assistant("a4"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer(recent_turns=3)
|
||||
# q4 at 6 (count=1), q3 at 4 (count=2), q2 at 2 (count=3) -> boundary=2
|
||||
assert trimmer._find_recent_boundary(items) == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Trimming behavior
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestTrimming:
|
||||
def test_empty_input(self) -> None:
|
||||
trimmer = ToolOutputTrimmer()
|
||||
data = _make_data([])
|
||||
result = trimmer(data)
|
||||
assert result.input == []
|
||||
|
||||
def test_no_trimming_when_all_recent(self) -> None:
|
||||
"""With only 1 user message, everything is recent."""
|
||||
large = "x" * 1000
|
||||
items = [
|
||||
_user("q"),
|
||||
_func_call("c1", "search"),
|
||||
_func_output("c1", large),
|
||||
_assistant("a"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer()
|
||||
result = trimmer(_make_data(items))
|
||||
assert _output(result, 2) == large
|
||||
|
||||
def test_trims_large_old_output(self) -> None:
|
||||
"""Large output in an old turn should be trimmed."""
|
||||
large = "x" * 1000
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_call("c1", "search"),
|
||||
_func_output("c1", large),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer()
|
||||
result = trimmer(_make_data(items))
|
||||
trimmed = _output(result, 2)
|
||||
assert "[Trimmed:" in trimmed
|
||||
assert "search" in trimmed
|
||||
assert "1000 chars" in trimmed
|
||||
assert len(trimmed) < len(large)
|
||||
|
||||
def test_preserves_small_old_output(self) -> None:
|
||||
"""Small outputs should never be trimmed."""
|
||||
small = "x" * 100
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_call("c1", "search"),
|
||||
_func_output("c1", small),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer(max_output_chars=500)
|
||||
result = trimmer(_make_data(items))
|
||||
assert _output(result, 2) == small
|
||||
|
||||
def test_respects_trimmable_tools_allowlist(self) -> None:
|
||||
"""Only outputs from tools in trimmable_tools should be trimmed."""
|
||||
large = "x" * 1000
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_call("c1", "search"),
|
||||
_func_output("c1", large),
|
||||
_func_call("c2", "resolve_entity"),
|
||||
_func_output("c2", large),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer(trimmable_tools=frozenset({"search"}))
|
||||
result = trimmer(_make_data(items))
|
||||
# search output trimmed
|
||||
assert "[Trimmed:" in _output(result, 2)
|
||||
# resolve_entity output preserved
|
||||
assert _output(result, 4) == large
|
||||
|
||||
def test_string_trimmable_tools_allowlist_matches_single_tool_name(self) -> None:
|
||||
"""A string trimmable_tools value should match one tool name, not characters."""
|
||||
large = "x" * 1000
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_call("c1", "search"),
|
||||
_func_output("c1", large),
|
||||
_func_call("c2", "s"),
|
||||
_func_output("c2", large),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer(trimmable_tools="search")
|
||||
result = trimmer(_make_data(items))
|
||||
assert "[Trimmed:" in _output(result, 2)
|
||||
assert _output(result, 4) == large
|
||||
|
||||
def test_respects_qualified_tool_names_allowlist(self) -> None:
|
||||
"""Qualified allowlist entries should match namespaced function tools."""
|
||||
large = "x" * 1000
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_call("c1", "lookup_account", namespace="billing"),
|
||||
_func_output("c1", large),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer(trimmable_tools=frozenset({"billing.lookup_account"}))
|
||||
result = trimmer(_make_data(items))
|
||||
assert "[Trimmed:" in _output(result, 2)
|
||||
assert "billing.lookup_account" in _output(result, 2)
|
||||
|
||||
def test_namespaced_tools_still_match_bare_allowlist_entries(self) -> None:
|
||||
"""Bare allowlist entries remain valid for namespaced tools."""
|
||||
large = "x" * 1000
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_call("c1", "lookup_account", namespace="billing"),
|
||||
_func_output("c1", large),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer(trimmable_tools=frozenset({"lookup_account"}))
|
||||
result = trimmer(_make_data(items))
|
||||
assert "[Trimmed:" in _output(result, 2)
|
||||
assert "billing.lookup_account" in _output(result, 2)
|
||||
|
||||
def test_synthetic_same_name_namespace_uses_bare_display_name(self) -> None:
|
||||
"""Deferred synthetic namespaces should not display as `name.name`."""
|
||||
large = "x" * 1000
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_call("c1", "get_weather", namespace="get_weather"),
|
||||
_func_output("c1", large),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer(trimmable_tools=frozenset({"get_weather"}))
|
||||
result = trimmer(_make_data(items))
|
||||
assert "[Trimmed:" in _output(result, 2)
|
||||
assert "get_weather.get_weather" not in _output(result, 2)
|
||||
assert "get_weather" in _output(result, 2)
|
||||
|
||||
def test_trims_tool_search_output_tool_definitions(self) -> None:
|
||||
"""Large tool_search_output tool definitions should be structurally trimmed."""
|
||||
verbose_schema = {
|
||||
"type": "object",
|
||||
"description": "schema " * 200,
|
||||
"properties": {
|
||||
"customer_id": {
|
||||
"type": "string",
|
||||
"description": "customer id " * 200,
|
||||
"default": "cust_123",
|
||||
}
|
||||
},
|
||||
"required": ["customer_id"],
|
||||
}
|
||||
items = [
|
||||
_user("q1"),
|
||||
{"type": "tool_search_call", "call_id": "ts1", "arguments": {"query": "profile"}},
|
||||
{
|
||||
"type": "tool_search_output",
|
||||
"call_id": "ts1",
|
||||
"tools": [
|
||||
{
|
||||
"type": "function",
|
||||
"name": "lookup_account",
|
||||
"description": "tool description " * 200,
|
||||
"parameters": verbose_schema,
|
||||
}
|
||||
],
|
||||
},
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
|
||||
original_len = len(json.dumps(items[2]["tools"], sort_keys=True))
|
||||
trimmer = ToolOutputTrimmer(max_output_chars=400, preview_chars=60)
|
||||
result = trimmer(_make_data(items))
|
||||
trimmed_item_dict = cast(dict[str, Any], result.input[2])
|
||||
|
||||
assert trimmed_item_dict["type"] == "tool_search_output"
|
||||
trimmed_tools = list(trimmed_item_dict["tools"])
|
||||
assert trimmed_tools[0]["name"] == "lookup_account"
|
||||
assert "description" not in trimmed_tools[0]["parameters"]
|
||||
assert trimmed_tools[0]["parameters"]["properties"]["customer_id"]["default"] == "cust_123"
|
||||
assert len(json.dumps(trimmed_tools, sort_keys=True)) < original_len
|
||||
|
||||
def test_trims_legacy_tool_search_output_results(self) -> None:
|
||||
"""Legacy tool_search_output snapshots with free-text results should still trim."""
|
||||
large = "x" * 2000
|
||||
items = [
|
||||
_user("q1"),
|
||||
{"type": "tool_search_call", "call_id": "ts1", "arguments": {"query": "profile"}},
|
||||
{
|
||||
"type": "tool_search_output",
|
||||
"call_id": "ts1",
|
||||
"results": [{"text": large}],
|
||||
},
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
|
||||
trimmer = ToolOutputTrimmer(max_output_chars=400, preview_chars=80)
|
||||
result = trimmer(_make_data(items))
|
||||
trimmed_item = cast(dict[str, Any], result.input[2])
|
||||
|
||||
assert trimmed_item["type"] == "tool_search_output"
|
||||
assert "[Trimmed: tool_search output" in trimmed_item["results"][0]["text"]
|
||||
|
||||
def test_trims_all_tools_when_allowlist_is_none(self) -> None:
|
||||
"""When trimmable_tools is None, all tools are eligible."""
|
||||
large = "x" * 1000
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_call("c1", "any_tool"),
|
||||
_func_output("c1", large),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer(trimmable_tools=None)
|
||||
result = trimmer(_make_data(items))
|
||||
assert "[Trimmed:" in _output(result, 2)
|
||||
|
||||
def test_preserves_recent_large_output(self) -> None:
|
||||
"""Large outputs in recent turns should never be trimmed."""
|
||||
large = "x" * 1000
|
||||
items = [
|
||||
_user("q1"),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_func_call("c1", "search"),
|
||||
_func_output("c1", large),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer()
|
||||
result = trimmer(_make_data(items))
|
||||
assert _output(result, 4) == large
|
||||
|
||||
def test_does_not_mutate_original_items(self) -> None:
|
||||
"""The filter must not mutate the original input items."""
|
||||
large = "x" * 1000
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_call("c1", "search"),
|
||||
_func_output("c1", large),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
original = copy.deepcopy(items)
|
||||
trimmer = ToolOutputTrimmer()
|
||||
trimmer(_make_data(items))
|
||||
assert items == original
|
||||
|
||||
def test_preserves_instructions(self) -> None:
|
||||
"""The instructions field should pass through unchanged."""
|
||||
items: list[Any] = [_user("hi")]
|
||||
model_data = ModelInputData(input=items, instructions="Custom prompt")
|
||||
data: CallModelData[Any] = CallModelData(
|
||||
model_data=model_data, agent=MagicMock(), context=None
|
||||
)
|
||||
trimmer = ToolOutputTrimmer()
|
||||
result = trimmer(data)
|
||||
assert result.instructions == "Custom prompt"
|
||||
|
||||
def test_multiple_old_outputs_trimmed(self) -> None:
|
||||
"""Multiple large outputs in old turns should all be trimmed."""
|
||||
large1 = "a" * 1000
|
||||
large2 = "b" * 2000
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_call("c1", "search"),
|
||||
_func_output("c1", large1),
|
||||
_func_call("c2", "execute"),
|
||||
_func_output("c2", large2),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer()
|
||||
result = trimmer(_make_data(items))
|
||||
assert "[Trimmed:" in _output(result, 2)
|
||||
assert "[Trimmed:" in _output(result, 4)
|
||||
assert "search" in _output(result, 2)
|
||||
assert "execute" in _output(result, 4)
|
||||
|
||||
def test_custom_preview_chars(self) -> None:
|
||||
"""Preview length should respect the preview_chars setting."""
|
||||
large = "abcdefghij" * 100 # 1000 chars
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_call("c1", "search"),
|
||||
_func_output("c1", large),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer(preview_chars=50)
|
||||
result = trimmer(_make_data(items))
|
||||
trimmed = _output(result, 2)
|
||||
# The preview portion should be exactly 50 chars of the original
|
||||
assert "abcdefghij" * 5 in trimmed
|
||||
|
||||
def test_preserves_user_and_assistant_messages(self) -> None:
|
||||
"""User and assistant messages are never modified."""
|
||||
items = [
|
||||
_user("important"),
|
||||
_assistant("detailed " * 100),
|
||||
_user("follow up"),
|
||||
_assistant("another"),
|
||||
_user("final"),
|
||||
_assistant("done"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer()
|
||||
result = trimmer(_make_data(items))
|
||||
assert result.input == items
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sliding window behavior
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSlidingWindow:
|
||||
"""Verify the trimmer acts as a sliding window across turns."""
|
||||
|
||||
def test_turn3_trims_turn1(self) -> None:
|
||||
"""On turn 3, turn 1 outputs should be trimmed."""
|
||||
large = "x" * 1000
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_call("c1", "search"),
|
||||
_func_output("c1", large),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_func_call("c2", "search"),
|
||||
_func_output("c2", large),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer()
|
||||
result = trimmer(_make_data(items))
|
||||
# Turn 1 (old) trimmed
|
||||
assert "[Trimmed:" in _output(result, 2)
|
||||
# Turn 2 (recent) preserved
|
||||
assert _output(result, 6) == large
|
||||
|
||||
def test_turn4_trims_turns_1_and_2(self) -> None:
|
||||
"""On turn 4, turns 1 and 2 outputs should both be trimmed."""
|
||||
large = "x" * 1000
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_call("c1", "s"),
|
||||
_func_output("c1", large),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_func_call("c2", "s"),
|
||||
_func_output("c2", large),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_func_call("c3", "s"),
|
||||
_func_output("c3", large),
|
||||
_assistant("a3"),
|
||||
_user("q4"),
|
||||
_assistant("a4"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer()
|
||||
result = trimmer(_make_data(items))
|
||||
# Turns 1 and 2 trimmed
|
||||
assert "[Trimmed:" in _output(result, 2)
|
||||
assert "[Trimmed:" in _output(result, 6)
|
||||
# Turn 3 (recent) preserved
|
||||
assert _output(result, 10) == large
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Edge cases
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEdgeCases:
|
||||
def test_skips_trim_when_summary_would_exceed_original(self) -> None:
|
||||
"""When preview_chars is large relative to the output, the summary can be
|
||||
longer than the original. In that case the output should be left untouched."""
|
||||
# Output is 501 chars (just above default max_output_chars=500).
|
||||
# With preview_chars=490, the summary header + 490-char preview + "..." will
|
||||
# easily exceed 501 chars, so trimming should be skipped.
|
||||
borderline = "x" * 501
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_call("c1", "search"),
|
||||
_func_output("c1", borderline),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer(max_output_chars=500, preview_chars=490)
|
||||
result = trimmer(_make_data(items))
|
||||
# Output left untouched because summary would be longer
|
||||
assert _output(result, 2) == borderline
|
||||
|
||||
def test_unknown_tool_name_fallback(self) -> None:
|
||||
"""When a function_call_output has no matching function_call, the summary
|
||||
should show 'unknown_tool' instead of a blank name."""
|
||||
large = "x" * 1000
|
||||
# Deliberately omit the _func_call so the call_id has no name mapping
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_output("orphan_id", large),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer()
|
||||
result = trimmer(_make_data(items))
|
||||
trimmed = _output(result, 1)
|
||||
assert "unknown_tool" in trimmed
|
||||
assert "[Trimmed:" in trimmed
|
||||
|
||||
def test_unresolved_tool_skipped_with_allowlist(self) -> None:
|
||||
"""When trimmable_tools is set and the tool name can't be resolved,
|
||||
the output should NOT be trimmed (empty string won't match the allowlist)."""
|
||||
large = "x" * 1000
|
||||
items = [
|
||||
_user("q1"),
|
||||
_func_output("orphan_id", large),
|
||||
_assistant("a1"),
|
||||
_user("q2"),
|
||||
_assistant("a2"),
|
||||
_user("q3"),
|
||||
_assistant("a3"),
|
||||
]
|
||||
trimmer = ToolOutputTrimmer(trimmable_tools=frozenset({"search"}))
|
||||
result = trimmer(_make_data(items))
|
||||
# Unresolved tool name is "" which is not in the allowlist — left untouched
|
||||
assert _output(result, 1) == large
|
||||
@@ -0,0 +1,367 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
from openai.types.responses import (
|
||||
Response,
|
||||
ResponseApplyPatchToolCall,
|
||||
ResponseCompletedEvent,
|
||||
ResponseContentPartAddedEvent,
|
||||
ResponseContentPartDoneEvent,
|
||||
ResponseCreatedEvent,
|
||||
ResponseFunctionCallArgumentsDeltaEvent,
|
||||
ResponseFunctionCallArgumentsDoneEvent,
|
||||
ResponseFunctionToolCall,
|
||||
ResponseInProgressEvent,
|
||||
ResponseOutputItemAddedEvent,
|
||||
ResponseOutputItemDoneEvent,
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputText,
|
||||
ResponseReasoningSummaryPartAddedEvent,
|
||||
ResponseReasoningSummaryPartDoneEvent,
|
||||
ResponseReasoningSummaryTextDeltaEvent,
|
||||
ResponseReasoningSummaryTextDoneEvent,
|
||||
ResponseTextDeltaEvent,
|
||||
ResponseTextDoneEvent,
|
||||
ResponseUsage,
|
||||
)
|
||||
from openai.types.responses.response_reasoning_item import ResponseReasoningItem
|
||||
from openai.types.responses.response_reasoning_summary_part_added_event import (
|
||||
Part as AddedEventPart,
|
||||
)
|
||||
from openai.types.responses.response_reasoning_summary_part_done_event import Part as DoneEventPart
|
||||
from openai.types.responses.response_usage import InputTokensDetails, OutputTokensDetails
|
||||
|
||||
from agents.agent_output import AgentOutputSchemaBase
|
||||
from agents.handoffs import Handoff
|
||||
from agents.items import (
|
||||
ModelResponse,
|
||||
TResponseInputItem,
|
||||
TResponseOutputItem,
|
||||
TResponseStreamEvent,
|
||||
)
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import Model, ModelTracing
|
||||
from agents.tool import Tool
|
||||
from agents.tracing import SpanError, generation_span
|
||||
from agents.usage import Usage
|
||||
|
||||
|
||||
class FakeModel(Model):
|
||||
def __init__(
|
||||
self,
|
||||
tracing_enabled: bool = False,
|
||||
initial_output: list[TResponseOutputItem] | Exception | None = None,
|
||||
):
|
||||
if initial_output is None:
|
||||
initial_output = []
|
||||
self.turn_outputs: list[list[TResponseOutputItem] | Exception] = (
|
||||
[initial_output] if initial_output else []
|
||||
)
|
||||
self.tracing_enabled = tracing_enabled
|
||||
self.last_turn_args: dict[str, Any] = {}
|
||||
self.first_turn_args: dict[str, Any] | None = None
|
||||
self.hardcoded_usage: Usage | None = None
|
||||
|
||||
def set_hardcoded_usage(self, usage: Usage):
|
||||
self.hardcoded_usage = usage
|
||||
|
||||
def set_next_output(self, output: list[TResponseOutputItem] | Exception):
|
||||
self.turn_outputs.append(output)
|
||||
|
||||
def add_multiple_turn_outputs(self, outputs: list[list[TResponseOutputItem] | Exception]):
|
||||
self.turn_outputs.extend(outputs)
|
||||
|
||||
def get_next_output(self) -> list[TResponseOutputItem] | Exception:
|
||||
if not self.turn_outputs:
|
||||
return []
|
||||
return self.turn_outputs.pop(0)
|
||||
|
||||
async def get_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem],
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Tool],
|
||||
output_schema: AgentOutputSchemaBase | None,
|
||||
handoffs: list[Handoff],
|
||||
tracing: ModelTracing,
|
||||
*,
|
||||
previous_response_id: str | None,
|
||||
conversation_id: str | None,
|
||||
prompt: Any | None,
|
||||
) -> ModelResponse:
|
||||
turn_args = {
|
||||
"system_instructions": system_instructions,
|
||||
"input": input,
|
||||
"model_settings": model_settings,
|
||||
"tools": tools,
|
||||
"output_schema": output_schema,
|
||||
"previous_response_id": previous_response_id,
|
||||
"conversation_id": conversation_id,
|
||||
}
|
||||
|
||||
if self.first_turn_args is None:
|
||||
self.first_turn_args = turn_args.copy()
|
||||
|
||||
self.last_turn_args = turn_args
|
||||
|
||||
with generation_span(disabled=not self.tracing_enabled) as span:
|
||||
output = self.get_next_output()
|
||||
|
||||
if isinstance(output, Exception):
|
||||
span.set_error(
|
||||
SpanError(
|
||||
message="Error",
|
||||
data={
|
||||
"name": output.__class__.__name__,
|
||||
"message": str(output),
|
||||
},
|
||||
)
|
||||
)
|
||||
raise output
|
||||
|
||||
converted_output = []
|
||||
for item in output:
|
||||
if isinstance(item, dict) and item.get("type") == "apply_patch_call":
|
||||
call_id = str(item.get("call_id") or item.get("id") or "")
|
||||
converted_output.append(
|
||||
ResponseApplyPatchToolCall(
|
||||
type="apply_patch_call",
|
||||
id=str(item.get("id") or call_id),
|
||||
call_id=call_id,
|
||||
status=item.get("status") or "completed",
|
||||
operation=item.get("operation"),
|
||||
)
|
||||
)
|
||||
else:
|
||||
converted_output.append(item)
|
||||
|
||||
return ModelResponse(
|
||||
output=converted_output,
|
||||
usage=self.hardcoded_usage or Usage(),
|
||||
response_id="resp-789",
|
||||
)
|
||||
|
||||
async def stream_response(
|
||||
self,
|
||||
system_instructions: str | None,
|
||||
input: str | list[TResponseInputItem],
|
||||
model_settings: ModelSettings,
|
||||
tools: list[Tool],
|
||||
output_schema: AgentOutputSchemaBase | None,
|
||||
handoffs: list[Handoff],
|
||||
tracing: ModelTracing,
|
||||
*,
|
||||
previous_response_id: str | None = None,
|
||||
conversation_id: str | None = None,
|
||||
prompt: Any | None = None,
|
||||
) -> AsyncIterator[TResponseStreamEvent]:
|
||||
turn_args = {
|
||||
"system_instructions": system_instructions,
|
||||
"input": input,
|
||||
"model_settings": model_settings,
|
||||
"tools": tools,
|
||||
"output_schema": output_schema,
|
||||
"previous_response_id": previous_response_id,
|
||||
"conversation_id": conversation_id,
|
||||
}
|
||||
|
||||
if self.first_turn_args is None:
|
||||
self.first_turn_args = turn_args.copy()
|
||||
|
||||
self.last_turn_args = turn_args
|
||||
with generation_span(disabled=not self.tracing_enabled) as span:
|
||||
output = self.get_next_output()
|
||||
if isinstance(output, Exception):
|
||||
span.set_error(
|
||||
SpanError(
|
||||
message="Error",
|
||||
data={
|
||||
"name": output.__class__.__name__,
|
||||
"message": str(output),
|
||||
},
|
||||
)
|
||||
)
|
||||
raise output
|
||||
|
||||
response = get_response_obj(output, usage=self.hardcoded_usage)
|
||||
sequence_number = 0
|
||||
|
||||
yield ResponseCreatedEvent(
|
||||
type="response.created",
|
||||
response=response,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseInProgressEvent(
|
||||
type="response.in_progress",
|
||||
response=response,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
for output_index, output_item in enumerate(output):
|
||||
yield ResponseOutputItemAddedEvent(
|
||||
type="response.output_item.added",
|
||||
item=output_item,
|
||||
output_index=output_index,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
if isinstance(output_item, ResponseReasoningItem):
|
||||
if output_item.summary:
|
||||
for summary_index, summary in enumerate(output_item.summary):
|
||||
yield ResponseReasoningSummaryPartAddedEvent(
|
||||
type="response.reasoning_summary_part.added",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
summary_index=summary_index,
|
||||
part=AddedEventPart(text=summary.text, type=summary.type),
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseReasoningSummaryTextDeltaEvent(
|
||||
type="response.reasoning_summary_text.delta",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
summary_index=summary_index,
|
||||
delta=summary.text,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseReasoningSummaryTextDoneEvent(
|
||||
type="response.reasoning_summary_text.done",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
summary_index=summary_index,
|
||||
text=summary.text,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseReasoningSummaryPartDoneEvent(
|
||||
type="response.reasoning_summary_part.done",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
summary_index=summary_index,
|
||||
part=DoneEventPart(text=summary.text, type=summary.type),
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
elif isinstance(output_item, ResponseFunctionToolCall):
|
||||
yield ResponseFunctionCallArgumentsDeltaEvent(
|
||||
type="response.function_call_arguments.delta",
|
||||
item_id=output_item.call_id,
|
||||
output_index=output_index,
|
||||
delta=output_item.arguments,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseFunctionCallArgumentsDoneEvent(
|
||||
type="response.function_call_arguments.done",
|
||||
item_id=output_item.call_id,
|
||||
output_index=output_index,
|
||||
arguments=output_item.arguments,
|
||||
name=output_item.name,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
elif isinstance(output_item, ResponseOutputMessage):
|
||||
for content_index, content_part in enumerate(output_item.content or []):
|
||||
if isinstance(content_part, ResponseOutputText):
|
||||
yield ResponseContentPartAddedEvent(
|
||||
type="response.content_part.added",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
part=content_part,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseTextDeltaEvent(
|
||||
type="response.output_text.delta",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
delta=content_part.text,
|
||||
logprobs=[],
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseTextDoneEvent(
|
||||
type="response.output_text.done",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
text=content_part.text,
|
||||
logprobs=[],
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseContentPartDoneEvent(
|
||||
type="response.content_part.done",
|
||||
item_id=output_item.id,
|
||||
output_index=output_index,
|
||||
content_index=content_index,
|
||||
part=content_part,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseOutputItemDoneEvent(
|
||||
type="response.output_item.done",
|
||||
item=output_item,
|
||||
output_index=output_index,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
sequence_number += 1
|
||||
|
||||
yield ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=response,
|
||||
sequence_number=sequence_number,
|
||||
)
|
||||
|
||||
|
||||
class PromptCacheFakeModel(FakeModel):
|
||||
def _supports_default_prompt_cache_key(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def get_response_obj(
|
||||
output: list[TResponseOutputItem],
|
||||
response_id: str | None = None,
|
||||
usage: Usage | None = None,
|
||||
) -> Response:
|
||||
return Response(
|
||||
id=response_id or "resp-789",
|
||||
created_at=123,
|
||||
model="test_model",
|
||||
object="response",
|
||||
output=output,
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
top_p=None,
|
||||
parallel_tool_calls=False,
|
||||
usage=ResponseUsage(
|
||||
input_tokens=usage.input_tokens if usage else 0,
|
||||
output_tokens=usage.output_tokens if usage else 0,
|
||||
total_tokens=usage.total_tokens if usage else 0,
|
||||
input_tokens_details=InputTokensDetails.model_validate(
|
||||
{"cache_write_tokens": 0, "cached_tokens": 0}
|
||||
),
|
||||
output_tokens_details=OutputTokensDetails(reasoning_tokens=0),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,30 @@
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
from fastapi import FastAPI
|
||||
from starlette.responses import StreamingResponse
|
||||
|
||||
from agents import Agent, Runner, RunResultStreaming
|
||||
|
||||
agent = Agent(
|
||||
name="Assistant",
|
||||
instructions="You are a helpful assistant.",
|
||||
)
|
||||
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
|
||||
@app.post("/stream")
|
||||
async def stream():
|
||||
result = Runner.run_streamed(agent, input="Tell me a joke")
|
||||
stream_handler = StreamHandler(result)
|
||||
return StreamingResponse(stream_handler.stream_events(), media_type="application/x-ndjson")
|
||||
|
||||
|
||||
class StreamHandler:
|
||||
def __init__(self, result: RunResultStreaming):
|
||||
self.result = result
|
||||
|
||||
async def stream_events(self) -> AsyncIterator[str]:
|
||||
async for event in self.result.stream_events():
|
||||
yield f"{event.type}\n\n"
|
||||
@@ -0,0 +1,41 @@
|
||||
import pytest
|
||||
from httpx import ASGITransport, AsyncClient
|
||||
from inline_snapshot import snapshot
|
||||
|
||||
from ..fake_model import FakeModel
|
||||
from ..test_responses import get_text_message
|
||||
from .streaming_app import agent, app
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_context():
|
||||
"""This ensures that FastAPI streaming works. The context for this test is that the Runner
|
||||
method was called in one async context, and the streaming was ended in another context,
|
||||
leading to a tracing error because the context was closed in the wrong context. This test
|
||||
ensures that this actually works.
|
||||
"""
|
||||
model = FakeModel()
|
||||
agent.model = model
|
||||
model.set_next_output([get_text_message("done")])
|
||||
|
||||
transport = ASGITransport(app)
|
||||
async with AsyncClient(transport=transport, base_url="http://test") as ac:
|
||||
async with ac.stream("POST", "/stream") as r:
|
||||
assert r.status_code == 200
|
||||
body = (await r.aread()).decode("utf-8")
|
||||
lines = [line for line in body.splitlines() if line]
|
||||
assert lines == snapshot(
|
||||
[
|
||||
"agent_updated_stream_event",
|
||||
"raw_response_event", # ResponseCreatedEvent
|
||||
"raw_response_event", # ResponseInProgressEvent
|
||||
"raw_response_event", # ResponseOutputItemAddedEvent
|
||||
"raw_response_event", # ResponseContentPartAddedEvent
|
||||
"raw_response_event", # ResponseTextDeltaEvent
|
||||
"raw_response_event", # ResponseTextDoneEvent
|
||||
"raw_response_event", # ResponseContentPartDoneEvent
|
||||
"raw_response_event", # ResponseOutputItemDoneEvent
|
||||
"raw_response_event", # ResponseCompletedEvent
|
||||
"run_item_stream_event", # MessageOutputItem
|
||||
]
|
||||
)
|
||||
@@ -0,0 +1,164 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import shutil
|
||||
from typing import Any
|
||||
|
||||
from mcp import Tool as MCPTool
|
||||
from mcp.types import (
|
||||
CallToolResult,
|
||||
Content,
|
||||
GetPromptResult,
|
||||
ListPromptsResult,
|
||||
ListResourcesResult,
|
||||
ListResourceTemplatesResult,
|
||||
PromptMessage,
|
||||
ReadResourceResult,
|
||||
TextContent,
|
||||
)
|
||||
|
||||
from agents.mcp import MCPServer
|
||||
from agents.mcp.server import _UNSET, _MCPServerWithClientSession, _UnsetType
|
||||
from agents.mcp.util import MCPToolCustomDataExtractor, MCPToolMetaResolver, ToolFilter
|
||||
from agents.tool import ToolErrorFunction
|
||||
|
||||
tee = shutil.which("tee") or ""
|
||||
assert tee, "tee not found"
|
||||
|
||||
|
||||
# Added dummy stream classes for patching stdio_client to avoid real I/O during tests
|
||||
class DummyStream:
|
||||
async def send(self, msg):
|
||||
pass
|
||||
|
||||
async def receive(self):
|
||||
raise Exception("Dummy receive not implemented")
|
||||
|
||||
|
||||
class DummyStreamsContextManager:
|
||||
async def __aenter__(self):
|
||||
return (DummyStream(), DummyStream())
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
pass
|
||||
|
||||
|
||||
class _TestFilterServer(_MCPServerWithClientSession):
|
||||
"""Minimal implementation of _MCPServerWithClientSession for testing tool filtering"""
|
||||
|
||||
def __init__(self, tool_filter: ToolFilter, server_name: str):
|
||||
# Initialize parent class properly to avoid type errors
|
||||
super().__init__(
|
||||
cache_tools_list=False,
|
||||
client_session_timeout_seconds=None,
|
||||
tool_filter=tool_filter,
|
||||
)
|
||||
self._server_name: str = server_name
|
||||
# Override some attributes for test isolation
|
||||
self.session = None
|
||||
self._cleanup_lock = asyncio.Lock()
|
||||
|
||||
def create_streams(self):
|
||||
raise NotImplementedError("Not needed for filtering tests")
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._server_name
|
||||
|
||||
|
||||
class FakeMCPServer(MCPServer):
|
||||
def __init__(
|
||||
self,
|
||||
tools: list[MCPTool] | None = None,
|
||||
tool_filter: ToolFilter = None,
|
||||
server_name: str = "fake_mcp_server",
|
||||
require_approval: object | None = None,
|
||||
failure_error_function: ToolErrorFunction | None | _UnsetType = _UNSET,
|
||||
tool_meta_resolver: MCPToolMetaResolver | None = None,
|
||||
custom_data_extractor: MCPToolCustomDataExtractor | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
use_structured_content=False,
|
||||
require_approval=require_approval, # type: ignore[arg-type]
|
||||
failure_error_function=failure_error_function,
|
||||
tool_meta_resolver=tool_meta_resolver,
|
||||
custom_data_extractor=custom_data_extractor,
|
||||
)
|
||||
self.tools: list[MCPTool] = tools or []
|
||||
self.tool_calls: list[str] = []
|
||||
self.tool_results: list[str] = []
|
||||
self.tool_metas: list[dict[str, Any] | None] = []
|
||||
self.tool_filter = tool_filter
|
||||
self._server_name = server_name
|
||||
self._custom_content: list[Content] | None = None
|
||||
self._response_meta: dict[str, Any] | None = None
|
||||
|
||||
def add_tool(self, name: str, input_schema: dict[str, Any]):
|
||||
self.tools.append(MCPTool(name=name, inputSchema=input_schema))
|
||||
|
||||
async def connect(self):
|
||||
pass
|
||||
|
||||
async def cleanup(self):
|
||||
pass
|
||||
|
||||
async def list_tools(self, run_context=None, agent=None):
|
||||
tools = self.tools
|
||||
|
||||
# Apply tool filtering using the REAL implementation
|
||||
if self.tool_filter is not None:
|
||||
# Use the real _MCPServerWithClientSession filtering logic
|
||||
filter_server = _TestFilterServer(self.tool_filter, self.name)
|
||||
tools = await filter_server._apply_tool_filter(tools, run_context, agent)
|
||||
|
||||
return tools
|
||||
|
||||
async def call_tool(
|
||||
self,
|
||||
tool_name: str,
|
||||
arguments: dict[str, Any] | None,
|
||||
meta: dict[str, Any] | None = None,
|
||||
) -> CallToolResult:
|
||||
self.tool_calls.append(tool_name)
|
||||
self.tool_results.append(f"result_{tool_name}_{json.dumps(arguments)}")
|
||||
self.tool_metas.append(meta)
|
||||
|
||||
# Allow testing custom content scenarios
|
||||
if self._custom_content is not None:
|
||||
return CallToolResult(content=self._custom_content)
|
||||
|
||||
return CallToolResult(
|
||||
content=[TextContent(text=self.tool_results[-1], type="text")],
|
||||
_meta=self._response_meta,
|
||||
)
|
||||
|
||||
async def list_prompts(self, run_context=None, agent=None) -> ListPromptsResult:
|
||||
"""Return empty list of prompts for fake server"""
|
||||
return ListPromptsResult(prompts=[])
|
||||
|
||||
async def get_prompt(
|
||||
self, name: str, arguments: dict[str, Any] | None = None
|
||||
) -> GetPromptResult:
|
||||
"""Return a simple prompt result for fake server"""
|
||||
content = f"Fake prompt content for {name}"
|
||||
message = PromptMessage(role="user", content=TextContent(type="text", text=content))
|
||||
return GetPromptResult(description=f"Fake prompt: {name}", messages=[message])
|
||||
|
||||
async def list_resources(self, cursor: str | None = None) -> ListResourcesResult:
|
||||
"""Return empty list of resources for fake server."""
|
||||
return ListResourcesResult(resources=[])
|
||||
|
||||
async def list_resource_templates(
|
||||
self, cursor: str | None = None
|
||||
) -> ListResourceTemplatesResult:
|
||||
"""Return empty list of resource templates for fake server."""
|
||||
return ListResourceTemplatesResult(resourceTemplates=[])
|
||||
|
||||
async def read_resource(self, uri: str) -> ReadResourceResult:
|
||||
"""Return empty resource contents for fake server."""
|
||||
return ReadResourceResult(contents=[])
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._server_name
|
||||
@@ -0,0 +1,63 @@
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from mcp.types import ListToolsResult, Tool as MCPTool
|
||||
|
||||
from agents import Agent
|
||||
from agents.mcp import MCPServerStdio
|
||||
from agents.run_context import RunContextWrapper
|
||||
|
||||
from .helpers import DummyStreamsContextManager, tee
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("mcp.client.stdio.stdio_client", return_value=DummyStreamsContextManager())
|
||||
@patch("mcp.client.session.ClientSession.initialize", new_callable=AsyncMock, return_value=None)
|
||||
@patch("mcp.client.session.ClientSession.list_tools")
|
||||
async def test_server_caching_works(
|
||||
mock_list_tools: AsyncMock, mock_initialize: AsyncMock, mock_stdio_client
|
||||
):
|
||||
"""Test that if we turn caching on, the list of tools is cached and not fetched from the server
|
||||
on each call to `list_tools()`.
|
||||
"""
|
||||
server = MCPServerStdio(
|
||||
params={
|
||||
"command": tee,
|
||||
},
|
||||
cache_tools_list=True,
|
||||
)
|
||||
|
||||
tools = [
|
||||
MCPTool(name="tool1", inputSchema={}),
|
||||
MCPTool(name="tool2", inputSchema={}),
|
||||
]
|
||||
|
||||
mock_list_tools.return_value = ListToolsResult(tools=tools)
|
||||
|
||||
async with server:
|
||||
# Create test context and agent
|
||||
run_context = RunContextWrapper(context=None)
|
||||
agent = Agent(name="test_agent", instructions="Test agent")
|
||||
|
||||
# Call list_tools() multiple times
|
||||
result_tools = await server.list_tools(run_context, agent)
|
||||
assert result_tools == tools
|
||||
|
||||
assert mock_list_tools.call_count == 1, "list_tools() should have been called once"
|
||||
|
||||
# Call list_tools() again, should return the cached value
|
||||
result_tools = await server.list_tools(run_context, agent)
|
||||
assert result_tools == tools
|
||||
|
||||
assert mock_list_tools.call_count == 1, "list_tools() should not have been called again"
|
||||
|
||||
# Invalidate the cache and call list_tools() again
|
||||
server.invalidate_tools_cache()
|
||||
result_tools = await server.list_tools(run_context, agent)
|
||||
assert result_tools == tools
|
||||
|
||||
assert mock_list_tools.call_count == 2, "list_tools() should be called again"
|
||||
|
||||
# Without invalidating the cache, calling list_tools() again should return the cached value
|
||||
result_tools = await server.list_tools(run_context, agent)
|
||||
assert result_tools == tools
|
||||
@@ -0,0 +1,574 @@
|
||||
import asyncio
|
||||
import sys
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import cast
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from anyio import ClosedResourceError
|
||||
from mcp import ClientSession, Tool as MCPTool
|
||||
from mcp.shared.exceptions import McpError
|
||||
from mcp.types import CallToolResult, ErrorData, GetPromptResult, ListPromptsResult, ListToolsResult
|
||||
|
||||
from agents.exceptions import UserError
|
||||
from agents.mcp.server import MCPServerStreamableHttp, _MCPServerWithClientSession
|
||||
|
||||
if sys.version_info < (3, 11):
|
||||
from exceptiongroup import BaseExceptionGroup # pyright: ignore[reportMissingImports]
|
||||
|
||||
|
||||
class DummySession:
|
||||
def __init__(self, fail_call_tool: int = 0, fail_list_tools: int = 0):
|
||||
self.fail_call_tool = fail_call_tool
|
||||
self.fail_list_tools = fail_list_tools
|
||||
self.call_tool_attempts = 0
|
||||
self.list_tools_attempts = 0
|
||||
|
||||
async def call_tool(self, tool_name, arguments, meta=None):
|
||||
self.call_tool_attempts += 1
|
||||
if self.call_tool_attempts <= self.fail_call_tool:
|
||||
raise RuntimeError("call_tool failure")
|
||||
return CallToolResult(content=[])
|
||||
|
||||
async def list_tools(self):
|
||||
self.list_tools_attempts += 1
|
||||
if self.list_tools_attempts <= self.fail_list_tools:
|
||||
raise RuntimeError("list_tools failure")
|
||||
return ListToolsResult(tools=[MCPTool(name="tool", inputSchema={})])
|
||||
|
||||
|
||||
class DummyServer(_MCPServerWithClientSession):
|
||||
def __init__(self, session: DummySession, retries: int, *, serialize_requests: bool = False):
|
||||
super().__init__(
|
||||
cache_tools_list=False,
|
||||
client_session_timeout_seconds=None,
|
||||
max_retry_attempts=retries,
|
||||
retry_backoff_seconds_base=0,
|
||||
)
|
||||
self.session = cast(ClientSession, session)
|
||||
self._serialize_session_requests = serialize_requests
|
||||
|
||||
def create_streams(self):
|
||||
raise NotImplementedError
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "dummy"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_tool_retries_until_success():
|
||||
session = DummySession(fail_call_tool=2)
|
||||
server = DummyServer(session=session, retries=2)
|
||||
result = await server.call_tool("tool", None)
|
||||
assert isinstance(result, CallToolResult)
|
||||
assert session.call_tool_attempts == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_tools_unlimited_retries():
|
||||
session = DummySession(fail_list_tools=3)
|
||||
server = DummyServer(session=session, retries=-1)
|
||||
tools = await server.list_tools()
|
||||
assert len(tools) == 1
|
||||
assert tools[0].name == "tool"
|
||||
assert session.list_tools_attempts == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_tool_validates_required_parameters_before_remote_call():
|
||||
session = DummySession()
|
||||
server = DummyServer(session=session, retries=0)
|
||||
server._tools_list = [ # noqa: SLF001
|
||||
MCPTool(
|
||||
name="tool",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {"param_a": {"type": "string"}},
|
||||
"required": ["param_a"],
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
with pytest.raises(UserError, match="missing required parameters: param_a"):
|
||||
await server.call_tool("tool", {})
|
||||
|
||||
assert session.call_tool_attempts == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_tool_with_required_parameters_still_calls_remote_tool():
|
||||
session = DummySession()
|
||||
server = DummyServer(session=session, retries=0)
|
||||
server._tools_list = [ # noqa: SLF001
|
||||
MCPTool(
|
||||
name="tool",
|
||||
inputSchema={
|
||||
"type": "object",
|
||||
"properties": {"param_a": {"type": "string"}},
|
||||
"required": ["param_a"],
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
result = await server.call_tool("tool", {"param_a": "value"})
|
||||
assert isinstance(result, CallToolResult)
|
||||
assert session.call_tool_attempts == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_tool_skips_validation_when_tool_is_missing_from_cache():
|
||||
session = DummySession()
|
||||
server = DummyServer(session=session, retries=0)
|
||||
server._tools_list = [MCPTool(name="different_tool", inputSchema={"required": ["param_a"]})] # noqa: SLF001
|
||||
|
||||
await server.call_tool("tool", {})
|
||||
assert session.call_tool_attempts == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_tool_skips_validation_when_required_list_is_absent():
|
||||
session = DummySession()
|
||||
server = DummyServer(session=session, retries=0)
|
||||
server._tools_list = [MCPTool(name="tool", inputSchema={"type": "object"})] # noqa: SLF001
|
||||
|
||||
await server.call_tool("tool", None)
|
||||
assert session.call_tool_attempts == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_tool_validates_required_parameters_when_arguments_is_none():
|
||||
session = DummySession()
|
||||
server = DummyServer(session=session, retries=0)
|
||||
server._tools_list = [MCPTool(name="tool", inputSchema={"required": ["param_a"]})] # noqa: SLF001
|
||||
|
||||
with pytest.raises(UserError, match="missing required parameters: param_a"):
|
||||
await server.call_tool("tool", None)
|
||||
|
||||
assert session.call_tool_attempts == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_tool_rejects_non_object_arguments_before_remote_call():
|
||||
session = DummySession()
|
||||
server = DummyServer(session=session, retries=0)
|
||||
server._tools_list = [MCPTool(name="tool", inputSchema={"required": ["param_a"]})] # noqa: SLF001
|
||||
|
||||
with pytest.raises(UserError, match="arguments must be an object"):
|
||||
await server.call_tool("tool", cast(dict[str, object] | None, ["bad"]))
|
||||
|
||||
assert session.call_tool_attempts == 0
|
||||
|
||||
|
||||
class ConcurrentCancellationSession:
|
||||
def __init__(self):
|
||||
self._slow_task: asyncio.Task[CallToolResult] | None = None
|
||||
self._slow_started = asyncio.Event()
|
||||
|
||||
async def call_tool(self, tool_name, arguments, meta=None):
|
||||
if tool_name == "slow":
|
||||
self._slow_task = cast(asyncio.Task[CallToolResult], asyncio.current_task())
|
||||
self._slow_started.set()
|
||||
await asyncio.sleep(0.1)
|
||||
return CallToolResult(content=[])
|
||||
|
||||
await self._slow_started.wait()
|
||||
assert self._slow_task is not None
|
||||
self._slow_task.cancel()
|
||||
raise RuntimeError("synthetic request failure")
|
||||
|
||||
|
||||
class CancelledToolSession:
|
||||
async def call_tool(self, tool_name, arguments, meta=None):
|
||||
raise asyncio.CancelledError("synthetic call cancellation")
|
||||
|
||||
|
||||
class MixedExceptionGroupSession:
|
||||
async def call_tool(self, tool_name, arguments, meta=None):
|
||||
req = httpx.Request("POST", "https://example.test/mcp")
|
||||
resp = httpx.Response(401, request=req)
|
||||
raise BaseExceptionGroup(
|
||||
"mixed request failure",
|
||||
[
|
||||
asyncio.CancelledError("synthetic call cancellation"),
|
||||
httpx.HTTPStatusError("HTTP error 401", request=req, response=resp),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class SharedHttpStatusSession:
|
||||
def __init__(self, status_code: int):
|
||||
self.status_code = status_code
|
||||
|
||||
async def call_tool(self, tool_name, arguments, meta=None):
|
||||
req = httpx.Request("POST", "https://example.test/mcp")
|
||||
resp = httpx.Response(self.status_code, request=req)
|
||||
raise httpx.HTTPStatusError(
|
||||
f"HTTP error {self.status_code}",
|
||||
request=req,
|
||||
response=resp,
|
||||
)
|
||||
|
||||
|
||||
class TimeoutSession:
|
||||
def __init__(self, message: str = "timed out"):
|
||||
self.call_tool_attempts = 0
|
||||
self.message = message
|
||||
|
||||
async def call_tool(self, tool_name, arguments, meta=None):
|
||||
self.call_tool_attempts += 1
|
||||
raise httpx.TimeoutException(self.message)
|
||||
|
||||
|
||||
class ClosedResourceSession:
|
||||
def __init__(self):
|
||||
self.call_tool_attempts = 0
|
||||
|
||||
async def call_tool(self, tool_name, arguments, meta=None):
|
||||
self.call_tool_attempts += 1
|
||||
raise ClosedResourceError()
|
||||
|
||||
|
||||
class McpRequestTimeoutSession:
|
||||
def __init__(self, message: str = "timed out"):
|
||||
self.call_tool_attempts = 0
|
||||
self.message = message
|
||||
|
||||
async def call_tool(self, tool_name, arguments, meta=None):
|
||||
self.call_tool_attempts += 1
|
||||
raise McpError(
|
||||
ErrorData(code=httpx.codes.REQUEST_TIMEOUT, message=self.message),
|
||||
)
|
||||
|
||||
|
||||
class IsolatedRetrySession:
|
||||
def __init__(self):
|
||||
self.call_tool_attempts = 0
|
||||
|
||||
async def call_tool(self, tool_name, arguments, meta=None):
|
||||
self.call_tool_attempts += 1
|
||||
return CallToolResult(content=[])
|
||||
|
||||
|
||||
class HangingSession:
|
||||
async def call_tool(self, tool_name, arguments, meta=None):
|
||||
await asyncio.sleep(10)
|
||||
|
||||
|
||||
class DummyStreamableHttpServer(MCPServerStreamableHttp):
|
||||
def __init__(self, shared_session: object, isolated_session: object):
|
||||
super().__init__(
|
||||
params={"url": "https://example.test/mcp"},
|
||||
client_session_timeout_seconds=None,
|
||||
max_retry_attempts=0,
|
||||
)
|
||||
self.session = cast(ClientSession, shared_session)
|
||||
self._isolated_session = cast(ClientSession, isolated_session)
|
||||
|
||||
@asynccontextmanager
|
||||
async def _isolated_client_session(self):
|
||||
yield self._isolated_session
|
||||
|
||||
async def list_tools(self, run_context=None, agent=None):
|
||||
return [MCPTool(name="tool", inputSchema={})]
|
||||
|
||||
async def list_prompts(self):
|
||||
return ListPromptsResult(prompts=[])
|
||||
|
||||
async def get_prompt(self, name, arguments=None):
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class IsolatedSessionEnterFailure:
|
||||
def __init__(self, server: "EnterFailingStreamableHttpServer", message: str):
|
||||
self.server = server
|
||||
self.message = message
|
||||
|
||||
async def __aenter__(self):
|
||||
self.server.isolated_enter_attempts += 1
|
||||
raise httpx.TimeoutException(self.message)
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
class EnterFailingStreamableHttpServer(DummyStreamableHttpServer):
|
||||
def __init__(self, shared_session: object, *, isolated_message: str):
|
||||
super().__init__(shared_session, IsolatedRetrySession())
|
||||
self.isolated_enter_attempts = 0
|
||||
self._isolated_message = isolated_message
|
||||
|
||||
def _isolated_client_session(self):
|
||||
return IsolatedSessionEnterFailure(self, self._isolated_message)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_http_retries_cancelled_request_on_isolated_session():
|
||||
shared_session = CancelledToolSession()
|
||||
isolated_session = IsolatedRetrySession()
|
||||
server = DummyStreamableHttpServer(shared_session, isolated_session)
|
||||
server.max_retry_attempts = 1
|
||||
|
||||
result = await server.call_tool("tool", None)
|
||||
|
||||
assert isinstance(result, CallToolResult)
|
||||
assert isolated_session.call_tool_attempts == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_http_retries_5xx_on_isolated_session():
|
||||
isolated_session = IsolatedRetrySession()
|
||||
server = DummyStreamableHttpServer(SharedHttpStatusSession(504), isolated_session)
|
||||
server.max_retry_attempts = 1
|
||||
|
||||
result = await server.call_tool("tool", None)
|
||||
|
||||
assert isinstance(result, CallToolResult)
|
||||
assert isolated_session.call_tool_attempts == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_http_retries_closed_resource_on_isolated_session():
|
||||
isolated_session = IsolatedRetrySession()
|
||||
server = DummyStreamableHttpServer(ClosedResourceSession(), isolated_session)
|
||||
server.max_retry_attempts = 1
|
||||
|
||||
result = await server.call_tool("tool", None)
|
||||
|
||||
assert isinstance(result, CallToolResult)
|
||||
assert isolated_session.call_tool_attempts == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_http_retries_mcp_408_on_isolated_session():
|
||||
isolated_session = IsolatedRetrySession()
|
||||
server = DummyStreamableHttpServer(
|
||||
McpRequestTimeoutSession("Timed out while waiting for response to ClientRequest."),
|
||||
isolated_session,
|
||||
)
|
||||
server.max_retry_attempts = 1
|
||||
|
||||
result = await server.call_tool("tool", None)
|
||||
|
||||
assert isinstance(result, CallToolResult)
|
||||
assert isolated_session.call_tool_attempts == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_http_does_not_retry_4xx_on_isolated_session():
|
||||
isolated_session = IsolatedRetrySession()
|
||||
server = DummyStreamableHttpServer(SharedHttpStatusSession(401), isolated_session)
|
||||
|
||||
with pytest.raises(UserError, match="HTTP error 401"):
|
||||
await server.call_tool("tool", None)
|
||||
|
||||
assert isolated_session.call_tool_attempts == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_http_does_not_isolated_retry_without_retry_budget():
|
||||
isolated_session = IsolatedRetrySession()
|
||||
server = DummyStreamableHttpServer(CancelledToolSession(), isolated_session)
|
||||
server.max_retry_attempts = 0
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await server.call_tool("tool", None)
|
||||
|
||||
assert isolated_session.call_tool_attempts == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_http_counts_isolated_retry_against_retry_budget():
|
||||
shared_session = TimeoutSession("shared timed out")
|
||||
isolated_session = TimeoutSession("isolated timed out")
|
||||
server = DummyStreamableHttpServer(shared_session, isolated_session)
|
||||
server.max_retry_attempts = 2
|
||||
|
||||
with pytest.raises(httpx.TimeoutException, match="shared timed out"):
|
||||
await server.call_tool("tool", None)
|
||||
|
||||
assert shared_session.call_tool_attempts == 2
|
||||
assert isolated_session.call_tool_attempts == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_http_counts_isolated_session_setup_failure_against_retry_budget():
|
||||
shared_session = TimeoutSession("shared timed out")
|
||||
server = EnterFailingStreamableHttpServer(
|
||||
shared_session,
|
||||
isolated_message="isolated setup timed out",
|
||||
)
|
||||
server.max_retry_attempts = 2
|
||||
|
||||
with pytest.raises(httpx.TimeoutException, match="shared timed out"):
|
||||
await server.call_tool("tool", None)
|
||||
|
||||
assert shared_session.call_tool_attempts == 2
|
||||
assert server.isolated_enter_attempts == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_http_does_not_retry_mixed_exception_groups():
|
||||
isolated_session = IsolatedRetrySession()
|
||||
server = DummyStreamableHttpServer(MixedExceptionGroupSession(), isolated_session)
|
||||
server.max_retry_attempts = 1
|
||||
|
||||
with pytest.raises(UserError, match="HTTP error 401"):
|
||||
await server.call_tool("tool", None)
|
||||
|
||||
assert isolated_session.call_tool_attempts == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_http_preserves_outer_cancellation():
|
||||
isolated_session = IsolatedRetrySession()
|
||||
server = DummyStreamableHttpServer(HangingSession(), isolated_session)
|
||||
|
||||
task = asyncio.create_task(server.call_tool("slow", None))
|
||||
await asyncio.sleep(0)
|
||||
task.cancel()
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
assert isolated_session.call_tool_attempts == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_http_preserves_outer_cancellation_during_isolated_retry():
|
||||
server = DummyStreamableHttpServer(CancelledToolSession(), HangingSession())
|
||||
server.max_retry_attempts = 1
|
||||
|
||||
task = asyncio.create_task(server.call_tool("tool", None))
|
||||
await asyncio.sleep(0)
|
||||
task.cancel()
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
|
||||
|
||||
class ConcurrentPromptCancellationSession(ConcurrentCancellationSession):
|
||||
async def list_tools(self):
|
||||
return ListToolsResult(tools=[MCPTool(name="tool", inputSchema={})])
|
||||
|
||||
async def list_prompts(self):
|
||||
await self._slow_started.wait()
|
||||
assert self._slow_task is not None
|
||||
self._slow_task.cancel()
|
||||
raise RuntimeError("synthetic request failure")
|
||||
|
||||
async def get_prompt(self, name, arguments=None):
|
||||
await self._slow_started.wait()
|
||||
assert self._slow_task is not None
|
||||
self._slow_task.cancel()
|
||||
raise RuntimeError("synthetic request failure")
|
||||
|
||||
|
||||
class OverlapTrackingSession:
|
||||
def __init__(self):
|
||||
self.in_flight = 0
|
||||
self.max_in_flight = 0
|
||||
|
||||
@asynccontextmanager
|
||||
async def _enter_request(self):
|
||||
self.in_flight += 1
|
||||
self.max_in_flight = max(self.max_in_flight, self.in_flight)
|
||||
try:
|
||||
await asyncio.sleep(0.02)
|
||||
yield
|
||||
finally:
|
||||
self.in_flight -= 1
|
||||
|
||||
async def call_tool(self, tool_name, arguments, meta=None):
|
||||
async with self._enter_request():
|
||||
return CallToolResult(content=[])
|
||||
|
||||
async def list_prompts(self):
|
||||
async with self._enter_request():
|
||||
return ListPromptsResult(prompts=[])
|
||||
|
||||
async def get_prompt(self, name, arguments=None):
|
||||
async with self._enter_request():
|
||||
return GetPromptResult(
|
||||
description=None,
|
||||
messages=[],
|
||||
)
|
||||
|
||||
|
||||
class DummyPromptStreamableHttpServer(DummyStreamableHttpServer):
|
||||
def __init__(
|
||||
self,
|
||||
shared_session: OverlapTrackingSession,
|
||||
isolated_session: IsolatedRetrySession,
|
||||
):
|
||||
super().__init__(shared_session, isolated_session)
|
||||
self.session = cast(ClientSession, shared_session)
|
||||
|
||||
async def list_prompts(self):
|
||||
session = self.session
|
||||
assert session is not None
|
||||
return await self._maybe_serialize_request(lambda: session.list_prompts())
|
||||
|
||||
async def get_prompt(self, name, arguments=None):
|
||||
session = self.session
|
||||
assert session is not None
|
||||
return await self._maybe_serialize_request(lambda: session.get_prompt(name, arguments))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_serialized_session_requests_prevent_sibling_cancellation():
|
||||
session = ConcurrentPromptCancellationSession()
|
||||
server = DummyServer(session=cast(DummySession, session), retries=0, serialize_requests=True)
|
||||
|
||||
results = await asyncio.gather(
|
||||
server.call_tool("slow", None),
|
||||
server.call_tool("fail", None),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
assert isinstance(results[0], CallToolResult)
|
||||
assert isinstance(results[1], RuntimeError)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("prompt_method", ["list_prompts", "get_prompt"])
|
||||
async def test_serialized_prompt_requests_prevent_tool_cancellation(prompt_method: str):
|
||||
session = ConcurrentPromptCancellationSession()
|
||||
server = DummyServer(session=cast(DummySession, session), retries=0, serialize_requests=True)
|
||||
|
||||
prompt_request = (
|
||||
server.list_prompts() if prompt_method == "list_prompts" else server.get_prompt("prompt")
|
||||
)
|
||||
results = await asyncio.gather(
|
||||
server.call_tool("slow", None),
|
||||
prompt_request,
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
assert isinstance(results[0], CallToolResult)
|
||||
assert isinstance(results[1], RuntimeError)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("prompt_method", ["list_prompts", "get_prompt"])
|
||||
async def test_streamable_http_serializes_call_tool_with_prompt_requests(prompt_method: str):
|
||||
shared_session = OverlapTrackingSession()
|
||||
isolated_session = IsolatedRetrySession()
|
||||
server = DummyPromptStreamableHttpServer(shared_session, isolated_session)
|
||||
|
||||
prompt_request = (
|
||||
server.list_prompts() if prompt_method == "list_prompts" else server.get_prompt("prompt")
|
||||
)
|
||||
results = await asyncio.gather(
|
||||
server.call_tool("slow", None),
|
||||
prompt_request,
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
assert isinstance(results[0], CallToolResult)
|
||||
if prompt_method == "list_prompts":
|
||||
assert isinstance(results[1], ListPromptsResult)
|
||||
else:
|
||||
assert isinstance(results[1], GetPromptResult)
|
||||
assert shared_session.max_in_flight == 1
|
||||
assert isolated_session.call_tool_attempts == 0
|
||||
@@ -0,0 +1,69 @@
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from mcp.types import ListToolsResult, Tool as MCPTool
|
||||
|
||||
from agents.mcp import MCPServerStdio
|
||||
|
||||
from .helpers import DummyStreamsContextManager, tee
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("mcp.client.stdio.stdio_client", return_value=DummyStreamsContextManager())
|
||||
@patch("mcp.client.session.ClientSession.initialize", new_callable=AsyncMock, return_value=None)
|
||||
@patch("mcp.client.session.ClientSession.list_tools")
|
||||
async def test_async_ctx_manager_works(
|
||||
mock_list_tools: AsyncMock, mock_initialize: AsyncMock, mock_stdio_client
|
||||
):
|
||||
"""Test that the async context manager works."""
|
||||
server = MCPServerStdio(
|
||||
params={
|
||||
"command": tee,
|
||||
},
|
||||
cache_tools_list=True,
|
||||
)
|
||||
|
||||
tools = [
|
||||
MCPTool(name="tool1", inputSchema={}),
|
||||
MCPTool(name="tool2", inputSchema={}),
|
||||
]
|
||||
|
||||
mock_list_tools.return_value = ListToolsResult(tools=tools)
|
||||
|
||||
assert server.session is None, "Server should not be connected"
|
||||
|
||||
async with server:
|
||||
assert server.session is not None, "Server should be connected"
|
||||
|
||||
assert server.session is None, "Server should be disconnected"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("mcp.client.stdio.stdio_client", return_value=DummyStreamsContextManager())
|
||||
@patch("mcp.client.session.ClientSession.initialize", new_callable=AsyncMock, return_value=None)
|
||||
@patch("mcp.client.session.ClientSession.list_tools")
|
||||
async def test_manual_connect_disconnect_works(
|
||||
mock_list_tools: AsyncMock, mock_initialize: AsyncMock, mock_stdio_client
|
||||
):
|
||||
"""Test that the async context manager works."""
|
||||
server = MCPServerStdio(
|
||||
params={
|
||||
"command": tee,
|
||||
},
|
||||
cache_tools_list=True,
|
||||
)
|
||||
|
||||
tools = [
|
||||
MCPTool(name="tool1", inputSchema={}),
|
||||
MCPTool(name="tool2", inputSchema={}),
|
||||
]
|
||||
|
||||
mock_list_tools.return_value = ListToolsResult(tools=tools)
|
||||
|
||||
assert server.session is None, "Server should not be connected"
|
||||
|
||||
await server.connect()
|
||||
assert server.session is not None, "Server should be connected"
|
||||
|
||||
await server.cleanup()
|
||||
assert server.session is None, "Server should be disconnected"
|
||||
@@ -0,0 +1,242 @@
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from mcp.types import Tool as MCPTool
|
||||
|
||||
from agents import Agent, RunContextWrapper, Runner
|
||||
from agents.exceptions import UserError
|
||||
|
||||
from ..fake_model import FakeModel
|
||||
from ..test_responses import get_function_tool_call, get_text_message
|
||||
from ..utils.hitl import queue_function_call_and_text, resume_after_first_approval
|
||||
from .helpers import FakeMCPServer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_require_approval_pauses_and_resumes():
|
||||
"""MCP servers should honor require_approval for non-hosted tools."""
|
||||
|
||||
server = FakeMCPServer(require_approval="always")
|
||||
server.add_tool("add", {"type": "object", "properties": {}})
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
|
||||
|
||||
queue_function_call_and_text(
|
||||
model,
|
||||
get_function_tool_call("add", "{}"),
|
||||
followup=[get_text_message("done")],
|
||||
)
|
||||
|
||||
first = await Runner.run(agent, "call add")
|
||||
|
||||
assert first.interruptions, "MCP tool should request approval"
|
||||
assert first.interruptions[0].tool_name == "add"
|
||||
|
||||
resumed = await resume_after_first_approval(agent, first, always_approve=True)
|
||||
|
||||
assert not resumed.interruptions
|
||||
assert server.tool_calls == ["add"]
|
||||
assert resumed.final_output == "done"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_require_approval_tool_lists():
|
||||
"""TS-style requireApproval toolNames should map to needs_approval."""
|
||||
|
||||
require_approval: dict[str, object] = {
|
||||
"always": {"tool_names": ["add"]},
|
||||
"never": {"tool_names": ["noop"]},
|
||||
}
|
||||
server = FakeMCPServer(require_approval=require_approval)
|
||||
server.add_tool("add", {"type": "object", "properties": {}})
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
|
||||
|
||||
queue_function_call_and_text(
|
||||
model,
|
||||
get_function_tool_call("add", "{}"),
|
||||
followup=[get_text_message("done")],
|
||||
)
|
||||
|
||||
first = await Runner.run(agent, "call add")
|
||||
assert first.interruptions, "add should require approval via require_approval toolNames"
|
||||
|
||||
resumed = await resume_after_first_approval(agent, first, always_approve=True)
|
||||
assert resumed.final_output == "done"
|
||||
assert server.tool_calls == ["add"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_require_approval_tool_mapping():
|
||||
"""Tool-name require_approval mappings should map to needs_approval."""
|
||||
|
||||
require_approval = {"add": "always", "noop": "never"}
|
||||
server = FakeMCPServer(require_approval=require_approval)
|
||||
server.add_tool("add", {"type": "object", "properties": {}})
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
|
||||
|
||||
queue_function_call_and_text(
|
||||
model,
|
||||
get_function_tool_call("add", "{}"),
|
||||
followup=[get_text_message("done")],
|
||||
)
|
||||
|
||||
first = await Runner.run(agent, "call add")
|
||||
assert first.interruptions, "add should require approval via require_approval mapping"
|
||||
|
||||
resumed = await resume_after_first_approval(agent, first, always_approve=True)
|
||||
assert resumed.final_output == "done"
|
||||
assert server.tool_calls == ["add"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_require_approval_mapping_allows_policy_keyword_tool_names():
|
||||
"""Tool-name mappings should treat literal 'always'/'never' as tool names."""
|
||||
|
||||
require_approval = {"always": "always", "never": "never"}
|
||||
server = FakeMCPServer(require_approval=require_approval)
|
||||
server.add_tool("always", {"type": "object", "properties": {}})
|
||||
server.add_tool("never", {"type": "object", "properties": {}})
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
|
||||
|
||||
queue_function_call_and_text(
|
||||
model,
|
||||
get_function_tool_call("always", "{}"),
|
||||
followup=[get_text_message("done")],
|
||||
)
|
||||
|
||||
first = await Runner.run(agent, "call always")
|
||||
assert first.interruptions, "tool named 'always' should require approval"
|
||||
assert first.interruptions[0].tool_name == "always"
|
||||
|
||||
resumed = await resume_after_first_approval(agent, first, always_approve=True)
|
||||
assert resumed.final_output == "done"
|
||||
|
||||
queue_function_call_and_text(
|
||||
model,
|
||||
get_function_tool_call("never", "{}"),
|
||||
followup=[get_text_message("done")],
|
||||
)
|
||||
|
||||
second = await Runner.run(agent, "call never")
|
||||
assert not second.interruptions, "tool named 'never' should not require approval"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("require_approval", "message"),
|
||||
[
|
||||
("alwyas", "expected 'always' or 'never'"),
|
||||
({"delete": "alwyas"}, "delete"),
|
||||
(
|
||||
{
|
||||
"always": {"tool_names": ["delete"]},
|
||||
"never": {"tool_names": ["delete"]},
|
||||
},
|
||||
"both always and never",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_mcp_require_approval_rejects_invalid_fail_open_policies(require_approval, message):
|
||||
"""Invalid MCP approval policies should not silently disable approvals."""
|
||||
|
||||
with pytest.raises(UserError, match=message):
|
||||
FakeMCPServer(require_approval=require_approval)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_require_approval_callable_can_allow_and_block_by_tool_name():
|
||||
"""Callable policies should decide approval dynamically for each MCP tool."""
|
||||
|
||||
seen: list[str] = []
|
||||
|
||||
def require_approval(
|
||||
_run_context: RunContextWrapper[object | None],
|
||||
_agent: Agent,
|
||||
tool: MCPTool,
|
||||
) -> bool:
|
||||
seen.append(tool.name)
|
||||
return tool.name == "guarded"
|
||||
|
||||
server = FakeMCPServer(require_approval=require_approval)
|
||||
server.add_tool("guarded", {"type": "object", "properties": {}})
|
||||
server.add_tool("safe", {"type": "object", "properties": {}})
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
|
||||
|
||||
queue_function_call_and_text(
|
||||
model,
|
||||
get_function_tool_call("guarded", "{}"),
|
||||
followup=[get_text_message("guarded done")],
|
||||
)
|
||||
first = await Runner.run(agent, "call guarded")
|
||||
assert first.interruptions, "guarded should require approval via callable policy"
|
||||
assert first.interruptions[0].tool_name == "guarded"
|
||||
|
||||
resumed = await resume_after_first_approval(agent, first, always_approve=True)
|
||||
assert resumed.final_output == "guarded done"
|
||||
|
||||
queue_function_call_and_text(
|
||||
model,
|
||||
get_function_tool_call("safe", "{}"),
|
||||
followup=[get_text_message("safe done")],
|
||||
)
|
||||
second = await Runner.run(agent, "call safe")
|
||||
assert not second.interruptions, "safe should bypass approval via callable policy"
|
||||
assert second.final_output == "safe done"
|
||||
|
||||
assert seen == ["guarded", "guarded", "safe"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_require_approval_async_callable_uses_run_context():
|
||||
"""Async callable policies should receive the run context and be awaited."""
|
||||
|
||||
seen_contexts: list[object | None] = []
|
||||
|
||||
async def require_approval(
|
||||
run_context: RunContextWrapper[dict[str, bool] | None],
|
||||
_agent: Agent,
|
||||
_tool,
|
||||
) -> bool:
|
||||
seen_contexts.append(run_context.context)
|
||||
await asyncio.sleep(0)
|
||||
return bool(run_context.context and run_context.context.get("needs_approval"))
|
||||
|
||||
server = FakeMCPServer(require_approval=require_approval)
|
||||
server.add_tool("conditional", {"type": "object", "properties": {}})
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="TestAgent", model=model, mcp_servers=[server])
|
||||
|
||||
queue_function_call_and_text(
|
||||
model,
|
||||
get_function_tool_call("conditional", "{}"),
|
||||
followup=[get_text_message("approved path")],
|
||||
)
|
||||
first = await Runner.run(agent, "call conditional", context={"needs_approval": True})
|
||||
assert first.interruptions, "run context should be able to trigger approval"
|
||||
|
||||
resumed = await resume_after_first_approval(agent, first, always_approve=True)
|
||||
assert resumed.final_output == "approved path"
|
||||
|
||||
queue_function_call_and_text(
|
||||
model,
|
||||
get_function_tool_call("conditional", "{}"),
|
||||
followup=[get_text_message("no approval path")],
|
||||
)
|
||||
second = await Runner.run(agent, "call conditional", context={"needs_approval": False})
|
||||
assert not second.interruptions, "run context should be able to skip approval"
|
||||
assert second.final_output == "no approval path"
|
||||
|
||||
assert seen_contexts == [
|
||||
{"needs_approval": True},
|
||||
{"needs_approval": True},
|
||||
{"needs_approval": False},
|
||||
]
|
||||
@@ -0,0 +1,179 @@
|
||||
"""Tests for auth and httpx_client_factory params on MCPServerSse and MCPServerStreamableHttp."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from agents.mcp import MCPServerSse, MCPServerStreamableHttp
|
||||
from agents.mcp.server import _create_default_streamable_http_client
|
||||
|
||||
|
||||
class TestMCPServerSseAuthAndFactory:
|
||||
"""Tests for auth and httpx_client_factory added to MCPServerSseParams."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sse_default_no_auth_no_factory(self):
|
||||
"""SSE create_streams falls back to the hardened default httpx_client_factory."""
|
||||
with patch("agents.mcp.server.sse_client") as mock_client:
|
||||
mock_client.return_value = MagicMock()
|
||||
server = MCPServerSse(params={"url": "http://localhost:8000/sse"})
|
||||
server.create_streams()
|
||||
mock_client.assert_called_once_with(
|
||||
url="http://localhost:8000/sse",
|
||||
headers=None,
|
||||
timeout=5,
|
||||
sse_read_timeout=300,
|
||||
httpx_client_factory=_create_default_streamable_http_client,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sse_with_auth(self):
|
||||
"""SSE create_streams forwards auth and still applies the hardened default factory."""
|
||||
auth = httpx.BasicAuth(username="user", password="pass")
|
||||
with patch("agents.mcp.server.sse_client") as mock_client:
|
||||
mock_client.return_value = MagicMock()
|
||||
server = MCPServerSse(params={"url": "http://localhost:8000/sse", "auth": auth})
|
||||
server.create_streams()
|
||||
mock_client.assert_called_once_with(
|
||||
url="http://localhost:8000/sse",
|
||||
headers=None,
|
||||
timeout=5,
|
||||
sse_read_timeout=300,
|
||||
auth=auth,
|
||||
httpx_client_factory=_create_default_streamable_http_client,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sse_with_httpx_client_factory(self):
|
||||
"""SSE create_streams forwards a custom httpx_client_factory when provided."""
|
||||
|
||||
def custom_factory(
|
||||
headers: dict[str, str] | None = None,
|
||||
timeout: httpx.Timeout | None = None,
|
||||
auth: httpx.Auth | None = None,
|
||||
) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(verify=False) # pragma: no cover
|
||||
|
||||
with patch("agents.mcp.server.sse_client") as mock_client:
|
||||
mock_client.return_value = MagicMock()
|
||||
server = MCPServerSse(
|
||||
params={
|
||||
"url": "http://localhost:8000/sse",
|
||||
"httpx_client_factory": custom_factory,
|
||||
}
|
||||
)
|
||||
server.create_streams()
|
||||
mock_client.assert_called_once_with(
|
||||
url="http://localhost:8000/sse",
|
||||
headers=None,
|
||||
timeout=5,
|
||||
sse_read_timeout=300,
|
||||
httpx_client_factory=custom_factory,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sse_with_auth_and_factory(self):
|
||||
"""SSE create_streams forwards both auth and httpx_client_factory together."""
|
||||
auth = httpx.BasicAuth(username="user", password="pass")
|
||||
|
||||
def custom_factory(
|
||||
headers: dict[str, str] | None = None,
|
||||
timeout: httpx.Timeout | None = None,
|
||||
auth: httpx.Auth | None = None,
|
||||
) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(verify=False) # pragma: no cover
|
||||
|
||||
with patch("agents.mcp.server.sse_client") as mock_client:
|
||||
mock_client.return_value = MagicMock()
|
||||
server = MCPServerSse(
|
||||
params={
|
||||
"url": "http://localhost:8000/sse",
|
||||
"headers": {"X-Token": "abc"},
|
||||
"auth": auth,
|
||||
"httpx_client_factory": custom_factory,
|
||||
}
|
||||
)
|
||||
server.create_streams()
|
||||
mock_client.assert_called_once_with(
|
||||
url="http://localhost:8000/sse",
|
||||
headers={"X-Token": "abc"},
|
||||
timeout=5,
|
||||
sse_read_timeout=300,
|
||||
auth=auth,
|
||||
httpx_client_factory=custom_factory,
|
||||
)
|
||||
|
||||
|
||||
class TestMCPServerStreamableHttpAuth:
|
||||
"""Tests for the auth parameter added to MCPServerStreamableHttpParams."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_http_default_no_auth(self):
|
||||
"""StreamableHttp create_streams omits auth when not provided."""
|
||||
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
|
||||
mock_client.return_value = MagicMock()
|
||||
server = MCPServerStreamableHttp(params={"url": "http://localhost:8000/mcp"})
|
||||
server.create_streams()
|
||||
mock_client.assert_called_once_with(
|
||||
url="http://localhost:8000/mcp",
|
||||
headers=None,
|
||||
timeout=5,
|
||||
sse_read_timeout=300,
|
||||
terminate_on_close=True,
|
||||
httpx_client_factory=_create_default_streamable_http_client,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_http_with_auth(self):
|
||||
"""StreamableHttp create_streams forwards the auth parameter when provided."""
|
||||
auth = httpx.BasicAuth(username="user", password="pass")
|
||||
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
|
||||
mock_client.return_value = MagicMock()
|
||||
server = MCPServerStreamableHttp(
|
||||
params={"url": "http://localhost:8000/mcp", "auth": auth}
|
||||
)
|
||||
server.create_streams()
|
||||
mock_client.assert_called_once_with(
|
||||
url="http://localhost:8000/mcp",
|
||||
headers=None,
|
||||
timeout=5,
|
||||
sse_read_timeout=300,
|
||||
terminate_on_close=True,
|
||||
httpx_client_factory=_create_default_streamable_http_client,
|
||||
auth=auth,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_http_with_auth_and_factory(self):
|
||||
"""StreamableHttp create_streams forwards both auth and httpx_client_factory."""
|
||||
auth = httpx.BasicAuth(username="user", password="pass")
|
||||
|
||||
def custom_factory(
|
||||
headers: dict[str, str] | None = None,
|
||||
timeout: httpx.Timeout | None = None,
|
||||
auth: httpx.Auth | None = None,
|
||||
) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(verify=False) # pragma: no cover
|
||||
|
||||
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
|
||||
mock_client.return_value = MagicMock()
|
||||
server = MCPServerStreamableHttp(
|
||||
params={
|
||||
"url": "http://localhost:8000/mcp",
|
||||
"auth": auth,
|
||||
"httpx_client_factory": custom_factory,
|
||||
}
|
||||
)
|
||||
server.create_streams()
|
||||
mock_client.assert_called_once_with(
|
||||
url="http://localhost:8000/mcp",
|
||||
headers=None,
|
||||
timeout=5,
|
||||
sse_read_timeout=300,
|
||||
terminate_on_close=True,
|
||||
auth=auth,
|
||||
httpx_client_factory=custom_factory,
|
||||
)
|
||||
@@ -0,0 +1,73 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import importlib.abc
|
||||
import sys
|
||||
from types import ModuleType
|
||||
|
||||
import pytest
|
||||
|
||||
_SERVER_EXPORTS = (
|
||||
"LocalMCPApprovalCallable",
|
||||
"MCPServer",
|
||||
"MCPServerSse",
|
||||
"MCPServerSseParams",
|
||||
"MCPServerStdio",
|
||||
"MCPServerStdioParams",
|
||||
"MCPServerStreamableHttp",
|
||||
"MCPServerStreamableHttpParams",
|
||||
)
|
||||
|
||||
|
||||
class _BrokenMCPServerImportFinder(importlib.abc.MetaPathFinder):
|
||||
def find_spec(
|
||||
self,
|
||||
fullname: str,
|
||||
path: object | None,
|
||||
target: ModuleType | None = None,
|
||||
) -> None:
|
||||
if fullname == "agents.mcp.server":
|
||||
raise ImportError("simulated dependency import failure")
|
||||
return None
|
||||
|
||||
|
||||
def _clear_mcp_server_imports(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
mcp_module: ModuleType,
|
||||
) -> None:
|
||||
monkeypatch.delitem(sys.modules, "agents.mcp.server", raising=False)
|
||||
monkeypatch.delitem(mcp_module.__dict__, "server", raising=False)
|
||||
for name in _SERVER_EXPORTS:
|
||||
monkeypatch.delitem(mcp_module.__dict__, name, raising=False)
|
||||
|
||||
|
||||
def test_mcp_package_import_does_not_eagerly_import_server(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import agents.mcp as mcp_module
|
||||
|
||||
_clear_mcp_server_imports(monkeypatch, mcp_module)
|
||||
finder = _BrokenMCPServerImportFinder()
|
||||
monkeypatch.setattr(sys, "meta_path", [finder, *sys.meta_path])
|
||||
|
||||
reloaded_mcp = importlib.reload(mcp_module)
|
||||
|
||||
assert reloaded_mcp.MCPUtil is not None
|
||||
|
||||
|
||||
def test_mcp_server_reexport_preserves_underlying_import_error(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
import agents.mcp as mcp_module
|
||||
|
||||
_clear_mcp_server_imports(monkeypatch, mcp_module)
|
||||
finder = _BrokenMCPServerImportFinder()
|
||||
monkeypatch.setattr(sys, "meta_path", [finder, *sys.meta_path])
|
||||
namespace: dict[str, object] = {}
|
||||
|
||||
with pytest.raises(ImportError) as exc_info:
|
||||
exec("from agents.mcp import MCPServerStreamableHttp", namespace)
|
||||
|
||||
assert "Failed to import MCPServerStreamableHttp from agents.mcp" in str(exc_info.value)
|
||||
assert isinstance(exc_info.value.__cause__, ImportError)
|
||||
assert "simulated dependency import failure" in str(exc_info.value.__cause__)
|
||||
@@ -0,0 +1,175 @@
|
||||
"""Tests for MCP server list_resources, list_resource_templates, and read_resource."""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from mcp.types import (
|
||||
ListResourcesResult,
|
||||
ListResourceTemplatesResult,
|
||||
ReadResourceResult,
|
||||
Resource,
|
||||
ResourceTemplate,
|
||||
TextResourceContents,
|
||||
)
|
||||
from pydantic import AnyUrl
|
||||
|
||||
from agents.mcp import MCPServerStreamableHttp
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def server():
|
||||
return MCPServerStreamableHttp(params={"url": "http://localhost:8000/mcp"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_resources_raises_when_not_connected(server: MCPServerStreamableHttp):
|
||||
"""list_resources raises UserError when server has not been connected."""
|
||||
from agents.exceptions import UserError
|
||||
|
||||
with pytest.raises(UserError, match="Server not initialized"):
|
||||
await server.list_resources()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_resource_templates_raises_when_not_connected(server: MCPServerStreamableHttp):
|
||||
"""list_resource_templates raises UserError when server has not been connected."""
|
||||
from agents.exceptions import UserError
|
||||
|
||||
with pytest.raises(UserError, match="Server not initialized"):
|
||||
await server.list_resource_templates()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_resource_raises_when_not_connected(server: MCPServerStreamableHttp):
|
||||
"""read_resource raises UserError when server has not been connected."""
|
||||
from agents.exceptions import UserError
|
||||
|
||||
with pytest.raises(UserError, match="Server not initialized"):
|
||||
await server.read_resource("file:///etc/hosts")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_resources_returns_result(server: MCPServerStreamableHttp):
|
||||
"""list_resources delegates to the underlying MCP session."""
|
||||
mock_session = MagicMock()
|
||||
expected = ListResourcesResult(
|
||||
resources=[
|
||||
Resource(uri=AnyUrl("file:///readme.md"), name="readme.md", mimeType="text/markdown"),
|
||||
]
|
||||
)
|
||||
mock_session.list_resources = AsyncMock(return_value=expected)
|
||||
server.session = mock_session
|
||||
|
||||
result = await server.list_resources()
|
||||
|
||||
assert result is expected
|
||||
mock_session.list_resources.assert_awaited_once_with(None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_resources_forwards_cursor(server: MCPServerStreamableHttp):
|
||||
"""list_resources forwards the cursor argument for pagination."""
|
||||
mock_session = MagicMock()
|
||||
page2 = ListResourcesResult(resources=[])
|
||||
mock_session.list_resources = AsyncMock(return_value=page2)
|
||||
server.session = mock_session
|
||||
|
||||
result = await server.list_resources(cursor="tok_abc")
|
||||
|
||||
assert result is page2
|
||||
mock_session.list_resources.assert_awaited_once_with("tok_abc")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_resource_templates_returns_result(server: MCPServerStreamableHttp):
|
||||
"""list_resource_templates delegates to the underlying MCP session."""
|
||||
mock_session = MagicMock()
|
||||
expected = ListResourceTemplatesResult(
|
||||
resourceTemplates=[
|
||||
ResourceTemplate(uriTemplate="file:///{path}", name="file"),
|
||||
]
|
||||
)
|
||||
mock_session.list_resource_templates = AsyncMock(return_value=expected)
|
||||
server.session = mock_session
|
||||
|
||||
result = await server.list_resource_templates()
|
||||
|
||||
assert result is expected
|
||||
mock_session.list_resource_templates.assert_awaited_once_with(None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_resource_templates_forwards_cursor(server: MCPServerStreamableHttp):
|
||||
"""list_resource_templates forwards the cursor argument for pagination."""
|
||||
mock_session = MagicMock()
|
||||
page2 = ListResourceTemplatesResult(resourceTemplates=[])
|
||||
mock_session.list_resource_templates = AsyncMock(return_value=page2)
|
||||
server.session = mock_session
|
||||
|
||||
result = await server.list_resource_templates(cursor="tok_xyz")
|
||||
|
||||
assert result is page2
|
||||
mock_session.list_resource_templates.assert_awaited_once_with("tok_xyz")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_resource_returns_result(server: MCPServerStreamableHttp):
|
||||
"""read_resource delegates to the underlying MCP session with the given URI."""
|
||||
mock_session = MagicMock()
|
||||
uri = "file:///readme.md"
|
||||
expected = ReadResourceResult(
|
||||
contents=[
|
||||
TextResourceContents(uri=AnyUrl(uri), text="# Hello", mimeType="text/markdown"),
|
||||
]
|
||||
)
|
||||
mock_session.read_resource = AsyncMock(return_value=expected)
|
||||
server.session = mock_session
|
||||
|
||||
result = await server.read_resource(uri)
|
||||
|
||||
assert result is expected
|
||||
mock_session.read_resource.assert_awaited_once_with(AnyUrl(uri))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_base_methods_raise_not_implemented():
|
||||
"""Bare MCPServer subclasses that don't override resource methods get NotImplementedError."""
|
||||
from mcp.types import CallToolResult, GetPromptResult, ListPromptsResult
|
||||
|
||||
from agents.mcp import MCPServer
|
||||
|
||||
class MinimalServer(MCPServer):
|
||||
"""Minimal subclass implementing only the truly abstract methods."""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "minimal"
|
||||
|
||||
async def connect(self) -> None:
|
||||
pass
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
pass
|
||||
|
||||
async def list_tools(self, run_context=None, agent=None):
|
||||
return []
|
||||
|
||||
async def call_tool(self, tool_name, tool_arguments, run_context=None, agent=None):
|
||||
return CallToolResult(content=[])
|
||||
|
||||
async def list_prompts(self):
|
||||
return ListPromptsResult(prompts=[])
|
||||
|
||||
async def get_prompt(self, name, arguments=None):
|
||||
return GetPromptResult(messages=[])
|
||||
|
||||
s = MinimalServer()
|
||||
|
||||
with pytest.raises(NotImplementedError, match="list_resources"):
|
||||
await s.list_resources()
|
||||
|
||||
with pytest.raises(NotImplementedError, match="list_resource_templates"):
|
||||
await s.list_resource_templates()
|
||||
|
||||
with pytest.raises(NotImplementedError, match="read_resource"):
|
||||
await s.read_resource("file:///test.txt")
|
||||
@@ -0,0 +1,552 @@
|
||||
import asyncio
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
from mcp.types import (
|
||||
CallToolResult,
|
||||
GetPromptResult,
|
||||
ListPromptsResult,
|
||||
ListResourcesResult,
|
||||
ListResourceTemplatesResult,
|
||||
ReadResourceResult,
|
||||
Tool as MCPTool,
|
||||
)
|
||||
|
||||
from agents.mcp import MCPServer, MCPServerManager
|
||||
from agents.run_context import RunContextWrapper
|
||||
|
||||
|
||||
class TaskBoundServer(MCPServer):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self._connect_task: asyncio.Task[object] | None = None
|
||||
self.cleaned = False
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "task-bound"
|
||||
|
||||
async def connect(self) -> None:
|
||||
self._connect_task = asyncio.current_task()
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
if self._connect_task is None:
|
||||
raise RuntimeError("Server was not connected")
|
||||
if asyncio.current_task() is not self._connect_task:
|
||||
raise RuntimeError("Attempted to exit cancel scope in a different task")
|
||||
self.cleaned = True
|
||||
|
||||
async def list_tools(
|
||||
self, run_context: RunContextWrapper[Any] | None = None, agent: Any | None = None
|
||||
) -> list[MCPTool]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def call_tool(
|
||||
self,
|
||||
tool_name: str,
|
||||
arguments: dict[str, Any] | None,
|
||||
meta: dict[str, Any] | None = None,
|
||||
) -> CallToolResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def list_prompts(self) -> ListPromptsResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_prompt(
|
||||
self, name: str, arguments: dict[str, Any] | None = None
|
||||
) -> GetPromptResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def list_resources(self, cursor: str | None = None) -> ListResourcesResult:
|
||||
return ListResourcesResult(resources=[])
|
||||
|
||||
async def list_resource_templates(
|
||||
self, cursor: str | None = None
|
||||
) -> ListResourceTemplatesResult:
|
||||
return ListResourceTemplatesResult(resourceTemplates=[])
|
||||
|
||||
async def read_resource(self, uri: str) -> ReadResourceResult:
|
||||
return ReadResourceResult(contents=[])
|
||||
|
||||
|
||||
class FlakyServer(MCPServer):
|
||||
def __init__(self, failures: int) -> None:
|
||||
super().__init__()
|
||||
self.failures_remaining = failures
|
||||
self.connect_calls = 0
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "flaky"
|
||||
|
||||
async def connect(self) -> None:
|
||||
self.connect_calls += 1
|
||||
if self.failures_remaining > 0:
|
||||
self.failures_remaining -= 1
|
||||
raise RuntimeError("connect failed")
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
return None
|
||||
|
||||
async def list_tools(
|
||||
self, run_context: RunContextWrapper[Any] | None = None, agent: Any | None = None
|
||||
) -> list[MCPTool]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def call_tool(
|
||||
self,
|
||||
tool_name: str,
|
||||
arguments: dict[str, Any] | None,
|
||||
meta: dict[str, Any] | None = None,
|
||||
) -> CallToolResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def list_prompts(self) -> ListPromptsResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_prompt(
|
||||
self, name: str, arguments: dict[str, Any] | None = None
|
||||
) -> GetPromptResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def list_resources(self, cursor: str | None = None) -> ListResourcesResult:
|
||||
return ListResourcesResult(resources=[])
|
||||
|
||||
async def list_resource_templates(
|
||||
self, cursor: str | None = None
|
||||
) -> ListResourceTemplatesResult:
|
||||
return ListResourceTemplatesResult(resourceTemplates=[])
|
||||
|
||||
async def read_resource(self, uri: str) -> ReadResourceResult:
|
||||
return ReadResourceResult(contents=[])
|
||||
|
||||
|
||||
class CleanupAwareServer(MCPServer):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.connect_calls = 0
|
||||
self.cleanup_calls = 0
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "cleanup-aware"
|
||||
|
||||
async def connect(self) -> None:
|
||||
if self.connect_calls > self.cleanup_calls:
|
||||
raise RuntimeError("connect called without cleanup")
|
||||
self.connect_calls += 1
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
self.cleanup_calls += 1
|
||||
|
||||
async def list_tools(
|
||||
self, run_context: RunContextWrapper[Any] | None = None, agent: Any | None = None
|
||||
) -> list[MCPTool]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def call_tool(
|
||||
self,
|
||||
tool_name: str,
|
||||
arguments: dict[str, Any] | None,
|
||||
meta: dict[str, Any] | None = None,
|
||||
) -> CallToolResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def list_prompts(self) -> ListPromptsResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_prompt(
|
||||
self, name: str, arguments: dict[str, Any] | None = None
|
||||
) -> GetPromptResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def list_resources(self, cursor: str | None = None) -> ListResourcesResult:
|
||||
return ListResourcesResult(resources=[])
|
||||
|
||||
async def list_resource_templates(
|
||||
self, cursor: str | None = None
|
||||
) -> ListResourceTemplatesResult:
|
||||
return ListResourceTemplatesResult(resourceTemplates=[])
|
||||
|
||||
async def read_resource(self, uri: str) -> ReadResourceResult:
|
||||
return ReadResourceResult(contents=[])
|
||||
|
||||
|
||||
class CancelledServer(MCPServer):
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "cancelled"
|
||||
|
||||
async def connect(self) -> None:
|
||||
raise asyncio.CancelledError()
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
return None
|
||||
|
||||
async def list_tools(
|
||||
self, run_context: RunContextWrapper[Any] | None = None, agent: Any | None = None
|
||||
) -> list[MCPTool]:
|
||||
raise NotImplementedError
|
||||
|
||||
async def call_tool(
|
||||
self,
|
||||
tool_name: str,
|
||||
arguments: dict[str, Any] | None,
|
||||
meta: dict[str, Any] | None = None,
|
||||
) -> CallToolResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def list_prompts(self) -> ListPromptsResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def get_prompt(
|
||||
self, name: str, arguments: dict[str, Any] | None = None
|
||||
) -> GetPromptResult:
|
||||
raise NotImplementedError
|
||||
|
||||
async def list_resources(self, cursor: str | None = None) -> ListResourcesResult:
|
||||
return ListResourcesResult(resources=[])
|
||||
|
||||
async def list_resource_templates(
|
||||
self, cursor: str | None = None
|
||||
) -> ListResourceTemplatesResult:
|
||||
return ListResourceTemplatesResult(resourceTemplates=[])
|
||||
|
||||
async def read_resource(self, uri: str) -> ReadResourceResult:
|
||||
return ReadResourceResult(contents=[])
|
||||
|
||||
|
||||
class FailingTaskBoundServer(TaskBoundServer):
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "failing-task-bound"
|
||||
|
||||
async def connect(self) -> None:
|
||||
await super().connect()
|
||||
raise RuntimeError("connect failed")
|
||||
|
||||
|
||||
class FatalError(BaseException):
|
||||
pass
|
||||
|
||||
|
||||
class FatalTaskBoundServer(TaskBoundServer):
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "fatal-task-bound"
|
||||
|
||||
async def connect(self) -> None:
|
||||
await super().connect()
|
||||
raise FatalError("fatal connect failed")
|
||||
|
||||
|
||||
class CleanupFailingServer(TaskBoundServer):
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "cleanup-failing"
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
await super().cleanup()
|
||||
raise RuntimeError("cleanup failed")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_keeps_connect_and_cleanup_in_same_task() -> None:
|
||||
server = TaskBoundServer()
|
||||
|
||||
async with MCPServerManager([server]) as manager:
|
||||
assert manager.active_servers == [server]
|
||||
|
||||
assert server.cleaned is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_connects_in_worker_tasks_when_parallel() -> None:
|
||||
server = TaskBoundServer()
|
||||
|
||||
async with MCPServerManager([server], connect_in_parallel=True) as manager:
|
||||
assert manager.active_servers == [server]
|
||||
assert server._connect_task is not None
|
||||
assert server._connect_task is not asyncio.current_task()
|
||||
|
||||
assert server.cleaned is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_task_cleanup_raises_without_manager() -> None:
|
||||
server = TaskBoundServer()
|
||||
|
||||
connect_task = asyncio.create_task(server.connect())
|
||||
await connect_task
|
||||
|
||||
with pytest.raises(RuntimeError, match="cancel scope"):
|
||||
await server.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_reconnect_failed_only() -> None:
|
||||
server = FlakyServer(failures=1)
|
||||
|
||||
async with MCPServerManager([server]) as manager:
|
||||
assert manager.active_servers == []
|
||||
assert manager.failed_servers == [server]
|
||||
|
||||
await manager.reconnect()
|
||||
assert manager.active_servers == [server]
|
||||
assert manager.failed_servers == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_reconnect_deduplicates_failures() -> None:
|
||||
server = FlakyServer(failures=2)
|
||||
|
||||
async with MCPServerManager([server], connect_in_parallel=True) as manager:
|
||||
assert manager.active_servers == []
|
||||
assert manager.failed_servers == [server]
|
||||
assert server.connect_calls == 1
|
||||
|
||||
await manager.reconnect()
|
||||
assert manager.active_servers == []
|
||||
assert manager.failed_servers == [server]
|
||||
assert server.connect_calls == 2
|
||||
|
||||
await manager.reconnect()
|
||||
assert manager.active_servers == [server]
|
||||
assert manager.failed_servers == []
|
||||
assert server.connect_calls == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_connect_all_retries_all_servers() -> None:
|
||||
server = FlakyServer(failures=1)
|
||||
manager = MCPServerManager([server])
|
||||
try:
|
||||
await manager.connect_all()
|
||||
assert manager.active_servers == []
|
||||
assert manager.failed_servers == [server]
|
||||
assert server.connect_calls == 1
|
||||
|
||||
await manager.connect_all()
|
||||
assert manager.active_servers == [server]
|
||||
assert manager.failed_servers == []
|
||||
assert server.connect_calls == 2
|
||||
finally:
|
||||
await manager.cleanup_all()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_connect_all_is_idempotent() -> None:
|
||||
server = CleanupAwareServer()
|
||||
|
||||
async with MCPServerManager([server]) as manager:
|
||||
assert server.connect_calls == 1
|
||||
await manager.connect_all()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_reconnect_all_avoids_duplicate_connections() -> None:
|
||||
server = CleanupAwareServer()
|
||||
|
||||
async with MCPServerManager([server]) as manager:
|
||||
assert server.connect_calls == 1
|
||||
await manager.reconnect(failed_only=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_strict_reconnect_refreshes_active_servers() -> None:
|
||||
server_a = FlakyServer(failures=1)
|
||||
server_b = FlakyServer(failures=2)
|
||||
|
||||
async with MCPServerManager([server_a, server_b]) as manager:
|
||||
assert manager.active_servers == []
|
||||
|
||||
manager.strict = True
|
||||
with pytest.raises(RuntimeError, match="connect failed"):
|
||||
await manager.reconnect()
|
||||
|
||||
assert manager.active_servers == [server_a]
|
||||
assert manager.failed_servers == [server_b]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_strict_connect_preserves_existing_active_servers() -> None:
|
||||
connected_server = TaskBoundServer()
|
||||
failing_server = FlakyServer(failures=2)
|
||||
manager = MCPServerManager([connected_server, failing_server])
|
||||
try:
|
||||
await manager.connect_all()
|
||||
assert manager.active_servers == [connected_server]
|
||||
assert manager.failed_servers == [failing_server]
|
||||
|
||||
manager.strict = True
|
||||
with pytest.raises(RuntimeError, match="connect failed"):
|
||||
await manager.connect_all()
|
||||
|
||||
assert manager.active_servers == [connected_server]
|
||||
assert manager.failed_servers == [failing_server]
|
||||
finally:
|
||||
await manager.cleanup_all()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_strict_connect_cleans_up_connected_servers() -> None:
|
||||
connected_server = TaskBoundServer()
|
||||
failing_server = FlakyServer(failures=1)
|
||||
manager = MCPServerManager([connected_server, failing_server], strict=True)
|
||||
|
||||
with pytest.raises(RuntimeError, match="connect failed"):
|
||||
await manager.connect_all()
|
||||
|
||||
assert connected_server.cleaned is True
|
||||
assert manager.active_servers == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_strict_connect_cleans_up_failed_server() -> None:
|
||||
failing_server = FailingTaskBoundServer()
|
||||
manager = MCPServerManager([failing_server], strict=True)
|
||||
|
||||
with pytest.raises(RuntimeError, match="connect failed"):
|
||||
await manager.connect_all()
|
||||
|
||||
assert failing_server.cleaned is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_strict_connect_parallel_cleans_up_failed_server() -> None:
|
||||
failing_server = FailingTaskBoundServer()
|
||||
manager = MCPServerManager([failing_server], strict=True, connect_in_parallel=True)
|
||||
|
||||
with pytest.raises(RuntimeError, match="connect failed"):
|
||||
await manager.connect_all()
|
||||
|
||||
assert failing_server.cleaned is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_strict_connect_parallel_cleans_up_workers() -> None:
|
||||
connected_server = TaskBoundServer()
|
||||
failing_server = FailingTaskBoundServer()
|
||||
manager = MCPServerManager(
|
||||
[connected_server, failing_server], strict=True, connect_in_parallel=True
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="connect failed"):
|
||||
await manager.connect_all()
|
||||
|
||||
assert connected_server.cleaned is True
|
||||
assert failing_server.cleaned is True
|
||||
assert manager._workers == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_parallel_cleanup_clears_worker_on_failure() -> None:
|
||||
server = CleanupFailingServer()
|
||||
manager = MCPServerManager([server], connect_in_parallel=True)
|
||||
await manager.connect_all()
|
||||
await manager.cleanup_all()
|
||||
|
||||
assert server not in manager._workers
|
||||
assert server not in manager._connected_servers
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_parallel_cleanup_drops_worker_after_error() -> None:
|
||||
class HangingCleanupWorker:
|
||||
def __init__(self) -> None:
|
||||
self.cleanup_calls = 0
|
||||
|
||||
@property
|
||||
def is_done(self) -> bool:
|
||||
return False
|
||||
|
||||
async def cleanup(self) -> None:
|
||||
self.cleanup_calls += 1
|
||||
raise RuntimeError("cleanup failed")
|
||||
|
||||
server = FlakyServer(failures=0)
|
||||
manager = MCPServerManager([server], connect_in_parallel=True)
|
||||
manager._workers[server] = cast(Any, HangingCleanupWorker())
|
||||
|
||||
await manager.cleanup_all()
|
||||
|
||||
assert manager._workers == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_parallel_suppresses_cancelled_error_in_strict_mode() -> None:
|
||||
server = CancelledServer()
|
||||
manager = MCPServerManager([server], connect_in_parallel=True, strict=True)
|
||||
try:
|
||||
await manager.connect_all()
|
||||
assert manager.active_servers == []
|
||||
assert manager.failed_servers == [server]
|
||||
finally:
|
||||
await manager.cleanup_all()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_parallel_propagates_cancelled_error_when_unsuppressed() -> None:
|
||||
server = CancelledServer()
|
||||
manager = MCPServerManager([server], connect_in_parallel=True, suppress_cancelled_error=False)
|
||||
try:
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await manager.connect_all()
|
||||
finally:
|
||||
await manager.cleanup_all()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_sequential_propagates_base_exception() -> None:
|
||||
server = FatalTaskBoundServer()
|
||||
manager = MCPServerManager([server])
|
||||
|
||||
with pytest.raises(FatalError, match="fatal connect failed"):
|
||||
await manager.connect_all()
|
||||
|
||||
assert server.cleaned is True
|
||||
assert manager.failed_servers == [server]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_parallel_propagates_base_exception() -> None:
|
||||
server = FatalTaskBoundServer()
|
||||
manager = MCPServerManager([server], connect_in_parallel=True)
|
||||
|
||||
with pytest.raises(FatalError, match="fatal connect failed"):
|
||||
await manager.connect_all()
|
||||
|
||||
assert server.cleaned is True
|
||||
assert manager._workers == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_parallel_prefers_cancelled_error_when_unsuppressed() -> None:
|
||||
cancelled_server = CancelledServer()
|
||||
fatal_server = FatalTaskBoundServer()
|
||||
manager = MCPServerManager(
|
||||
[fatal_server, cancelled_server],
|
||||
connect_in_parallel=True,
|
||||
suppress_cancelled_error=False,
|
||||
)
|
||||
try:
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await manager.connect_all()
|
||||
finally:
|
||||
await manager.cleanup_all()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_manager_cleanup_runs_on_cancelled_error_during_connect() -> None:
|
||||
server = CleanupAwareServer()
|
||||
cancelled_server = CancelledServer()
|
||||
manager = MCPServerManager(
|
||||
[server, cancelled_server],
|
||||
suppress_cancelled_error=False,
|
||||
)
|
||||
try:
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await manager.connect_all()
|
||||
assert server.cleanup_calls == 1
|
||||
finally:
|
||||
await manager.cleanup_all()
|
||||
@@ -0,0 +1,274 @@
|
||||
import pytest
|
||||
from inline_snapshot import snapshot
|
||||
|
||||
from agents import Agent, RunConfig, Runner
|
||||
|
||||
from ..fake_model import FakeModel
|
||||
from ..test_responses import get_function_tool, get_function_tool_call, get_text_message
|
||||
from ..testing_processor import SPAN_PROCESSOR_TESTING, fetch_normalized_spans
|
||||
from .helpers import FakeMCPServer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_tracing():
|
||||
model = FakeModel()
|
||||
server = FakeMCPServer()
|
||||
server.add_tool("test_tool_1", {})
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
mcp_servers=[server],
|
||||
tools=[get_function_tool("non_mcp_tool", "tool_result")],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[get_text_message("a_message"), get_function_tool_call("test_tool_1", "")],
|
||||
# Second turn: text message
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
# First run: should list MCP tools before first and second steps
|
||||
x = Runner.run_streamed(agent, input="first_test")
|
||||
async for _ in x.stream_events():
|
||||
pass
|
||||
|
||||
assert x.final_output == "done"
|
||||
spans = fetch_normalized_spans()
|
||||
|
||||
# Should have a single tool listing, and the function span should have MCP data
|
||||
assert spans == snapshot(
|
||||
[
|
||||
{
|
||||
"workflow_name": "Agent workflow",
|
||||
"children": [
|
||||
{
|
||||
"type": "mcp_tools",
|
||||
"data": {"server": "fake_mcp_server", "result": ["test_tool_1"]},
|
||||
},
|
||||
{
|
||||
"type": "agent",
|
||||
"data": {
|
||||
"name": "test",
|
||||
"handoffs": [],
|
||||
"tools": ["test_tool_1", "non_mcp_tool"],
|
||||
"output_type": "str",
|
||||
},
|
||||
"children": [
|
||||
{
|
||||
"type": "function",
|
||||
"data": {
|
||||
"name": "test_tool_1",
|
||||
"input": "",
|
||||
"output": "{'type': 'text', 'text': 'result_test_tool_1_{}'}", # noqa: E501
|
||||
"mcp_data": {"server": "fake_mcp_server"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "mcp_tools",
|
||||
"data": {"server": "fake_mcp_server", "result": ["test_tool_1"]},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
server.add_tool("test_tool_2", {})
|
||||
|
||||
SPAN_PROCESSOR_TESTING.clear()
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[
|
||||
get_text_message("a_message"),
|
||||
get_function_tool_call("non_mcp_tool", ""),
|
||||
get_function_tool_call("test_tool_2", ""),
|
||||
],
|
||||
# Second turn: text message
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
await Runner.run(agent, input="second_test")
|
||||
spans = fetch_normalized_spans()
|
||||
|
||||
# Should have a single tool listing, and the function span should have MCP data, and the non-mcp
|
||||
# tool function span should not have MCP data
|
||||
assert spans == snapshot(
|
||||
[
|
||||
{
|
||||
"workflow_name": "Agent workflow",
|
||||
"children": [
|
||||
{
|
||||
"type": "mcp_tools",
|
||||
"data": {
|
||||
"server": "fake_mcp_server",
|
||||
"result": ["test_tool_1", "test_tool_2"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "agent",
|
||||
"data": {
|
||||
"name": "test",
|
||||
"handoffs": [],
|
||||
"tools": ["test_tool_1", "test_tool_2", "non_mcp_tool"],
|
||||
"output_type": "str",
|
||||
},
|
||||
"children": [
|
||||
{
|
||||
"type": "function",
|
||||
"data": {
|
||||
"name": "non_mcp_tool",
|
||||
"input": "",
|
||||
"output": "tool_result",
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"data": {
|
||||
"name": "test_tool_2",
|
||||
"input": "",
|
||||
"output": "{'type': 'text', 'text': 'result_test_tool_2_{}'}", # noqa: E501
|
||||
"mcp_data": {"server": "fake_mcp_server"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "mcp_tools",
|
||||
"data": {
|
||||
"server": "fake_mcp_server",
|
||||
"result": ["test_tool_1", "test_tool_2"],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
SPAN_PROCESSOR_TESTING.clear()
|
||||
|
||||
# Add more tools to the server
|
||||
server.add_tool("test_tool_3", {})
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[get_text_message("a_message"), get_function_tool_call("test_tool_3", "")],
|
||||
# Second turn: text message
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
await Runner.run(agent, input="third_test")
|
||||
|
||||
spans = fetch_normalized_spans()
|
||||
|
||||
# Should have a single tool listing, and the function span should have MCP data, and the non-mcp
|
||||
# tool function span should not have MCP data
|
||||
assert spans == snapshot(
|
||||
[
|
||||
{
|
||||
"workflow_name": "Agent workflow",
|
||||
"children": [
|
||||
{
|
||||
"type": "mcp_tools",
|
||||
"data": {
|
||||
"server": "fake_mcp_server",
|
||||
"result": ["test_tool_1", "test_tool_2", "test_tool_3"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "agent",
|
||||
"data": {
|
||||
"name": "test",
|
||||
"handoffs": [],
|
||||
"tools": ["test_tool_1", "test_tool_2", "test_tool_3", "non_mcp_tool"],
|
||||
"output_type": "str",
|
||||
},
|
||||
"children": [
|
||||
{
|
||||
"type": "function",
|
||||
"data": {
|
||||
"name": "test_tool_3",
|
||||
"input": "",
|
||||
"output": "{'type': 'text', 'text': 'result_test_tool_3_{}'}", # noqa: E501
|
||||
"mcp_data": {"server": "fake_mcp_server"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "mcp_tools",
|
||||
"data": {
|
||||
"server": "fake_mcp_server",
|
||||
"result": ["test_tool_1", "test_tool_2", "test_tool_3"],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_tracing_redacts_output_when_sensitive_data_disabled():
|
||||
model = FakeModel()
|
||||
server = FakeMCPServer()
|
||||
server.add_tool("test_tool_1", {})
|
||||
agent = Agent(name="test", model=model, mcp_servers=[server])
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
[get_function_tool_call("test_tool_1", "")],
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
await Runner.run(
|
||||
agent,
|
||||
input="redaction_test",
|
||||
run_config=RunConfig(trace_include_sensitive_data=False),
|
||||
)
|
||||
|
||||
spans = fetch_normalized_spans()
|
||||
assert spans == snapshot(
|
||||
[
|
||||
{
|
||||
"workflow_name": "Agent workflow",
|
||||
"children": [
|
||||
{
|
||||
"type": "mcp_tools",
|
||||
"data": {"server": "fake_mcp_server", "result": ["test_tool_1"]},
|
||||
},
|
||||
{
|
||||
"type": "agent",
|
||||
"data": {
|
||||
"name": "test",
|
||||
"handoffs": [],
|
||||
"tools": ["test_tool_1"],
|
||||
"output_type": "str",
|
||||
},
|
||||
"children": [
|
||||
{
|
||||
"type": "function",
|
||||
"data": {
|
||||
"name": "test_tool_1",
|
||||
"mcp_data": {"server": "fake_mcp_server"},
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "mcp_tools",
|
||||
"data": {"server": "fake_mcp_server", "result": ["test_tool_1"]},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,133 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
from typing import Union
|
||||
|
||||
import anyio
|
||||
import pytest
|
||||
from mcp.client.session import MessageHandlerFnT
|
||||
from mcp.shared.message import SessionMessage
|
||||
from mcp.shared.session import RequestResponder
|
||||
from mcp.types import (
|
||||
ClientResult,
|
||||
Implementation,
|
||||
InitializeResult,
|
||||
ServerCapabilities,
|
||||
ServerNotification,
|
||||
ServerRequest,
|
||||
)
|
||||
|
||||
from agents.mcp.server import (
|
||||
MCPServerSse,
|
||||
MCPServerStdio,
|
||||
MCPServerStreamableHttp,
|
||||
_MCPServerWithClientSession,
|
||||
)
|
||||
|
||||
HandlerMessage = Union[ # noqa: UP007
|
||||
RequestResponder[ServerRequest, ClientResult], ServerNotification, Exception
|
||||
]
|
||||
|
||||
|
||||
class _StubClientSession:
|
||||
"""Stub ClientSession that records the configured message handler."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
read_stream,
|
||||
write_stream,
|
||||
read_timeout_seconds,
|
||||
*,
|
||||
message_handler=None,
|
||||
**_: object,
|
||||
) -> None:
|
||||
self.message_handler = message_handler
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
async def initialize(self) -> InitializeResult:
|
||||
capabilities = ServerCapabilities.model_construct()
|
||||
server_info = Implementation.model_construct(name="stub", version="1.0")
|
||||
return InitializeResult(
|
||||
protocolVersion="2024-11-05",
|
||||
capabilities=capabilities,
|
||||
serverInfo=server_info,
|
||||
)
|
||||
|
||||
|
||||
class _MessageHandlerTestServer(_MCPServerWithClientSession):
|
||||
def __init__(self, handler: MessageHandlerFnT | None):
|
||||
super().__init__(
|
||||
cache_tools_list=False,
|
||||
client_session_timeout_seconds=None,
|
||||
message_handler=handler,
|
||||
)
|
||||
|
||||
def create_streams(self):
|
||||
@contextlib.asynccontextmanager
|
||||
async def _streams():
|
||||
send_stream, recv_stream = anyio.create_memory_object_stream[
|
||||
SessionMessage | Exception
|
||||
](1)
|
||||
try:
|
||||
yield recv_stream, send_stream, None
|
||||
finally:
|
||||
await recv_stream.aclose()
|
||||
await send_stream.aclose()
|
||||
|
||||
return _streams()
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "test-server"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_session_receives_message_handler(monkeypatch):
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def _recording_client_session(*args, **kwargs):
|
||||
session = _StubClientSession(*args, **kwargs)
|
||||
captured["message_handler"] = session.message_handler
|
||||
return session
|
||||
|
||||
monkeypatch.setattr("agents.mcp.server.ClientSession", _recording_client_session)
|
||||
|
||||
class _AsyncHandler:
|
||||
async def __call__(self, message: HandlerMessage) -> None:
|
||||
del message
|
||||
|
||||
handler: MessageHandlerFnT = _AsyncHandler()
|
||||
|
||||
server = _MessageHandlerTestServer(handler)
|
||||
|
||||
try:
|
||||
await server.connect()
|
||||
finally:
|
||||
await server.cleanup()
|
||||
|
||||
assert captured["message_handler"] is handler
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"server_cls, params",
|
||||
[
|
||||
(MCPServerSse, {"url": "https://example.com"}),
|
||||
(MCPServerStreamableHttp, {"url": "https://example.com"}),
|
||||
(MCPServerStdio, {"command": "python"}),
|
||||
],
|
||||
)
|
||||
def test_message_handler_propagates_to_server_base(server_cls, params):
|
||||
class _AsyncHandler:
|
||||
async def __call__(self, message: HandlerMessage) -> None:
|
||||
del message
|
||||
|
||||
handler: MessageHandlerFnT = _AsyncHandler()
|
||||
|
||||
server = server_cls(params, message_handler=handler)
|
||||
|
||||
assert server.message_handler is handler
|
||||
@@ -0,0 +1,324 @@
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from mcp.types import ListResourcesResult, ListResourceTemplatesResult, ReadResourceResult
|
||||
|
||||
from agents import Agent, Runner
|
||||
from agents.mcp import MCPServer, MCPToolMetaResolver
|
||||
|
||||
from ..fake_model import FakeModel
|
||||
from ..test_responses import get_text_message
|
||||
|
||||
|
||||
class FakeMCPPromptServer(MCPServer):
|
||||
"""Fake MCP server for testing prompt functionality"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
server_name: str = "fake_prompt_server",
|
||||
tool_meta_resolver: MCPToolMetaResolver | None = None,
|
||||
):
|
||||
super().__init__(tool_meta_resolver=tool_meta_resolver)
|
||||
self.prompts: list[Any] = []
|
||||
self.prompt_results: dict[str, str] = {}
|
||||
self._server_name = server_name
|
||||
|
||||
def add_prompt(self, name: str, description: str, arguments: dict[str, Any] | None = None):
|
||||
"""Add a prompt to the fake server"""
|
||||
from mcp.types import Prompt
|
||||
|
||||
prompt = Prompt(name=name, description=description, arguments=[])
|
||||
self.prompts.append(prompt)
|
||||
|
||||
def set_prompt_result(self, name: str, result: str):
|
||||
"""Set the result that should be returned for a prompt"""
|
||||
self.prompt_results[name] = result
|
||||
|
||||
async def connect(self):
|
||||
pass
|
||||
|
||||
async def cleanup(self):
|
||||
pass
|
||||
|
||||
async def list_prompts(self, run_context=None, agent=None):
|
||||
"""List available prompts"""
|
||||
from mcp.types import ListPromptsResult
|
||||
|
||||
return ListPromptsResult(prompts=self.prompts)
|
||||
|
||||
async def get_prompt(self, name: str, arguments: dict[str, Any] | None = None):
|
||||
"""Get a prompt with arguments"""
|
||||
from mcp.types import GetPromptResult, PromptMessage, TextContent
|
||||
|
||||
if name not in self.prompt_results:
|
||||
raise ValueError(f"Prompt '{name}' not found")
|
||||
|
||||
content = self.prompt_results[name]
|
||||
|
||||
# If it's a format string, try to format it with arguments
|
||||
if arguments and "{" in content:
|
||||
try:
|
||||
content = content.format(**arguments)
|
||||
except KeyError:
|
||||
pass # Use original content if formatting fails
|
||||
|
||||
message = PromptMessage(role="user", content=TextContent(type="text", text=content))
|
||||
|
||||
return GetPromptResult(description=f"Generated prompt for {name}", messages=[message])
|
||||
|
||||
async def list_tools(self, run_context=None, agent=None):
|
||||
return []
|
||||
|
||||
async def call_tool(
|
||||
self,
|
||||
tool_name: str,
|
||||
arguments: dict[str, Any] | None = None,
|
||||
meta: dict[str, Any] | None = None,
|
||||
):
|
||||
raise NotImplementedError("This fake server doesn't support tools")
|
||||
|
||||
async def list_resources(self, cursor: str | None = None) -> ListResourcesResult:
|
||||
return ListResourcesResult(resources=[])
|
||||
|
||||
async def list_resource_templates(
|
||||
self, cursor: str | None = None
|
||||
) -> ListResourceTemplatesResult:
|
||||
return ListResourceTemplatesResult(resourceTemplates=[])
|
||||
|
||||
async def read_resource(self, uri: str) -> ReadResourceResult:
|
||||
return ReadResourceResult(contents=[])
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return self._server_name
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_prompts():
|
||||
"""Test listing available prompts"""
|
||||
server = FakeMCPPromptServer()
|
||||
server.add_prompt(
|
||||
"generate_code_review_instructions", "Generate agent instructions for code review tasks"
|
||||
)
|
||||
|
||||
result = await server.list_prompts()
|
||||
|
||||
assert len(result.prompts) == 1
|
||||
assert result.prompts[0].name == "generate_code_review_instructions"
|
||||
assert result.prompts[0].description is not None
|
||||
assert "code review" in result.prompts[0].description
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_prompt_without_arguments():
|
||||
"""Test getting a prompt without arguments"""
|
||||
server = FakeMCPPromptServer()
|
||||
server.add_prompt("simple_prompt", "A simple prompt")
|
||||
server.set_prompt_result("simple_prompt", "You are a helpful assistant.")
|
||||
|
||||
result = await server.get_prompt("simple_prompt")
|
||||
|
||||
assert len(result.messages) == 1
|
||||
assert result.messages[0].content.text == "You are a helpful assistant."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_prompt_with_arguments():
|
||||
"""Test getting a prompt with arguments"""
|
||||
server = FakeMCPPromptServer()
|
||||
server.add_prompt(
|
||||
"generate_code_review_instructions", "Generate agent instructions for code review tasks"
|
||||
)
|
||||
server.set_prompt_result(
|
||||
"generate_code_review_instructions",
|
||||
"You are a senior {language} code review specialist. Focus on {focus}.",
|
||||
)
|
||||
|
||||
result = await server.get_prompt(
|
||||
"generate_code_review_instructions",
|
||||
{"focus": "security vulnerabilities", "language": "python"},
|
||||
)
|
||||
|
||||
assert len(result.messages) == 1
|
||||
expected_text = (
|
||||
"You are a senior python code review specialist. Focus on security vulnerabilities."
|
||||
)
|
||||
assert result.messages[0].content.text == expected_text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_prompt_not_found():
|
||||
"""Test getting a prompt that doesn't exist"""
|
||||
server = FakeMCPPromptServer()
|
||||
|
||||
with pytest.raises(ValueError, match="Prompt 'nonexistent' not found"):
|
||||
await server.get_prompt("nonexistent")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_with_prompt_instructions():
|
||||
"""Test using prompt-generated instructions with an agent"""
|
||||
server = FakeMCPPromptServer()
|
||||
server.add_prompt(
|
||||
"generate_code_review_instructions", "Generate agent instructions for code review tasks"
|
||||
)
|
||||
server.set_prompt_result(
|
||||
"generate_code_review_instructions",
|
||||
"You are a code reviewer. Analyze the provided code for security issues.",
|
||||
)
|
||||
|
||||
# Get instructions from prompt
|
||||
prompt_result = await server.get_prompt("generate_code_review_instructions")
|
||||
instructions = prompt_result.messages[0].content.text
|
||||
|
||||
# Create agent with prompt-generated instructions
|
||||
model = FakeModel()
|
||||
agent = Agent(name="prompt_agent", instructions=instructions, model=model, mcp_servers=[server])
|
||||
|
||||
# Mock model response
|
||||
model.add_multiple_turn_outputs(
|
||||
[[get_text_message("Code analysis complete. Found security vulnerability.")]]
|
||||
)
|
||||
|
||||
# Run the agent
|
||||
result = await Runner.run(agent, input="Review this code: def unsafe_exec(cmd): os.system(cmd)")
|
||||
|
||||
assert "Code analysis complete" in result.final_output
|
||||
assert (
|
||||
agent.instructions
|
||||
== "You are a code reviewer. Analyze the provided code for security issues."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
async def test_agent_with_prompt_instructions_streaming(streaming: bool):
|
||||
"""Test using prompt-generated instructions with streaming and non-streaming"""
|
||||
server = FakeMCPPromptServer()
|
||||
server.add_prompt(
|
||||
"generate_code_review_instructions", "Generate agent instructions for code review tasks"
|
||||
)
|
||||
server.set_prompt_result(
|
||||
"generate_code_review_instructions",
|
||||
"You are a {language} code reviewer focusing on {focus}.",
|
||||
)
|
||||
|
||||
# Get instructions from prompt with arguments
|
||||
prompt_result = await server.get_prompt(
|
||||
"generate_code_review_instructions", {"language": "Python", "focus": "security"}
|
||||
)
|
||||
instructions = prompt_result.messages[0].content.text
|
||||
|
||||
# Create agent
|
||||
model = FakeModel()
|
||||
agent = Agent(
|
||||
name="streaming_prompt_agent", instructions=instructions, model=model, mcp_servers=[server]
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs([[get_text_message("Security analysis complete.")]])
|
||||
|
||||
if streaming:
|
||||
streaming_result = Runner.run_streamed(agent, input="Review code")
|
||||
async for _ in streaming_result.stream_events():
|
||||
pass
|
||||
final_result = streaming_result.final_output
|
||||
else:
|
||||
result = await Runner.run(agent, input="Review code")
|
||||
final_result = result.final_output
|
||||
|
||||
assert "Security analysis complete" in final_result
|
||||
assert agent.instructions == "You are a Python code reviewer focusing on security."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_prompts():
|
||||
"""Test server with multiple prompts"""
|
||||
server = FakeMCPPromptServer()
|
||||
|
||||
# Add multiple prompts
|
||||
server.add_prompt(
|
||||
"generate_code_review_instructions", "Generate agent instructions for code review tasks"
|
||||
)
|
||||
server.add_prompt(
|
||||
"generate_testing_instructions", "Generate agent instructions for testing tasks"
|
||||
)
|
||||
|
||||
server.set_prompt_result("generate_code_review_instructions", "You are a code reviewer.")
|
||||
server.set_prompt_result("generate_testing_instructions", "You are a test engineer.")
|
||||
|
||||
# Test listing prompts
|
||||
prompts_result = await server.list_prompts()
|
||||
assert len(prompts_result.prompts) == 2
|
||||
|
||||
prompt_names = [p.name for p in prompts_result.prompts]
|
||||
assert "generate_code_review_instructions" in prompt_names
|
||||
assert "generate_testing_instructions" in prompt_names
|
||||
|
||||
# Test getting each prompt
|
||||
review_result = await server.get_prompt("generate_code_review_instructions")
|
||||
assert review_result.messages[0].content.text == "You are a code reviewer."
|
||||
|
||||
testing_result = await server.get_prompt("generate_testing_instructions")
|
||||
assert testing_result.messages[0].content.text == "You are a test engineer."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_with_complex_arguments():
|
||||
"""Test prompt with complex argument formatting"""
|
||||
server = FakeMCPPromptServer()
|
||||
server.add_prompt(
|
||||
"generate_detailed_instructions", "Generate detailed instructions with multiple parameters"
|
||||
)
|
||||
server.set_prompt_result(
|
||||
"generate_detailed_instructions",
|
||||
"You are a {role} specialist. Your focus is on {focus}. "
|
||||
+ "You work with {language} code. Your experience level is {level}.",
|
||||
)
|
||||
|
||||
arguments = {
|
||||
"role": "security",
|
||||
"focus": "vulnerability detection",
|
||||
"language": "Python",
|
||||
"level": "senior",
|
||||
}
|
||||
|
||||
result = await server.get_prompt("generate_detailed_instructions", arguments)
|
||||
|
||||
expected = (
|
||||
"You are a security specialist. Your focus is on vulnerability detection. "
|
||||
"You work with Python code. Your experience level is senior."
|
||||
)
|
||||
assert result.messages[0].content.text == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_with_missing_arguments():
|
||||
"""Test prompt with missing arguments in format string"""
|
||||
server = FakeMCPPromptServer()
|
||||
server.add_prompt("incomplete_prompt", "Prompt with missing arguments")
|
||||
server.set_prompt_result("incomplete_prompt", "You are a {role} working on {task}.")
|
||||
|
||||
# Only provide one of the required arguments
|
||||
result = await server.get_prompt("incomplete_prompt", {"role": "developer"})
|
||||
|
||||
# Should return the original string since formatting fails
|
||||
assert result.messages[0].content.text == "You are a {role} working on {task}."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_prompt_server_cleanup():
|
||||
"""Test that prompt server cleanup works correctly"""
|
||||
server = FakeMCPPromptServer()
|
||||
server.add_prompt("test_prompt", "Test prompt")
|
||||
server.set_prompt_result("test_prompt", "Test result")
|
||||
|
||||
# Test that server works before cleanup
|
||||
result = await server.get_prompt("test_prompt")
|
||||
assert result.messages[0].content.text == "Test result"
|
||||
|
||||
# Cleanup should not raise any errors
|
||||
await server.cleanup()
|
||||
|
||||
# Server should still work after cleanup (in this fake implementation)
|
||||
result = await server.get_prompt("test_prompt")
|
||||
assert result.messages[0].content.text == "Test result"
|
||||
@@ -0,0 +1,402 @@
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
||||
from agents import (
|
||||
Agent,
|
||||
FunctionTool,
|
||||
ModelBehaviorError,
|
||||
RunContextWrapper,
|
||||
Runner,
|
||||
UserError,
|
||||
default_tool_error_function,
|
||||
handoff,
|
||||
)
|
||||
from agents.exceptions import AgentsException
|
||||
|
||||
from ..fake_model import FakeModel
|
||||
from ..test_responses import get_function_tool_call, get_text_message
|
||||
from .helpers import FakeMCPServer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
async def test_runner_calls_mcp_tool(streaming: bool):
|
||||
"""Test that the runner calls an MCP tool when the model produces a tool call."""
|
||||
server = FakeMCPServer()
|
||||
server.add_tool("test_tool_1", {})
|
||||
server.add_tool("test_tool_2", {})
|
||||
server.add_tool("test_tool_3", {})
|
||||
model = FakeModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
mcp_servers=[server],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[get_text_message("a_message"), get_function_tool_call("test_tool_2", "")],
|
||||
# Second turn: text message
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
if streaming:
|
||||
result = Runner.run_streamed(agent, input="user_message")
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
else:
|
||||
await Runner.run(agent, input="user_message")
|
||||
|
||||
assert server.tool_calls == ["test_tool_2"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
async def test_runner_asserts_when_mcp_tool_not_found(streaming: bool):
|
||||
"""Test that the runner asserts when an MCP tool is not found."""
|
||||
server = FakeMCPServer()
|
||||
server.add_tool("test_tool_1", {})
|
||||
server.add_tool("test_tool_2", {})
|
||||
server.add_tool("test_tool_3", {})
|
||||
model = FakeModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
mcp_servers=[server],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[get_text_message("a_message"), get_function_tool_call("test_tool_doesnt_exist", "")],
|
||||
# Second turn: text message
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(ModelBehaviorError):
|
||||
if streaming:
|
||||
result = Runner.run_streamed(agent, input="user_message")
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
else:
|
||||
await Runner.run(agent, input="user_message")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
async def test_runner_works_with_multiple_mcp_servers(streaming: bool):
|
||||
"""Test that the runner works with multiple MCP servers."""
|
||||
server1 = FakeMCPServer()
|
||||
server1.add_tool("test_tool_1", {})
|
||||
|
||||
server2 = FakeMCPServer()
|
||||
server2.add_tool("test_tool_2", {})
|
||||
server2.add_tool("test_tool_3", {})
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
mcp_servers=[server1, server2],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[get_text_message("a_message"), get_function_tool_call("test_tool_2", "")],
|
||||
# Second turn: text message
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
if streaming:
|
||||
result = Runner.run_streamed(agent, input="user_message")
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
else:
|
||||
await Runner.run(agent, input="user_message")
|
||||
|
||||
assert server1.tool_calls == []
|
||||
assert server2.tool_calls == ["test_tool_2"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
async def test_runner_errors_when_mcp_tools_clash(streaming: bool):
|
||||
"""Test that the runner errors when multiple servers have the same tool name."""
|
||||
server1 = FakeMCPServer()
|
||||
server1.add_tool("test_tool_1", {})
|
||||
server1.add_tool("test_tool_2", {})
|
||||
|
||||
server2 = FakeMCPServer()
|
||||
server2.add_tool("test_tool_2", {})
|
||||
server2.add_tool("test_tool_3", {})
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
mcp_servers=[server1, server2],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[get_text_message("a_message"), get_function_tool_call("test_tool_3", "")],
|
||||
# Second turn: text message
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
with pytest.raises(UserError):
|
||||
if streaming:
|
||||
result = Runner.run_streamed(agent, input="user_message")
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
else:
|
||||
await Runner.run(agent, input="user_message")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
async def test_runner_can_call_server_prefixed_mcp_tool_names(streaming: bool):
|
||||
server1 = FakeMCPServer(server_name="docs")
|
||||
server1.add_tool("search", {})
|
||||
|
||||
server2 = FakeMCPServer(server_name="calendar")
|
||||
server2.add_tool("search", {})
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
mcp_servers=[server1, server2],
|
||||
mcp_config={"include_server_in_tool_names": True},
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
[get_text_message("a_message"), get_function_tool_call("mcp_calendar__search", "")],
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
if streaming:
|
||||
result = Runner.run_streamed(agent, input="user_message")
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
else:
|
||||
await Runner.run(agent, input="user_message")
|
||||
|
||||
assert server1.tool_calls == []
|
||||
assert server2.tool_calls == ["search"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_agent_tools(streaming: bool):
|
||||
server1 = FakeMCPServer(server_name="docs")
|
||||
server1.add_tool("search", {})
|
||||
|
||||
server2 = FakeMCPServer(server_name="calendar")
|
||||
server2.add_tool("search", {})
|
||||
|
||||
local_tool_calls: list[str] = []
|
||||
|
||||
async def invoke_local_tool(context: Any, input_json: str) -> str:
|
||||
local_tool_calls.append(input_json)
|
||||
return "local"
|
||||
|
||||
local_tool = FunctionTool(
|
||||
name="mcp_calendar__search",
|
||||
description="Local tool that intentionally collides with the natural MCP prefix.",
|
||||
params_json_schema={"type": "object", "properties": {}, "additionalProperties": False},
|
||||
on_invoke_tool=invoke_local_tool,
|
||||
)
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
tools=[local_tool],
|
||||
mcp_servers=[server1, server2],
|
||||
mcp_config={"include_server_in_tool_names": True},
|
||||
)
|
||||
|
||||
mcp_tools = await agent.get_mcp_tools(RunContextWrapper(context=None))
|
||||
calendar_search_tool_name = next(
|
||||
tool.name
|
||||
for tool in mcp_tools
|
||||
if getattr(getattr(tool, "_tool_origin", None), "mcp_server_name", None) == "calendar"
|
||||
)
|
||||
assert calendar_search_tool_name != "mcp_calendar__search"
|
||||
assert calendar_search_tool_name.startswith("mcp_calendar__search_")
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
[get_text_message("a_message"), get_function_tool_call(calendar_search_tool_name, "")],
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
if streaming:
|
||||
result = Runner.run_streamed(agent, input="user_message")
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
else:
|
||||
await Runner.run(agent, input="user_message")
|
||||
|
||||
assert local_tool_calls == []
|
||||
assert server1.tool_calls == []
|
||||
assert server2.tool_calls == ["search"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_handoffs(streaming: bool):
|
||||
server = FakeMCPServer(server_name="calendar")
|
||||
server.add_tool("search", {})
|
||||
|
||||
target_model = FakeModel()
|
||||
target_agent = Agent(name="calendar_agent", model=target_model)
|
||||
target_model.add_multiple_turn_outputs([[get_text_message("handoff target")]])
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
handoffs=[handoff(target_agent, tool_name_override="mcp_calendar__search")],
|
||||
mcp_servers=[server],
|
||||
mcp_config={"include_server_in_tool_names": True},
|
||||
)
|
||||
|
||||
mcp_tools = await agent.get_mcp_tools(RunContextWrapper(context=None))
|
||||
assert len(mcp_tools) == 1
|
||||
calendar_search_tool_name = mcp_tools[0].name
|
||||
assert calendar_search_tool_name != "mcp_calendar__search"
|
||||
assert calendar_search_tool_name.startswith("mcp_calendar__search_")
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
[get_text_message("a_message"), get_function_tool_call(calendar_search_tool_name, "")],
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
if streaming:
|
||||
result = Runner.run_streamed(agent, input="user_message")
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
else:
|
||||
await Runner.run(agent, input="user_message")
|
||||
|
||||
assert server.tool_calls == ["search"]
|
||||
assert target_model.first_turn_args is None
|
||||
|
||||
|
||||
class Foo(BaseModel):
|
||||
bar: str
|
||||
baz: int
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
async def test_runner_calls_mcp_tool_with_args(streaming: bool):
|
||||
"""Test that the runner calls an MCP tool when the model produces a tool call."""
|
||||
server = FakeMCPServer()
|
||||
await server.connect()
|
||||
server.add_tool("test_tool_1", {})
|
||||
server.add_tool("test_tool_2", Foo.model_json_schema())
|
||||
server.add_tool("test_tool_3", {})
|
||||
model = FakeModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
mcp_servers=[server],
|
||||
)
|
||||
|
||||
json_args = json.dumps(Foo(bar="baz", baz=1).model_dump())
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
# First turn: a message and tool call
|
||||
[get_text_message("a_message"), get_function_tool_call("test_tool_2", json_args)],
|
||||
# Second turn: text message
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
if streaming:
|
||||
result = Runner.run_streamed(agent, input="user_message")
|
||||
async for _ in result.stream_events():
|
||||
pass
|
||||
else:
|
||||
await Runner.run(agent, input="user_message")
|
||||
|
||||
assert server.tool_calls == ["test_tool_2"]
|
||||
assert server.tool_results == [f"result_test_tool_2_{json_args}"]
|
||||
|
||||
await server.cleanup()
|
||||
|
||||
|
||||
class CrashingFakeMCPServer(FakeMCPServer):
|
||||
async def call_tool(
|
||||
self,
|
||||
tool_name: str,
|
||||
arguments: dict[str, object] | None,
|
||||
meta: dict[str, object] | None = None,
|
||||
):
|
||||
raise Exception("Crash!")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("streaming", [False, True])
|
||||
async def test_runner_emits_mcp_error_tool_call_output_item(streaming: bool):
|
||||
"""Runner should emit tool_call_output_item with failure output when MCP tool raises."""
|
||||
server = CrashingFakeMCPServer()
|
||||
server.add_tool("crashing_tool", {})
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
mcp_servers=[server],
|
||||
)
|
||||
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
[get_text_message("a_message"), get_function_tool_call("crashing_tool", "{}")],
|
||||
[get_text_message("done")],
|
||||
]
|
||||
)
|
||||
|
||||
if streaming:
|
||||
streamed_result = Runner.run_streamed(agent, input="user_message")
|
||||
async for _ in streamed_result.stream_events():
|
||||
pass
|
||||
tool_output_items = [
|
||||
item for item in streamed_result.new_items if item.type == "tool_call_output_item"
|
||||
]
|
||||
assert streamed_result.final_output == "done"
|
||||
else:
|
||||
non_streamed_result = await Runner.run(agent, input="user_message")
|
||||
tool_output_items = [
|
||||
item for item in non_streamed_result.new_items if item.type == "tool_call_output_item"
|
||||
]
|
||||
assert non_streamed_result.final_output == "done"
|
||||
|
||||
assert tool_output_items, "Expected tool_call_output_item for MCP failure"
|
||||
wrapped_error = AgentsException(
|
||||
"Error invoking MCP tool crashing_tool on server 'fake_mcp_server': Crash!"
|
||||
)
|
||||
expected_error_message = default_tool_error_function(
|
||||
RunContextWrapper(context=None),
|
||||
wrapped_error,
|
||||
)
|
||||
assert tool_output_items[0].output == expected_error_message
|
||||
@@ -0,0 +1,85 @@
|
||||
import builtins
|
||||
import sys
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from agents import Agent
|
||||
from agents.exceptions import UserError
|
||||
from agents.mcp.server import MCPServerStreamableHttp, _MCPServerWithClientSession
|
||||
from agents.run_context import RunContextWrapper
|
||||
|
||||
# Handle Python version compatibility for ExceptionGroups
|
||||
if sys.version_info < (3, 11):
|
||||
from exceptiongroup import BaseExceptionGroup
|
||||
else:
|
||||
BaseExceptionGroup = builtins.BaseExceptionGroup
|
||||
|
||||
|
||||
class CrashingClientSessionServer(_MCPServerWithClientSession):
|
||||
def __init__(self):
|
||||
super().__init__(cache_tools_list=False, client_session_timeout_seconds=5)
|
||||
self.cleanup_called = False
|
||||
|
||||
def create_streams(self):
|
||||
raise ValueError("Crash!")
|
||||
|
||||
async def cleanup(self):
|
||||
self.cleanup_called = True
|
||||
await super().cleanup()
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
return "crashing_client_session_server"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_server_errors_cause_error_and_cleanup_called():
|
||||
server = CrashingClientSessionServer()
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
await server.connect()
|
||||
|
||||
assert server.cleanup_called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_not_calling_connect_causes_error():
|
||||
server = CrashingClientSessionServer()
|
||||
|
||||
run_context = RunContextWrapper(context=None)
|
||||
agent = Agent(name="test_agent", instructions="Test agent")
|
||||
|
||||
with pytest.raises(UserError):
|
||||
await server.list_tools(run_context, agent)
|
||||
|
||||
with pytest.raises(UserError):
|
||||
await server.call_tool("foo", {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_call_tool_nested_exception_group_mapping():
|
||||
"""
|
||||
Regression test ensuring that nested ExceptionGroups containing HTTP errors
|
||||
are recursively extracted and mapped to a UserError in call_tool().
|
||||
"""
|
||||
# 1. Initialize the server with mock streamable parameters
|
||||
server = MCPServerStreamableHttp(params={"url": "http://fake-mcp-server"})
|
||||
|
||||
# 2. Simulate an active connection by mocking the session object
|
||||
server.session = MagicMock()
|
||||
|
||||
# 3. Construct a nested ExceptionGroup hierarchy containing a connection error
|
||||
http_error = httpx.ConnectError("Network unreachable")
|
||||
inner_group = BaseExceptionGroup("inner_failures", [http_error])
|
||||
outer_group = BaseExceptionGroup("outer_failures", [inner_group])
|
||||
|
||||
# 4 & 5. Mock the internal retry handler to raise the nested group, and assert UserError
|
||||
with patch.object(server, "_call_tool_with_isolated_retry", side_effect=outer_group):
|
||||
with pytest.raises(UserError) as exc_info:
|
||||
await server.call_tool(tool_name="test_tool", arguments={})
|
||||
|
||||
# 6. Verify that the user-facing message is mapped correctly based on the root cause
|
||||
assert "Connection lost" in str(exc_info.value)
|
||||
assert exc_info.value.__cause__ is http_error
|
||||
@@ -0,0 +1,442 @@
|
||||
"""Tests for MCPServerStreamableHttp httpx_client_factory functionality."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from anyio import create_memory_object_stream
|
||||
from mcp.shared.message import SessionMessage
|
||||
from mcp.types import JSONRPCMessage, JSONRPCNotification, JSONRPCRequest
|
||||
|
||||
from agents.mcp import MCPServerStreamableHttp
|
||||
from agents.mcp.server import (
|
||||
_create_default_streamable_http_client,
|
||||
_InitializedNotificationTolerantStreamableHTTPTransport,
|
||||
_streamablehttp_client_with_transport,
|
||||
)
|
||||
|
||||
|
||||
class TestMCPServerStreamableHttpClientFactory:
|
||||
"""Test cases for custom httpx_client_factory parameter."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_httpx_client_factory(self):
|
||||
"""Test that default behavior works when no custom factory is provided."""
|
||||
# Mock the streamablehttp_client to avoid actual network calls
|
||||
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
|
||||
mock_client.return_value = MagicMock()
|
||||
|
||||
server = MCPServerStreamableHttp(
|
||||
params={
|
||||
"url": "http://localhost:8000/mcp",
|
||||
"headers": {"Authorization": "Bearer token"},
|
||||
"timeout": 10,
|
||||
}
|
||||
)
|
||||
|
||||
server.create_streams()
|
||||
|
||||
# Verify streamablehttp_client was called with the hardened default factory.
|
||||
mock_client.assert_called_once_with(
|
||||
url="http://localhost:8000/mcp",
|
||||
headers={"Authorization": "Bearer token"},
|
||||
timeout=10,
|
||||
sse_read_timeout=300, # Default value
|
||||
terminate_on_close=True, # Default value
|
||||
httpx_client_factory=_create_default_streamable_http_client,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_custom_httpx_client_factory(self):
|
||||
"""Test that custom httpx_client_factory is passed correctly."""
|
||||
|
||||
# Create a custom factory function
|
||||
def custom_factory(
|
||||
headers: dict[str, str] | None = None,
|
||||
timeout: httpx.Timeout | None = None,
|
||||
auth: httpx.Auth | None = None,
|
||||
) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(
|
||||
verify=False, # Disable SSL verification for testing
|
||||
timeout=httpx.Timeout(60.0),
|
||||
headers={"X-Custom-Header": "test"},
|
||||
)
|
||||
|
||||
# Mock the streamablehttp_client to avoid actual network calls
|
||||
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
|
||||
mock_client.return_value = MagicMock()
|
||||
|
||||
server = MCPServerStreamableHttp(
|
||||
params={
|
||||
"url": "http://localhost:8000/mcp",
|
||||
"headers": {"Authorization": "Bearer token"},
|
||||
"timeout": 10,
|
||||
"httpx_client_factory": custom_factory,
|
||||
}
|
||||
)
|
||||
|
||||
# Create streams should pass the custom factory
|
||||
server.create_streams()
|
||||
|
||||
# Verify streamablehttp_client was called with the custom factory
|
||||
mock_client.assert_called_once_with(
|
||||
url="http://localhost:8000/mcp",
|
||||
headers={"Authorization": "Bearer token"},
|
||||
timeout=10,
|
||||
sse_read_timeout=300, # Default value
|
||||
terminate_on_close=True, # Default value
|
||||
httpx_client_factory=custom_factory,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_custom_httpx_client_factory_with_ssl_cert(self):
|
||||
"""Test custom factory with SSL certificate configuration."""
|
||||
|
||||
def ssl_cert_factory(
|
||||
headers: dict[str, str] | None = None,
|
||||
timeout: httpx.Timeout | None = None,
|
||||
auth: httpx.Auth | None = None,
|
||||
) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(
|
||||
verify="/path/to/cert.pem", # Custom SSL certificate
|
||||
timeout=httpx.Timeout(120.0),
|
||||
)
|
||||
|
||||
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
|
||||
mock_client.return_value = MagicMock()
|
||||
|
||||
server = MCPServerStreamableHttp(
|
||||
params={
|
||||
"url": "https://secure-server.com/mcp",
|
||||
"timeout": 30,
|
||||
"httpx_client_factory": ssl_cert_factory,
|
||||
}
|
||||
)
|
||||
|
||||
server.create_streams()
|
||||
|
||||
mock_client.assert_called_once_with(
|
||||
url="https://secure-server.com/mcp",
|
||||
headers=None,
|
||||
timeout=30,
|
||||
sse_read_timeout=300,
|
||||
terminate_on_close=True,
|
||||
httpx_client_factory=ssl_cert_factory,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_custom_httpx_client_factory_with_proxy(self):
|
||||
"""Test custom factory with proxy configuration."""
|
||||
|
||||
def proxy_factory(
|
||||
headers: dict[str, str] | None = None,
|
||||
timeout: httpx.Timeout | None = None,
|
||||
auth: httpx.Auth | None = None,
|
||||
) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(
|
||||
proxy="http://proxy.example.com:8080",
|
||||
timeout=httpx.Timeout(60.0),
|
||||
)
|
||||
|
||||
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
|
||||
mock_client.return_value = MagicMock()
|
||||
|
||||
server = MCPServerStreamableHttp(
|
||||
params={
|
||||
"url": "http://localhost:8000/mcp",
|
||||
"httpx_client_factory": proxy_factory,
|
||||
}
|
||||
)
|
||||
|
||||
server.create_streams()
|
||||
|
||||
mock_client.assert_called_once_with(
|
||||
url="http://localhost:8000/mcp",
|
||||
headers=None,
|
||||
timeout=5, # Default value
|
||||
sse_read_timeout=300,
|
||||
terminate_on_close=True,
|
||||
httpx_client_factory=proxy_factory,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_custom_httpx_client_factory_with_retry_logic(self):
|
||||
"""Test custom factory with retry logic configuration."""
|
||||
|
||||
def retry_factory(
|
||||
headers: dict[str, str] | None = None,
|
||||
timeout: httpx.Timeout | None = None,
|
||||
auth: httpx.Auth | None = None,
|
||||
) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(
|
||||
timeout=httpx.Timeout(30.0),
|
||||
# Note: httpx doesn't have built-in retry, but this shows how
|
||||
# a custom factory could be used to configure retry behavior
|
||||
# through middleware or other mechanisms
|
||||
)
|
||||
|
||||
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
|
||||
mock_client.return_value = MagicMock()
|
||||
|
||||
server = MCPServerStreamableHttp(
|
||||
params={
|
||||
"url": "http://localhost:8000/mcp",
|
||||
"httpx_client_factory": retry_factory,
|
||||
}
|
||||
)
|
||||
|
||||
server.create_streams()
|
||||
|
||||
mock_client.assert_called_once_with(
|
||||
url="http://localhost:8000/mcp",
|
||||
headers=None,
|
||||
timeout=5,
|
||||
sse_read_timeout=300,
|
||||
terminate_on_close=True,
|
||||
httpx_client_factory=retry_factory,
|
||||
)
|
||||
|
||||
def test_httpx_client_factory_type_annotation(self):
|
||||
"""Test that the type annotation is correct for httpx_client_factory."""
|
||||
from agents.mcp.server import MCPServerStreamableHttpParams
|
||||
|
||||
# This test ensures the type annotation is properly set
|
||||
# We can't easily test the TypedDict at runtime, but we can verify
|
||||
# that the import works and the type is available
|
||||
assert hasattr(MCPServerStreamableHttpParams, "__annotations__")
|
||||
|
||||
# Verify that the httpx_client_factory parameter is in the annotations
|
||||
annotations = MCPServerStreamableHttpParams.__annotations__
|
||||
assert "httpx_client_factory" in annotations
|
||||
|
||||
# The annotation should contain the string representation of the type
|
||||
annotation_str = str(annotations["httpx_client_factory"])
|
||||
assert "HttpClientFactory" in annotation_str
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_all_parameters_with_custom_factory(self):
|
||||
"""Test that all parameters work together with custom factory."""
|
||||
|
||||
def comprehensive_factory(
|
||||
headers: dict[str, str] | None = None,
|
||||
timeout: httpx.Timeout | None = None,
|
||||
auth: httpx.Auth | None = None,
|
||||
) -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(
|
||||
verify=False,
|
||||
timeout=httpx.Timeout(90.0),
|
||||
headers={"X-Test": "value"},
|
||||
)
|
||||
|
||||
with patch("agents.mcp.server.streamablehttp_client") as mock_client:
|
||||
mock_client.return_value = MagicMock()
|
||||
|
||||
server = MCPServerStreamableHttp(
|
||||
params={
|
||||
"url": "https://api.example.com/mcp",
|
||||
"headers": {"Authorization": "Bearer token"},
|
||||
"timeout": 45,
|
||||
"sse_read_timeout": 600,
|
||||
"terminate_on_close": False,
|
||||
"httpx_client_factory": comprehensive_factory,
|
||||
}
|
||||
)
|
||||
|
||||
server.create_streams()
|
||||
|
||||
mock_client.assert_called_once_with(
|
||||
url="https://api.example.com/mcp",
|
||||
headers={"Authorization": "Bearer token"},
|
||||
timeout=45,
|
||||
sse_read_timeout=600,
|
||||
terminate_on_close=False,
|
||||
httpx_client_factory=comprehensive_factory,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialized_notification_failure_returns_synthetic_success():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(503, request=request)
|
||||
|
||||
transport = _InitializedNotificationTolerantStreamableHTTPTransport("https://example.test/mcp")
|
||||
read_stream_writer, _ = create_memory_object_stream[SessionMessage | Exception](0)
|
||||
client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
||||
try:
|
||||
ctx = MagicMock()
|
||||
ctx.client = client
|
||||
ctx.read_stream_writer = read_stream_writer
|
||||
ctx.session_message = SessionMessage(
|
||||
JSONRPCMessage(
|
||||
JSONRPCNotification(
|
||||
jsonrpc="2.0",
|
||||
method="notifications/initialized",
|
||||
params={},
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
await transport._handle_post_request(ctx)
|
||||
finally:
|
||||
await client.aclose()
|
||||
await read_stream_writer.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_initialized_notification_transport_exception_returns_synthetic_success():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ConnectError("boom", request=request)
|
||||
|
||||
transport = _InitializedNotificationTolerantStreamableHTTPTransport("https://example.test/mcp")
|
||||
read_stream_writer, _ = create_memory_object_stream[SessionMessage | Exception](0)
|
||||
client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
||||
try:
|
||||
ctx = MagicMock()
|
||||
ctx.client = client
|
||||
ctx.read_stream_writer = read_stream_writer
|
||||
ctx.session_message = SessionMessage(
|
||||
JSONRPCMessage(
|
||||
JSONRPCNotification(
|
||||
jsonrpc="2.0",
|
||||
method="notifications/initialized",
|
||||
params={},
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
await transport._handle_post_request(ctx)
|
||||
finally:
|
||||
await client.aclose()
|
||||
await read_stream_writer.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamable_http_server_passes_ignore_initialized_notification_failure():
|
||||
with patch("agents.mcp.server._streamablehttp_client_with_transport") as mock_client:
|
||||
mock_client.return_value = MagicMock()
|
||||
|
||||
server = MCPServerStreamableHttp(
|
||||
params={
|
||||
"url": "http://localhost:8000/mcp",
|
||||
"ignore_initialized_notification_failure": True,
|
||||
}
|
||||
)
|
||||
|
||||
server.create_streams()
|
||||
|
||||
kwargs = mock_client.call_args.kwargs
|
||||
assert kwargs["url"] == "http://localhost:8000/mcp"
|
||||
assert kwargs["headers"] is None
|
||||
assert kwargs["timeout"] == 5
|
||||
assert kwargs["sse_read_timeout"] == 300
|
||||
assert kwargs["terminate_on_close"] is True
|
||||
assert kwargs["httpx_client_factory"] is _create_default_streamable_http_client
|
||||
assert (
|
||||
kwargs["transport_factory"] is _InitializedNotificationTolerantStreamableHTTPTransport
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transport_preserves_non_initialized_failures():
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ConnectError("boom", request=request)
|
||||
|
||||
transport = _InitializedNotificationTolerantStreamableHTTPTransport("https://example.test/mcp")
|
||||
read_stream_writer, _ = create_memory_object_stream[SessionMessage | Exception](0)
|
||||
client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
|
||||
try:
|
||||
ctx = MagicMock()
|
||||
ctx.client = client
|
||||
ctx.read_stream_writer = read_stream_writer
|
||||
ctx.session_message = SessionMessage(
|
||||
JSONRPCMessage(
|
||||
JSONRPCRequest(
|
||||
jsonrpc="2.0",
|
||||
id=1,
|
||||
method="tools/list",
|
||||
params={},
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(httpx.ConnectError):
|
||||
await transport._handle_post_request(ctx)
|
||||
finally:
|
||||
await client.aclose()
|
||||
await read_stream_writer.aclose()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_client_preserves_custom_factory_headers_timeout_and_auth():
|
||||
seen: dict[str, object] = {}
|
||||
|
||||
class RecordingAuth(httpx.Auth):
|
||||
def auth_flow(self, request: httpx.Request):
|
||||
request.headers["Authorization"] = f"Basic {base64.b64encode(b'user:pass').decode()}"
|
||||
yield request
|
||||
|
||||
async def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen["request_headers"] = dict(request.headers)
|
||||
return httpx.Response(200, request=request)
|
||||
|
||||
def base_factory(
|
||||
headers: dict[str, str] | None = None,
|
||||
timeout: httpx.Timeout | None = None,
|
||||
auth: httpx.Auth | None = None,
|
||||
) -> httpx.AsyncClient:
|
||||
seen["factory_headers"] = headers
|
||||
seen["factory_timeout"] = timeout
|
||||
seen["factory_auth"] = auth
|
||||
return httpx.AsyncClient(
|
||||
headers=headers,
|
||||
timeout=timeout,
|
||||
auth=auth,
|
||||
transport=httpx.MockTransport(handler),
|
||||
)
|
||||
|
||||
timeout = httpx.Timeout(12.0)
|
||||
auth = RecordingAuth()
|
||||
async with _streamablehttp_client_with_transport(
|
||||
"https://example.test/mcp",
|
||||
headers={"X-Test": "value"},
|
||||
timeout=12.0,
|
||||
sse_read_timeout=30.0,
|
||||
httpx_client_factory=base_factory,
|
||||
auth=auth,
|
||||
transport_factory=_InitializedNotificationTolerantStreamableHTTPTransport,
|
||||
):
|
||||
pass
|
||||
|
||||
assert seen["factory_headers"] == {"X-Test": "value"}
|
||||
seen_timeout = seen["factory_timeout"]
|
||||
assert isinstance(seen_timeout, httpx.Timeout)
|
||||
assert seen_timeout.connect == timeout.connect
|
||||
assert seen_timeout.read == 30.0
|
||||
assert seen_timeout.write == timeout.write
|
||||
assert seen_timeout.pool == timeout.pool
|
||||
assert seen["factory_auth"] is auth
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_streamable_http_client_matches_expected_defaults():
|
||||
timeout = httpx.Timeout(12.0)
|
||||
auth = httpx.BasicAuth("user", "pass")
|
||||
|
||||
client = _create_default_streamable_http_client(
|
||||
headers={"X-Test": "value"},
|
||||
timeout=timeout,
|
||||
auth=auth,
|
||||
)
|
||||
try:
|
||||
assert client.headers["X-Test"] == "value"
|
||||
assert client.timeout.connect == timeout.connect
|
||||
assert client.timeout.read == timeout.read
|
||||
assert client.timeout.write == timeout.write
|
||||
assert client.timeout.pool == timeout.pool
|
||||
assert client.auth is auth
|
||||
assert client.follow_redirects is False
|
||||
finally:
|
||||
await client.aclose()
|
||||
@@ -0,0 +1,115 @@
|
||||
"""Tests for MCPServerStreamableHttp.session_id property (issue #924)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agents.mcp import MCPServerStreamableHttp
|
||||
|
||||
|
||||
class TestStreamableHttpSessionId:
|
||||
"""Tests that the session_id property is correctly exposed."""
|
||||
|
||||
def test_session_id_is_none_before_connect(self):
|
||||
"""session_id should be None when the server has not been connected yet."""
|
||||
server = MCPServerStreamableHttp(params={"url": "http://localhost:9999/mcp"})
|
||||
assert server.session_id is None
|
||||
|
||||
def test_session_id_returns_none_when_callback_is_none(self):
|
||||
"""session_id should be None when _get_session_id callback is None."""
|
||||
server = MCPServerStreamableHttp(params={"url": "http://localhost:9999/mcp"})
|
||||
server._get_session_id = None
|
||||
assert server.session_id is None
|
||||
|
||||
def test_session_id_returns_callback_value(self):
|
||||
"""session_id should return the value from the get_session_id callback."""
|
||||
server = MCPServerStreamableHttp(params={"url": "http://localhost:9999/mcp"})
|
||||
mock_get_session_id = MagicMock(return_value="test-session-abc123")
|
||||
server._get_session_id = mock_get_session_id
|
||||
assert server.session_id == "test-session-abc123"
|
||||
mock_get_session_id.assert_called_once()
|
||||
|
||||
def test_session_id_returns_none_when_callback_returns_none(self):
|
||||
"""session_id should return None when the callback itself returns None."""
|
||||
server = MCPServerStreamableHttp(params={"url": "http://localhost:9999/mcp"})
|
||||
mock_get_session_id = MagicMock(return_value=None)
|
||||
server._get_session_id = mock_get_session_id
|
||||
assert server.session_id is None
|
||||
|
||||
def test_session_id_reflects_updated_callback_value(self):
|
||||
"""session_id should reflect the latest value from the callback each time."""
|
||||
server = MCPServerStreamableHttp(params={"url": "http://localhost:9999/mcp"})
|
||||
call_count = 0
|
||||
|
||||
def changing_callback() -> str | None:
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
return f"session-{call_count}"
|
||||
|
||||
server._get_session_id = changing_callback
|
||||
assert server.session_id == "session-1"
|
||||
assert server.session_id == "session-2"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_connect_captures_get_session_id_callback(self):
|
||||
"""connect() should capture the third element of the transport tuple as _get_session_id."""
|
||||
server = MCPServerStreamableHttp(params={"url": "http://localhost:9999/mcp"})
|
||||
|
||||
mock_read = AsyncMock()
|
||||
mock_write = AsyncMock()
|
||||
mock_get_session_id = MagicMock(return_value="captured-session-xyz")
|
||||
|
||||
mock_initialize_result = MagicMock()
|
||||
mock_session = AsyncMock()
|
||||
mock_session.initialize = AsyncMock(return_value=mock_initialize_result)
|
||||
|
||||
# Simulate the full 3-tuple that streamablehttp_client returns
|
||||
transport_tuple = (mock_read, mock_write, mock_get_session_id)
|
||||
|
||||
with patch("agents.mcp.server.ClientSession") as mock_client_session_cls:
|
||||
mock_client_session_cls.return_value.__aenter__ = AsyncMock(return_value=mock_session)
|
||||
mock_client_session_cls.return_value.__aexit__ = AsyncMock(return_value=None)
|
||||
|
||||
with patch.object(
|
||||
server,
|
||||
"create_streams",
|
||||
) as mock_create_streams:
|
||||
mock_cm = MagicMock()
|
||||
mock_cm.__aenter__ = AsyncMock(return_value=transport_tuple)
|
||||
mock_cm.__aexit__ = AsyncMock(return_value=None)
|
||||
mock_create_streams.return_value = mock_cm
|
||||
|
||||
with patch.object(server.exit_stack, "enter_async_context") as mock_enter:
|
||||
# First call returns transport, second call returns session
|
||||
mock_enter.side_effect = [transport_tuple, mock_session]
|
||||
mock_session.initialize.return_value = mock_initialize_result
|
||||
|
||||
await server.connect()
|
||||
|
||||
# After connect, _get_session_id should be the callable from the transport
|
||||
assert server._get_session_id is mock_get_session_id
|
||||
assert server.session_id == "captured-session-xyz"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_id_is_none_after_cleanup():
|
||||
"""session_id must return None after disconnect (cleanup clears _get_session_id)."""
|
||||
server = MCPServerStreamableHttp(params={"url": "http://localhost:8000/mcp"})
|
||||
|
||||
mock_get_session_id = MagicMock(return_value="session-to-clear")
|
||||
# Manually inject a session-id callback to simulate a connected state
|
||||
server._get_session_id = mock_get_session_id
|
||||
server.session = MagicMock() # pretend connected
|
||||
|
||||
assert server.session_id == "session-to-clear"
|
||||
|
||||
# Now simulate cleanup completing (exit_stack.aclose is a no-op here)
|
||||
with patch.object(server.exit_stack, "aclose", new_callable=AsyncMock):
|
||||
await server.cleanup()
|
||||
|
||||
# After cleanup both session and _get_session_id must be None
|
||||
assert server.session is None
|
||||
assert server._get_session_id is None
|
||||
assert server.session_id is None
|
||||
@@ -0,0 +1,246 @@
|
||||
"""
|
||||
Tool filtering tests use FakeMCPServer instead of real MCPServer implementations to avoid
|
||||
external dependencies (processes, network connections) and ensure fast, reliable unit tests.
|
||||
FakeMCPServer delegates filtering logic to the real _MCPServerWithClientSession implementation.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
from mcp import Tool as MCPTool
|
||||
|
||||
from agents import Agent
|
||||
from agents.mcp import ToolFilterContext, create_static_tool_filter
|
||||
from agents.run_context import RunContextWrapper
|
||||
|
||||
from .helpers import FakeMCPServer
|
||||
|
||||
|
||||
def create_test_agent(name: str = "test_agent") -> Agent:
|
||||
"""Create a test agent for filtering tests."""
|
||||
return Agent(name=name, instructions="Test agent")
|
||||
|
||||
|
||||
def create_test_context() -> RunContextWrapper:
|
||||
"""Create a test run context for filtering tests."""
|
||||
return RunContextWrapper(context=None)
|
||||
|
||||
|
||||
# === Static Tool Filtering Tests ===
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_static_tool_filtering():
|
||||
"""Test all static tool filtering scenarios: allowed, blocked, both, none, etc."""
|
||||
server = FakeMCPServer(server_name="test_server")
|
||||
server.add_tool("tool1", {})
|
||||
server.add_tool("tool2", {})
|
||||
server.add_tool("tool3", {})
|
||||
server.add_tool("tool4", {})
|
||||
|
||||
# Create test context and agent for all calls
|
||||
run_context = create_test_context()
|
||||
agent = create_test_agent()
|
||||
|
||||
# Test allowed_tool_names only
|
||||
server.tool_filter = {"allowed_tool_names": ["tool1", "tool2"]}
|
||||
tools = await server.list_tools(run_context, agent)
|
||||
assert len(tools) == 2
|
||||
assert {t.name for t in tools} == {"tool1", "tool2"}
|
||||
|
||||
# Test blocked_tool_names only
|
||||
server.tool_filter = {"blocked_tool_names": ["tool3", "tool4"]}
|
||||
tools = await server.list_tools(run_context, agent)
|
||||
assert len(tools) == 2
|
||||
assert {t.name for t in tools} == {"tool1", "tool2"}
|
||||
|
||||
# Test both filters together (allowed first, then blocked)
|
||||
server.tool_filter = {
|
||||
"allowed_tool_names": ["tool1", "tool2", "tool3"],
|
||||
"blocked_tool_names": ["tool3"],
|
||||
}
|
||||
tools = await server.list_tools(run_context, agent)
|
||||
assert len(tools) == 2
|
||||
assert {t.name for t in tools} == {"tool1", "tool2"}
|
||||
|
||||
# Test no filter
|
||||
server.tool_filter = None
|
||||
tools = await server.list_tools(run_context, agent)
|
||||
assert len(tools) == 4
|
||||
|
||||
# Test helper function
|
||||
server.tool_filter = create_static_tool_filter(
|
||||
allowed_tool_names=["tool1", "tool2"], blocked_tool_names=["tool2"]
|
||||
)
|
||||
tools = await server.list_tools(run_context, agent)
|
||||
assert len(tools) == 1
|
||||
assert tools[0].name == "tool1"
|
||||
|
||||
|
||||
# === Dynamic Tool Filtering Core Tests ===
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dynamic_filter_sync_and_async():
|
||||
"""Test both synchronous and asynchronous dynamic filters"""
|
||||
server = FakeMCPServer(server_name="test_server")
|
||||
server.add_tool("allowed_tool", {})
|
||||
server.add_tool("blocked_tool", {})
|
||||
server.add_tool("restricted_tool", {})
|
||||
|
||||
# Create test context and agent
|
||||
run_context = create_test_context()
|
||||
agent = create_test_agent()
|
||||
|
||||
# Test sync filter
|
||||
def sync_filter(context: ToolFilterContext, tool: MCPTool) -> bool:
|
||||
return tool.name.startswith("allowed")
|
||||
|
||||
server.tool_filter = sync_filter
|
||||
tools = await server.list_tools(run_context, agent)
|
||||
assert len(tools) == 1
|
||||
assert tools[0].name == "allowed_tool"
|
||||
|
||||
# Test async filter
|
||||
async def async_filter(context: ToolFilterContext, tool: MCPTool) -> bool:
|
||||
await asyncio.sleep(0.001) # Simulate async operation
|
||||
return "restricted" not in tool.name
|
||||
|
||||
server.tool_filter = async_filter
|
||||
tools = await server.list_tools(run_context, agent)
|
||||
assert len(tools) == 2
|
||||
assert {t.name for t in tools} == {"allowed_tool", "blocked_tool"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dynamic_filter_context_handling():
|
||||
"""Test dynamic filters with context access"""
|
||||
server = FakeMCPServer(server_name="test_server")
|
||||
server.add_tool("admin_tool", {})
|
||||
server.add_tool("user_tool", {})
|
||||
server.add_tool("guest_tool", {})
|
||||
|
||||
# Test context-independent filter
|
||||
def context_independent_filter(context: ToolFilterContext, tool: MCPTool) -> bool:
|
||||
return not tool.name.startswith("admin")
|
||||
|
||||
server.tool_filter = context_independent_filter
|
||||
run_context = create_test_context()
|
||||
agent = create_test_agent()
|
||||
tools = await server.list_tools(run_context, agent)
|
||||
assert len(tools) == 2
|
||||
assert {t.name for t in tools} == {"user_tool", "guest_tool"}
|
||||
|
||||
# Test context-dependent filter (needs context)
|
||||
def context_dependent_filter(context: ToolFilterContext, tool: MCPTool) -> bool:
|
||||
assert context is not None
|
||||
assert context.run_context is not None
|
||||
assert context.agent is not None
|
||||
assert context.server_name == "test_server"
|
||||
|
||||
# Only admin tools for agents with "admin" in name
|
||||
if "admin" in context.agent.name.lower():
|
||||
return True
|
||||
else:
|
||||
return not tool.name.startswith("admin")
|
||||
|
||||
server.tool_filter = context_dependent_filter
|
||||
|
||||
# Should work with context
|
||||
run_context = RunContextWrapper(context=None)
|
||||
regular_agent = create_test_agent("regular_user")
|
||||
tools = await server.list_tools(run_context, regular_agent)
|
||||
assert len(tools) == 2
|
||||
assert {t.name for t in tools} == {"user_tool", "guest_tool"}
|
||||
|
||||
admin_agent = create_test_agent("admin_user")
|
||||
tools = await server.list_tools(run_context, admin_agent)
|
||||
assert len(tools) == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dynamic_filter_error_handling():
|
||||
"""Test error handling in dynamic filters"""
|
||||
server = FakeMCPServer(server_name="test_server")
|
||||
server.add_tool("good_tool", {})
|
||||
server.add_tool("error_tool", {})
|
||||
server.add_tool("another_good_tool", {})
|
||||
|
||||
def error_prone_filter(context: ToolFilterContext, tool: MCPTool) -> bool:
|
||||
if tool.name == "error_tool":
|
||||
raise ValueError("Simulated filter error")
|
||||
return True
|
||||
|
||||
server.tool_filter = error_prone_filter
|
||||
|
||||
# Test with server call
|
||||
run_context = create_test_context()
|
||||
agent = create_test_agent()
|
||||
tools = await server.list_tools(run_context, agent)
|
||||
assert len(tools) == 2
|
||||
assert {t.name for t in tools} == {"good_tool", "another_good_tool"}
|
||||
|
||||
|
||||
# === Integration Tests ===
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_dynamic_filtering_integration():
|
||||
"""Test dynamic filtering integration with Agent methods"""
|
||||
server = FakeMCPServer()
|
||||
server.add_tool("file_read", {"type": "object", "properties": {"path": {"type": "string"}}})
|
||||
server.add_tool(
|
||||
"file_write",
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {"path": {"type": "string"}, "content": {"type": "string"}},
|
||||
},
|
||||
)
|
||||
server.add_tool(
|
||||
"database_query", {"type": "object", "properties": {"query": {"type": "string"}}}
|
||||
)
|
||||
server.add_tool(
|
||||
"network_request", {"type": "object", "properties": {"url": {"type": "string"}}}
|
||||
)
|
||||
|
||||
# Role-based filter for comprehensive testing
|
||||
async def role_based_filter(context: ToolFilterContext, tool: MCPTool) -> bool:
|
||||
# Simulate async permission check
|
||||
await asyncio.sleep(0.001)
|
||||
|
||||
agent_name = context.agent.name.lower()
|
||||
if "admin" in agent_name:
|
||||
return True
|
||||
elif "readonly" in agent_name:
|
||||
return "read" in tool.name or "query" in tool.name
|
||||
else:
|
||||
return tool.name.startswith("file_")
|
||||
|
||||
server.tool_filter = role_based_filter
|
||||
|
||||
# Test admin agent
|
||||
admin_agent = Agent(name="admin_user", instructions="Admin", mcp_servers=[server])
|
||||
run_context = RunContextWrapper(context=None)
|
||||
admin_tools = await admin_agent.get_mcp_tools(run_context)
|
||||
assert len(admin_tools) == 4
|
||||
|
||||
# Test readonly agent
|
||||
readonly_agent = Agent(name="readonly_viewer", instructions="Read-only", mcp_servers=[server])
|
||||
readonly_tools = await readonly_agent.get_mcp_tools(run_context)
|
||||
assert len(readonly_tools) == 2
|
||||
assert {t.name for t in readonly_tools} == {"file_read", "database_query"}
|
||||
|
||||
# Test regular agent
|
||||
regular_agent = Agent(name="regular_user", instructions="Regular", mcp_servers=[server])
|
||||
regular_tools = await regular_agent.get_mcp_tools(run_context)
|
||||
assert len(regular_tools) == 2
|
||||
assert {t.name for t in regular_tools} == {"file_read", "file_write"}
|
||||
|
||||
# Test get_all_tools method
|
||||
all_tools = await regular_agent.get_all_tools(run_context)
|
||||
mcp_tool_names = {
|
||||
t.name
|
||||
for t in all_tools
|
||||
if t.name in {"file_read", "file_write", "database_query", "network_request"}
|
||||
}
|
||||
assert mcp_tool_names == {"file_read", "file_write"}
|
||||
@@ -0,0 +1,475 @@
|
||||
"""Tests for OpenAI Conversations Session functionality."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agents import Agent, Runner, TResponseInputItem
|
||||
from agents.memory.openai_conversations_session import (
|
||||
OpenAIConversationsSession,
|
||||
start_openai_conversations_session,
|
||||
)
|
||||
from tests.fake_model import FakeModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_openai_client():
|
||||
"""Create a mock OpenAI client for testing."""
|
||||
client = AsyncMock()
|
||||
|
||||
# Mock conversations.create
|
||||
client.conversations.create.return_value = MagicMock(id="test_conversation_id")
|
||||
|
||||
# Mock conversations.delete
|
||||
client.conversations.delete.return_value = None
|
||||
|
||||
# Mock conversations.items.create
|
||||
client.conversations.items.create.return_value = None
|
||||
|
||||
# Mock conversations.items.delete
|
||||
client.conversations.items.delete.return_value = None
|
||||
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def agent() -> Agent:
|
||||
"""Fixture for a basic agent with a fake model."""
|
||||
return Agent(name="test", model=FakeModel())
|
||||
|
||||
|
||||
class TestStartOpenAIConversationsSession:
|
||||
"""Test the standalone start_openai_conversations_session function."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_with_provided_client(self, mock_openai_client):
|
||||
"""Test starting a conversation session with a provided client."""
|
||||
conversation_id = await start_openai_conversations_session(mock_openai_client)
|
||||
|
||||
assert conversation_id == "test_conversation_id"
|
||||
mock_openai_client.conversations.create.assert_called_once_with(items=[])
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_with_none_client(self):
|
||||
"""Test starting a conversation session with None client (uses default)."""
|
||||
with patch(
|
||||
"agents.memory.openai_conversations_session.get_default_openai_client"
|
||||
) as mock_get_default:
|
||||
with patch("agents.memory.openai_conversations_session.AsyncOpenAI"):
|
||||
# Test case 1: get_default_openai_client returns a client
|
||||
mock_default_client = AsyncMock()
|
||||
mock_default_client.conversations.create.return_value = MagicMock(
|
||||
id="default_client_id"
|
||||
)
|
||||
mock_get_default.return_value = mock_default_client
|
||||
|
||||
conversation_id = await start_openai_conversations_session(None)
|
||||
|
||||
assert conversation_id == "default_client_id"
|
||||
mock_get_default.assert_called_once()
|
||||
mock_default_client.conversations.create.assert_called_once_with(items=[])
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_start_with_none_client_fallback(self):
|
||||
"""Test starting a conversation session when get_default_openai_client returns None."""
|
||||
with patch(
|
||||
"agents.memory.openai_conversations_session.get_default_openai_client"
|
||||
) as mock_get_default:
|
||||
with patch(
|
||||
"agents.memory.openai_conversations_session.AsyncOpenAI"
|
||||
) as mock_async_openai:
|
||||
# Test case 2: get_default_openai_client returns None, fallback to AsyncOpenAI()
|
||||
mock_get_default.return_value = None
|
||||
mock_fallback_client = AsyncMock()
|
||||
mock_fallback_client.conversations.create.return_value = MagicMock(
|
||||
id="fallback_client_id"
|
||||
)
|
||||
mock_async_openai.return_value = mock_fallback_client
|
||||
|
||||
conversation_id = await start_openai_conversations_session(None)
|
||||
|
||||
assert conversation_id == "fallback_client_id"
|
||||
mock_get_default.assert_called_once()
|
||||
mock_async_openai.assert_called_once()
|
||||
mock_fallback_client.conversations.create.assert_called_once_with(items=[])
|
||||
|
||||
|
||||
class TestOpenAIConversationsSessionConstructor:
|
||||
"""Test OpenAIConversationsSession constructor and client handling."""
|
||||
|
||||
def test_init_with_conversation_id_and_client(self, mock_openai_client):
|
||||
"""Test constructor with both conversation_id and openai_client provided."""
|
||||
session = OpenAIConversationsSession(
|
||||
conversation_id="test_id", openai_client=mock_openai_client
|
||||
)
|
||||
|
||||
assert session._session_id == "test_id"
|
||||
assert session._openai_client is mock_openai_client
|
||||
|
||||
def test_init_with_conversation_id_only(self):
|
||||
"""Test constructor with only conversation_id, client should be created."""
|
||||
with patch(
|
||||
"agents.memory.openai_conversations_session.get_default_openai_client"
|
||||
) as mock_get_default:
|
||||
with patch("agents.memory.openai_conversations_session.AsyncOpenAI"):
|
||||
mock_default_client = AsyncMock()
|
||||
mock_get_default.return_value = mock_default_client
|
||||
|
||||
session = OpenAIConversationsSession(conversation_id="test_id")
|
||||
|
||||
assert session._session_id == "test_id"
|
||||
assert session._openai_client is mock_default_client
|
||||
mock_get_default.assert_called_once()
|
||||
|
||||
def test_init_with_client_only(self, mock_openai_client):
|
||||
"""Test constructor with only openai_client, no conversation_id."""
|
||||
session = OpenAIConversationsSession(openai_client=mock_openai_client)
|
||||
|
||||
assert session._session_id is None
|
||||
assert session._openai_client is mock_openai_client
|
||||
|
||||
def test_init_with_no_args_fallback(self):
|
||||
"""Test constructor with no args, should create default client."""
|
||||
with patch(
|
||||
"agents.memory.openai_conversations_session.get_default_openai_client"
|
||||
) as mock_get_default:
|
||||
with patch(
|
||||
"agents.memory.openai_conversations_session.AsyncOpenAI"
|
||||
) as mock_async_openai:
|
||||
# Test fallback when get_default_openai_client returns None
|
||||
mock_get_default.return_value = None
|
||||
mock_fallback_client = AsyncMock()
|
||||
mock_async_openai.return_value = mock_fallback_client
|
||||
|
||||
session = OpenAIConversationsSession()
|
||||
|
||||
assert session._session_id is None
|
||||
assert session._openai_client is mock_fallback_client
|
||||
mock_get_default.assert_called_once()
|
||||
mock_async_openai.assert_called_once()
|
||||
|
||||
|
||||
class TestOpenAIConversationsSessionLifecycle:
|
||||
"""Test session ID lifecycle management."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_session_id_with_existing_id(self, mock_openai_client):
|
||||
"""Test _get_session_id when session_id already exists."""
|
||||
session = OpenAIConversationsSession(
|
||||
conversation_id="existing_id", openai_client=mock_openai_client
|
||||
)
|
||||
|
||||
session_id = await session._get_session_id()
|
||||
|
||||
assert session_id == "existing_id"
|
||||
# Should not call conversations.create since ID already exists
|
||||
mock_openai_client.conversations.create.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_session_id_creates_new_conversation(self, mock_openai_client):
|
||||
"""Test _get_session_id when session_id is None, should create new conversation."""
|
||||
session = OpenAIConversationsSession(openai_client=mock_openai_client)
|
||||
|
||||
session_id = await session._get_session_id()
|
||||
|
||||
assert session_id == "test_conversation_id"
|
||||
assert session._session_id == "test_conversation_id"
|
||||
mock_openai_client.conversations.create.assert_called_once_with(items=[])
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_session_id(self, mock_openai_client):
|
||||
"""Test _clear_session_id sets session_id to None."""
|
||||
session = OpenAIConversationsSession(
|
||||
conversation_id="test_id", openai_client=mock_openai_client
|
||||
)
|
||||
|
||||
await session._clear_session_id()
|
||||
|
||||
assert session._session_id is None
|
||||
|
||||
|
||||
class TestOpenAIConversationsSessionBasicOperations:
|
||||
"""Test basic CRUD operations with simple mocking."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_items_simple(self, mock_openai_client):
|
||||
"""Test adding items to the conversation."""
|
||||
session = OpenAIConversationsSession(
|
||||
conversation_id="test_id", openai_client=mock_openai_client
|
||||
)
|
||||
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there!"},
|
||||
]
|
||||
|
||||
await session.add_items(items)
|
||||
|
||||
mock_openai_client.conversations.items.create.assert_called_once_with(
|
||||
conversation_id="test_id", items=items
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_items_creates_session_id(self, mock_openai_client):
|
||||
"""Test that add_items creates session_id if it doesn't exist."""
|
||||
session = OpenAIConversationsSession(openai_client=mock_openai_client)
|
||||
|
||||
items: list[TResponseInputItem] = [{"role": "user", "content": "Hello"}]
|
||||
|
||||
await session.add_items(items)
|
||||
|
||||
# Should create conversation first
|
||||
mock_openai_client.conversations.create.assert_called_once_with(items=[])
|
||||
# Then add items
|
||||
mock_openai_client.conversations.items.create.assert_called_once_with(
|
||||
conversation_id="test_conversation_id", items=items
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pop_item_with_items(self, mock_openai_client):
|
||||
"""Test popping item when items exist using method patching."""
|
||||
session = OpenAIConversationsSession(
|
||||
conversation_id="test_id", openai_client=mock_openai_client
|
||||
)
|
||||
|
||||
# Mock get_items to return one item
|
||||
latest_item = {"id": "item_123", "role": "assistant", "content": "Latest message"}
|
||||
|
||||
with patch.object(session, "get_items", return_value=[latest_item]):
|
||||
popped_item = await session.pop_item()
|
||||
|
||||
assert popped_item == latest_item
|
||||
mock_openai_client.conversations.items.delete.assert_called_once_with(
|
||||
conversation_id="test_id", item_id="item_123"
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pop_item_empty_session(self, mock_openai_client):
|
||||
"""Test popping item from empty session."""
|
||||
session = OpenAIConversationsSession(
|
||||
conversation_id="test_id", openai_client=mock_openai_client
|
||||
)
|
||||
|
||||
# Mock get_items to return empty list
|
||||
with patch.object(session, "get_items", return_value=[]):
|
||||
popped_item = await session.pop_item()
|
||||
|
||||
assert popped_item is None
|
||||
mock_openai_client.conversations.items.delete.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_session(self, mock_openai_client):
|
||||
"""Test clearing the entire session."""
|
||||
session = OpenAIConversationsSession(
|
||||
conversation_id="test_id", openai_client=mock_openai_client
|
||||
)
|
||||
|
||||
await session.clear_session()
|
||||
|
||||
# Should delete the conversation and clear session ID
|
||||
mock_openai_client.conversations.delete.assert_called_once_with(conversation_id="test_id")
|
||||
assert session._session_id is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_clear_session_creates_session_id_first(self, mock_openai_client):
|
||||
"""Test that clear_session creates session_id if it doesn't exist."""
|
||||
session = OpenAIConversationsSession(openai_client=mock_openai_client)
|
||||
|
||||
await session.clear_session()
|
||||
|
||||
# Should create conversation first, then delete it
|
||||
mock_openai_client.conversations.create.assert_called_once_with(items=[])
|
||||
mock_openai_client.conversations.delete.assert_called_once_with(
|
||||
conversation_id="test_conversation_id"
|
||||
)
|
||||
assert session._session_id is None
|
||||
|
||||
|
||||
class TestOpenAIConversationsSessionRunnerIntegration:
|
||||
"""Test integration with Agent Runner using simple mocking."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_integration_basic(self, agent: Agent, mock_openai_client):
|
||||
"""Test that OpenAIConversationsSession works with Agent Runner."""
|
||||
session = OpenAIConversationsSession(openai_client=mock_openai_client)
|
||||
|
||||
# Mock the session methods to avoid complex async iterator setup
|
||||
with patch.object(session, "get_items", return_value=[]):
|
||||
with patch.object(session, "add_items") as mock_add_items:
|
||||
# Run the agent
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("San Francisco")])
|
||||
|
||||
result = await Runner.run(
|
||||
agent, "What city is the Golden Gate Bridge in?", session=session
|
||||
)
|
||||
|
||||
assert result.final_output == "San Francisco"
|
||||
|
||||
# Verify session interactions occurred
|
||||
mock_add_items.assert_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_with_conversation_history(self, agent: Agent, mock_openai_client):
|
||||
"""Test that conversation history is preserved across Runner calls."""
|
||||
session = OpenAIConversationsSession(openai_client=mock_openai_client)
|
||||
|
||||
# Mock conversation history
|
||||
conversation_history = [
|
||||
{"role": "user", "content": "What city is the Golden Gate Bridge in?"},
|
||||
{"role": "assistant", "content": "San Francisco"},
|
||||
]
|
||||
|
||||
with patch.object(session, "get_items", return_value=conversation_history):
|
||||
with patch.object(session, "add_items"):
|
||||
# Second turn - should have access to previous conversation
|
||||
assert isinstance(agent.model, FakeModel)
|
||||
agent.model.set_next_output([get_text_message("California")])
|
||||
|
||||
result = await Runner.run(agent, "What state is it in?", session=session)
|
||||
|
||||
assert result.final_output == "California"
|
||||
|
||||
# Verify that the model received the conversation history
|
||||
last_input = agent.model.last_turn_args["input"]
|
||||
assert len(last_input) > 1 # Should include previous messages
|
||||
|
||||
# Check that previous conversation is included
|
||||
input_contents = [str(item.get("content", "")) for item in last_input]
|
||||
assert any("Golden Gate Bridge" in content for content in input_contents)
|
||||
|
||||
|
||||
class TestOpenAIConversationsSessionErrorHandling:
|
||||
"""Test error handling for various failure scenarios."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_failure_during_conversation_creation(self, mock_openai_client):
|
||||
"""Test handling of API failures during conversation creation."""
|
||||
session = OpenAIConversationsSession(openai_client=mock_openai_client)
|
||||
|
||||
# Mock API failure
|
||||
mock_openai_client.conversations.create.side_effect = Exception("API Error")
|
||||
|
||||
with pytest.raises(Exception, match="API Error"):
|
||||
await session._get_session_id()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_failure_during_add_items(self, mock_openai_client):
|
||||
"""Test handling of API failures during add_items."""
|
||||
session = OpenAIConversationsSession(
|
||||
conversation_id="test_id", openai_client=mock_openai_client
|
||||
)
|
||||
|
||||
mock_openai_client.conversations.items.create.side_effect = Exception("Add items failed")
|
||||
|
||||
items: list[TResponseInputItem] = [{"role": "user", "content": "Hello"}]
|
||||
|
||||
with pytest.raises(Exception, match="Add items failed"):
|
||||
await session.add_items(items)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_failure_during_clear_session(self, mock_openai_client):
|
||||
"""Test handling of API failures during clear_session."""
|
||||
session = OpenAIConversationsSession(
|
||||
conversation_id="test_id", openai_client=mock_openai_client
|
||||
)
|
||||
|
||||
mock_openai_client.conversations.delete.side_effect = Exception("Clear session failed")
|
||||
|
||||
with pytest.raises(Exception, match="Clear session failed"):
|
||||
await session.clear_session()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_item_id_in_pop_item(self, mock_openai_client):
|
||||
"""Test handling of invalid item ID during pop_item."""
|
||||
session = OpenAIConversationsSession(
|
||||
conversation_id="test_id", openai_client=mock_openai_client
|
||||
)
|
||||
|
||||
# Mock item without ID
|
||||
invalid_item = {"role": "assistant", "content": "No ID"}
|
||||
|
||||
with patch.object(session, "get_items", return_value=[invalid_item]):
|
||||
# This should raise a KeyError because 'id' field is missing
|
||||
with pytest.raises(KeyError, match="'id'"):
|
||||
await session.pop_item()
|
||||
|
||||
|
||||
class TestOpenAIConversationsSessionConcurrentAccess:
|
||||
"""Test concurrent access patterns with simple scenarios."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_sessions_different_conversation_ids(self, mock_openai_client):
|
||||
"""Test that multiple sessions with different conversation IDs are isolated."""
|
||||
session1 = OpenAIConversationsSession(
|
||||
conversation_id="conversation_1", openai_client=mock_openai_client
|
||||
)
|
||||
session2 = OpenAIConversationsSession(
|
||||
conversation_id="conversation_2", openai_client=mock_openai_client
|
||||
)
|
||||
|
||||
items1: list[TResponseInputItem] = [{"role": "user", "content": "Session 1 message"}]
|
||||
items2: list[TResponseInputItem] = [{"role": "user", "content": "Session 2 message"}]
|
||||
|
||||
# Add items to both sessions
|
||||
await session1.add_items(items1)
|
||||
await session2.add_items(items2)
|
||||
|
||||
# Verify calls were made with correct conversation IDs
|
||||
assert mock_openai_client.conversations.items.create.call_count == 2
|
||||
|
||||
# Check the calls
|
||||
calls = mock_openai_client.conversations.items.create.call_args_list
|
||||
assert calls[0][1]["conversation_id"] == "conversation_1"
|
||||
assert calls[0][1]["items"] == items1
|
||||
assert calls[1][1]["conversation_id"] == "conversation_2"
|
||||
assert calls[1][1]["items"] == items2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_id_lazy_creation_consistency(self, mock_openai_client):
|
||||
"""Test that session ID creation is consistent across multiple calls."""
|
||||
session = OpenAIConversationsSession(openai_client=mock_openai_client)
|
||||
|
||||
# Call _get_session_id multiple times
|
||||
id1 = await session._get_session_id()
|
||||
id2 = await session._get_session_id()
|
||||
id3 = await session._get_session_id()
|
||||
|
||||
# All should return the same session ID
|
||||
assert id1 == id2 == id3 == "test_conversation_id"
|
||||
|
||||
# Conversation should only be created once
|
||||
mock_openai_client.conversations.create.assert_called_once()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# SessionSettings Tests
|
||||
# ============================================================================
|
||||
|
||||
|
||||
class TestOpenAIConversationsSessionSettings:
|
||||
"""Test SessionSettings integration with OpenAIConversationsSession."""
|
||||
|
||||
def test_session_settings_default(self, mock_openai_client):
|
||||
"""Test that session_settings defaults to empty SessionSettings."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
session = OpenAIConversationsSession(openai_client=mock_openai_client)
|
||||
|
||||
# Should have default SessionSettings
|
||||
assert isinstance(session.session_settings, SessionSettings)
|
||||
assert session.session_settings.limit is None
|
||||
|
||||
def test_session_settings_constructor(self, mock_openai_client):
|
||||
"""Test passing session_settings via constructor."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
session = OpenAIConversationsSession(
|
||||
openai_client=mock_openai_client, session_settings=SessionSettings(limit=5)
|
||||
)
|
||||
|
||||
assert session.session_settings is not None
|
||||
assert session.session_settings.limit == 5
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,808 @@
|
||||
"""Tests for session memory functionality."""
|
||||
|
||||
import asyncio
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from agents import Agent, RunConfig, Runner, SQLiteSession, TResponseInputItem
|
||||
from tests.fake_model import FakeModel
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
|
||||
# Helper functions for parametrized testing of different Runner methods
|
||||
def _run_sync_wrapper(agent, input_data, **kwargs):
|
||||
"""Wrapper for run_sync that properly sets up an event loop."""
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
try:
|
||||
return Runner.run_sync(agent, input_data, **kwargs)
|
||||
finally:
|
||||
loop.close()
|
||||
|
||||
|
||||
async def run_agent_async(runner_method: str, agent, input_data, **kwargs):
|
||||
"""Helper function to run agent with different methods."""
|
||||
if runner_method == "run":
|
||||
return await Runner.run(agent, input_data, **kwargs)
|
||||
elif runner_method == "run_sync":
|
||||
# For run_sync, we need to run it in a thread with its own event loop
|
||||
return await asyncio.to_thread(_run_sync_wrapper, agent, input_data, **kwargs)
|
||||
elif runner_method == "run_streamed":
|
||||
result = Runner.run_streamed(agent, input_data, **kwargs)
|
||||
# For streaming, we first try to get at least one event to trigger any early exceptions
|
||||
# If there's an exception in setup (like memory validation), it will be raised here
|
||||
try:
|
||||
first_event = None
|
||||
async for event in result.stream_events():
|
||||
if first_event is None:
|
||||
first_event = event
|
||||
# Continue consuming all events
|
||||
pass
|
||||
except Exception:
|
||||
# If an exception occurs during streaming, we let it propagate up
|
||||
raise
|
||||
return result
|
||||
else:
|
||||
raise ValueError(f"Unknown runner method: {runner_method}")
|
||||
|
||||
|
||||
# Parametrized tests for different runner methods
|
||||
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_memory_basic_functionality_parametrized(runner_method):
|
||||
"""Test basic session memory functionality with SQLite backend across all runner methods."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_memory.db"
|
||||
session_id = "test_session_123"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# First turn
|
||||
model.set_next_output([get_text_message("San Francisco")])
|
||||
result1 = await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
"What city is the Golden Gate Bridge in?",
|
||||
session=session,
|
||||
)
|
||||
assert result1.final_output == "San Francisco"
|
||||
|
||||
# Second turn - should have conversation history
|
||||
model.set_next_output([get_text_message("California")])
|
||||
result2 = await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
"What state is it in?",
|
||||
session=session,
|
||||
)
|
||||
assert result2.final_output == "California"
|
||||
|
||||
# Verify that the input to the second turn includes the previous conversation
|
||||
# The model should have received the full conversation history
|
||||
last_input = model.last_turn_args["input"]
|
||||
assert len(last_input) > 1 # Should have more than just the current message
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_memory_with_explicit_instance_parametrized(runner_method):
|
||||
"""Test session memory with an explicit SQLiteSession instance across all runner methods."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_memory.db"
|
||||
session_id = "test_session_456"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# First turn
|
||||
model.set_next_output([get_text_message("Hello")])
|
||||
result1 = await run_agent_async(runner_method, agent, "Hi there", session=session)
|
||||
assert result1.final_output == "Hello"
|
||||
|
||||
# Second turn
|
||||
model.set_next_output([get_text_message("I remember you said hi")])
|
||||
result2 = await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
"Do you remember what I said?",
|
||||
session=session,
|
||||
)
|
||||
assert result2.final_output == "I remember you said hi"
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_memory_disabled_parametrized(runner_method):
|
||||
"""Test that session memory is disabled when session=None across all runner methods."""
|
||||
model = FakeModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# First turn (no session parameters = disabled)
|
||||
model.set_next_output([get_text_message("Hello")])
|
||||
result1 = await run_agent_async(runner_method, agent, "Hi there")
|
||||
assert result1.final_output == "Hello"
|
||||
|
||||
# Second turn - should NOT have conversation history
|
||||
model.set_next_output([get_text_message("I don't remember")])
|
||||
result2 = await run_agent_async(runner_method, agent, "Do you remember what I said?")
|
||||
assert result2.final_output == "I don't remember"
|
||||
|
||||
# Verify that the input to the second turn is just the current message
|
||||
last_input = model.last_turn_args["input"]
|
||||
assert len(last_input) == 1 # Should only have the current message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_memory_different_sessions_parametrized(runner_method):
|
||||
"""Test that different session IDs maintain separate conversation histories across all runner
|
||||
methods."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_memory.db"
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# Session 1
|
||||
session_id_1 = "session_1"
|
||||
session_1 = SQLiteSession(session_id_1, db_path)
|
||||
|
||||
model.set_next_output([get_text_message("I like cats")])
|
||||
result1 = await run_agent_async(runner_method, agent, "I like cats", session=session_1)
|
||||
assert result1.final_output == "I like cats"
|
||||
|
||||
# Session 2 - different session
|
||||
session_id_2 = "session_2"
|
||||
session_2 = SQLiteSession(session_id_2, db_path)
|
||||
|
||||
model.set_next_output([get_text_message("I like dogs")])
|
||||
result2 = await run_agent_async(runner_method, agent, "I like dogs", session=session_2)
|
||||
assert result2.final_output == "I like dogs"
|
||||
|
||||
# Back to Session 1 - should remember cats, not dogs
|
||||
model.set_next_output([get_text_message("Yes, you mentioned cats")])
|
||||
result3 = await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
"What did I say I like?",
|
||||
session=session_1,
|
||||
)
|
||||
assert result3.final_output == "Yes, you mentioned cats"
|
||||
|
||||
session_1.close()
|
||||
session_2.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sqlite_session_memory_direct():
|
||||
"""Test SQLiteSession class directly."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_direct.db"
|
||||
session_id = "direct_test"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
# Test adding and retrieving items
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there!"},
|
||||
]
|
||||
|
||||
await session.add_items(items)
|
||||
retrieved = await session.get_items()
|
||||
|
||||
assert len(retrieved) == 2
|
||||
assert retrieved[0].get("role") == "user"
|
||||
assert retrieved[0].get("content") == "Hello"
|
||||
assert retrieved[1].get("role") == "assistant"
|
||||
assert retrieved[1].get("content") == "Hi there!"
|
||||
|
||||
# Test clearing session
|
||||
await session.clear_session()
|
||||
retrieved_after_clear = await session.get_items()
|
||||
assert len(retrieved_after_clear) == 0
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sqlite_session_close_closes_worker_thread_connections():
|
||||
"""Test that close cleans up connections opened by async worker threads."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_worker_thread_close.db"
|
||||
session = SQLiteSession("worker_thread_close", db_path)
|
||||
|
||||
await session.add_items([{"role": "user", "content": "Hello"}])
|
||||
connections = list(session._connections)
|
||||
|
||||
assert connections
|
||||
|
||||
session.close()
|
||||
|
||||
assert session._connections == set()
|
||||
with pytest.raises(sqlite3.ProgrammingError):
|
||||
connections[0].execute("SELECT 1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sqlite_session_memory_pop_item():
|
||||
"""Test SQLiteSession pop_item functionality."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_pop.db"
|
||||
session_id = "pop_test"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
# Test popping from empty session
|
||||
popped = await session.pop_item()
|
||||
assert popped is None
|
||||
|
||||
# Add items
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there!"},
|
||||
{"role": "user", "content": "How are you?"},
|
||||
]
|
||||
|
||||
await session.add_items(items)
|
||||
|
||||
# Verify all items are there
|
||||
retrieved = await session.get_items()
|
||||
assert len(retrieved) == 3
|
||||
|
||||
# Pop the most recent item
|
||||
popped = await session.pop_item()
|
||||
assert popped is not None
|
||||
assert popped.get("role") == "user"
|
||||
assert popped.get("content") == "How are you?"
|
||||
|
||||
# Verify item was removed
|
||||
retrieved_after_pop = await session.get_items()
|
||||
assert len(retrieved_after_pop) == 2
|
||||
assert retrieved_after_pop[-1].get("content") == "Hi there!"
|
||||
|
||||
# Pop another item
|
||||
popped2 = await session.pop_item()
|
||||
assert popped2 is not None
|
||||
assert popped2.get("role") == "assistant"
|
||||
assert popped2.get("content") == "Hi there!"
|
||||
|
||||
# Pop the last item
|
||||
popped3 = await session.pop_item()
|
||||
assert popped3 is not None
|
||||
assert popped3.get("role") == "user"
|
||||
assert popped3.get("content") == "Hello"
|
||||
|
||||
# Try to pop from empty session again
|
||||
popped4 = await session.pop_item()
|
||||
assert popped4 is None
|
||||
|
||||
# Verify session is empty
|
||||
final_items = await session.get_items()
|
||||
assert len(final_items) == 0
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_memory_pop_different_sessions():
|
||||
"""Test that pop_item only affects the specified session."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_pop_sessions.db"
|
||||
|
||||
session_1_id = "session_1"
|
||||
session_2_id = "session_2"
|
||||
session_1 = SQLiteSession(session_1_id, db_path)
|
||||
session_2 = SQLiteSession(session_2_id, db_path)
|
||||
|
||||
# Add items to both sessions
|
||||
items_1: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Session 1 message"},
|
||||
]
|
||||
items_2: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Session 2 message 1"},
|
||||
{"role": "user", "content": "Session 2 message 2"},
|
||||
]
|
||||
|
||||
await session_1.add_items(items_1)
|
||||
await session_2.add_items(items_2)
|
||||
|
||||
# Pop from session 2
|
||||
popped = await session_2.pop_item()
|
||||
assert popped is not None
|
||||
assert popped.get("content") == "Session 2 message 2"
|
||||
|
||||
# Verify session 1 is unaffected
|
||||
session_1_items = await session_1.get_items()
|
||||
assert len(session_1_items) == 1
|
||||
assert session_1_items[0].get("content") == "Session 1 message"
|
||||
|
||||
# Verify session 2 has one item left
|
||||
session_2_items = await session_2.get_items()
|
||||
assert len(session_2_items) == 1
|
||||
assert session_2_items[0].get("content") == "Session 2 message 1"
|
||||
|
||||
session_1.close()
|
||||
session_2.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sqlite_session_pop_item_skips_corrupt_most_recent():
|
||||
"""pop_item skips corrupt newest rows and returns the next valid item."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_pop_corrupt.db"
|
||||
session = SQLiteSession("pop_corrupt", db_path)
|
||||
|
||||
valid_item: TResponseInputItem = {"role": "user", "content": "valid"}
|
||||
await session.add_items([valid_item])
|
||||
|
||||
with session._locked_connection() as conn:
|
||||
conn.execute(
|
||||
f"INSERT INTO {session.messages_table} (session_id, message_data) VALUES (?, ?)",
|
||||
(session.session_id, "not valid json {{{"),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
assert await session.pop_item() == valid_item
|
||||
assert await session.get_items() == []
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sqlite_session_pop_item_returns_none_after_dropping_only_corrupt_rows():
|
||||
"""pop_item removes corrupt rows and returns None when no valid items remain."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_pop_only_corrupt.db"
|
||||
session = SQLiteSession("pop_only_corrupt", db_path)
|
||||
|
||||
with session._locked_connection() as conn:
|
||||
conn.execute(
|
||||
f"INSERT INTO {session.messages_table} (session_id, message_data) VALUES (?, ?)",
|
||||
(session.session_id, "not valid json {{{"),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
assert await session.pop_item() is None
|
||||
assert await session.get_items() == []
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sqlite_session_get_items_with_limit():
|
||||
"""Test SQLiteSession get_items with limit parameter."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_count.db"
|
||||
session_id = "count_test"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
# Add multiple items
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Message 1"},
|
||||
{"role": "assistant", "content": "Response 1"},
|
||||
{"role": "user", "content": "Message 2"},
|
||||
{"role": "assistant", "content": "Response 2"},
|
||||
{"role": "user", "content": "Message 3"},
|
||||
{"role": "assistant", "content": "Response 3"},
|
||||
]
|
||||
|
||||
await session.add_items(items)
|
||||
|
||||
# Test getting all items (default behavior)
|
||||
all_items = await session.get_items()
|
||||
assert len(all_items) == 6
|
||||
assert all_items[0].get("content") == "Message 1"
|
||||
assert all_items[-1].get("content") == "Response 3"
|
||||
|
||||
# Test getting latest 2 items
|
||||
latest_2 = await session.get_items(limit=2)
|
||||
assert len(latest_2) == 2
|
||||
assert latest_2[0].get("content") == "Message 3"
|
||||
assert latest_2[1].get("content") == "Response 3"
|
||||
|
||||
# Test getting latest 4 items
|
||||
latest_4 = await session.get_items(limit=4)
|
||||
assert len(latest_4) == 4
|
||||
assert latest_4[0].get("content") == "Message 2"
|
||||
assert latest_4[1].get("content") == "Response 2"
|
||||
assert latest_4[2].get("content") == "Message 3"
|
||||
assert latest_4[3].get("content") == "Response 3"
|
||||
|
||||
# Test getting more items than available
|
||||
latest_10 = await session.get_items(limit=10)
|
||||
assert len(latest_10) == 6 # Should return all available items
|
||||
assert latest_10[0].get("content") == "Message 1"
|
||||
assert latest_10[-1].get("content") == "Response 3"
|
||||
|
||||
# Test getting 0 items
|
||||
latest_0 = await session.get_items(limit=0)
|
||||
assert len(latest_0) == 0
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_memory_appends_list_input_by_default(runner_method):
|
||||
"""Test that list inputs are appended to session history when no callback is provided."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_validation.db"
|
||||
session_id = "test_validation_parametrized"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
initial_history: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Earlier message"},
|
||||
{"role": "assistant", "content": "Saved reply"},
|
||||
]
|
||||
await session.add_items(initial_history)
|
||||
|
||||
list_input = [{"role": "user", "content": "Test message"}]
|
||||
|
||||
model.set_next_output([get_text_message("This should run")])
|
||||
await run_agent_async(runner_method, agent, list_input, session=session)
|
||||
|
||||
assert model.last_turn_args["input"] == initial_history + list_input
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_callback_prepared_input(runner_method):
|
||||
"""Test if the user passes a list of items and want to append them."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_memory.db"
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# Session
|
||||
session_id = "session_1"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
# Add first messages manually
|
||||
initial_history: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "Hello there."},
|
||||
{"role": "assistant", "content": "Hi, I'm here to assist you."},
|
||||
]
|
||||
try:
|
||||
await session.add_items(initial_history)
|
||||
|
||||
def filter_assistant_messages(history, new_input):
|
||||
# Only include user messages from history
|
||||
return [item for item in history if item["role"] == "user"] + new_input
|
||||
|
||||
new_turn_input = [{"role": "user", "content": "What your name?"}]
|
||||
model.set_next_output([get_text_message("I'm gpt-4o")])
|
||||
|
||||
# Run the agent with the callable
|
||||
await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
new_turn_input,
|
||||
session=session,
|
||||
run_config=RunConfig(session_input_callback=filter_assistant_messages),
|
||||
)
|
||||
|
||||
expected_model_input = [
|
||||
initial_history[0], # From history
|
||||
new_turn_input[0], # New input
|
||||
]
|
||||
|
||||
assert len(model.last_turn_args["input"]) == 2
|
||||
assert model.last_turn_args["input"] == expected_model_input
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sqlite_session_unicode_content():
|
||||
"""Test that session correctly stores and retrieves unicode/non-ASCII content."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_unicode.db"
|
||||
session_id = "unicode_test"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
# Add unicode content to the session
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "こんにちは"},
|
||||
{"role": "assistant", "content": "😊👍"},
|
||||
{"role": "user", "content": "Привет"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
# Retrieve items and verify unicode content
|
||||
retrieved = await session.get_items()
|
||||
assert retrieved[0].get("content") == "こんにちは"
|
||||
assert retrieved[1].get("content") == "😊👍"
|
||||
assert retrieved[2].get("content") == "Привет"
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sqlite_session_special_characters_and_sql_injection():
|
||||
"""
|
||||
Test that session safely stores and retrieves items with special characters and SQL keywords.
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_special_chars.db"
|
||||
session_id = "special_chars_test"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
# Add items with special characters and SQL keywords
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": "O'Reilly"},
|
||||
{"role": "assistant", "content": "DROP TABLE sessions;"},
|
||||
{"role": "user", "content": ('"SELECT * FROM users WHERE name = "admin";"')},
|
||||
{"role": "assistant", "content": "Robert'); DROP TABLE students;--"},
|
||||
{"role": "user", "content": "Normal message"},
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
# Retrieve all items and verify they are stored correctly
|
||||
retrieved = await session.get_items()
|
||||
assert len(retrieved) == len(items)
|
||||
assert retrieved[0].get("content") == "O'Reilly"
|
||||
assert retrieved[1].get("content") == "DROP TABLE sessions;"
|
||||
assert retrieved[2].get("content") == '"SELECT * FROM users WHERE name = "admin";"'
|
||||
assert retrieved[3].get("content") == "Robert'); DROP TABLE students;--"
|
||||
assert retrieved[4].get("content") == "Normal message"
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sqlite_session_concurrent_access():
|
||||
"""
|
||||
Test concurrent access to the same session to verify data integrity.
|
||||
"""
|
||||
import concurrent.futures
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_concurrent.db"
|
||||
session_id = "concurrent_test"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
# Add initial item
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": f"Message {i}"} for i in range(10)
|
||||
]
|
||||
|
||||
# Use ThreadPoolExecutor to simulate concurrent writes
|
||||
def add_item(item):
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
loop.run_until_complete(session.add_items([item]))
|
||||
loop.close()
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
|
||||
executor.map(add_item, items)
|
||||
|
||||
# Retrieve all items and verify all are present
|
||||
retrieved = await session.get_items()
|
||||
contents = {
|
||||
content
|
||||
for item in retrieved
|
||||
for content in [item.get("content")]
|
||||
if isinstance(content, str)
|
||||
}
|
||||
expected = {f"Message {i}" for i in range(10)}
|
||||
assert contents == expected
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sqlite_session_file_lock_is_shared_across_instances():
|
||||
"""File-backed sessions pointing at the same DB path should reuse one process-local lock."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_shared_lock.db"
|
||||
lock_path = db_path.resolve()
|
||||
|
||||
session_1 = SQLiteSession("session_1", db_path)
|
||||
session_2 = SQLiteSession("session_2", db_path)
|
||||
|
||||
assert session_1._lock is session_2._lock
|
||||
assert SQLiteSession._file_lock_counts[lock_path] == 2
|
||||
|
||||
await asyncio.gather(
|
||||
session_1.add_items([{"role": "user", "content": "session_1"}]),
|
||||
session_2.add_items([{"role": "user", "content": "session_2"}]),
|
||||
)
|
||||
|
||||
assert [item.get("content") for item in await session_1.get_items()] == ["session_1"]
|
||||
assert [item.get("content") for item in await session_2.get_items()] == ["session_2"]
|
||||
|
||||
session_1.close()
|
||||
assert SQLiteSession._file_lock_counts[lock_path] == 1
|
||||
assert lock_path in SQLiteSession._file_locks
|
||||
|
||||
session_2.close()
|
||||
assert lock_path not in SQLiteSession._file_lock_counts
|
||||
assert lock_path not in SQLiteSession._file_locks
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_add_items_exception_propagates_in_streamed():
|
||||
"""Test that exceptions from session.add_items are properly propagated
|
||||
in run_streamed instead of causing the stream to hang forever.
|
||||
Regression test for https://github.com/openai/openai-agents-python/issues/2130
|
||||
"""
|
||||
session = SQLiteSession("test_exception_session")
|
||||
|
||||
async def _failing_add_items(_items):
|
||||
raise RuntimeError("Simulated session.add_items failure")
|
||||
|
||||
session.add_items = _failing_add_items # type: ignore[method-assign]
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
model.set_next_output([get_text_message("This should not be reached")])
|
||||
|
||||
result = Runner.run_streamed(agent, "Hello", session=session)
|
||||
|
||||
async def consume_stream():
|
||||
async for _event in result.stream_events():
|
||||
pass
|
||||
|
||||
with pytest.raises(RuntimeError, match="Simulated session.add_items failure"):
|
||||
# Timeout ensures test fails fast instead of hanging forever if bug regresses
|
||||
await asyncio.wait_for(consume_stream(), timeout=5.0)
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# SessionSettings Tests
|
||||
# ============================================================================
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_settings_default():
|
||||
"""Test that session_settings defaults to empty SessionSettings."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
session = SQLiteSession("default_settings_test")
|
||||
|
||||
# Should have default SessionSettings
|
||||
assert isinstance(session.session_settings, SessionSettings)
|
||||
assert session.session_settings.limit is None
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_settings_constructor():
|
||||
"""Test passing session_settings via constructor."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
session = SQLiteSession("constructor_settings_test", session_settings=SessionSettings(limit=5))
|
||||
|
||||
assert session.session_settings is not None
|
||||
assert session.session_settings.limit == 5
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_items_uses_session_settings_limit():
|
||||
"""Test that get_items uses session_settings.limit as default."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_settings_limit.db"
|
||||
session = SQLiteSession(
|
||||
"uses_settings_limit_test", db_path, session_settings=SessionSettings(limit=3)
|
||||
)
|
||||
|
||||
# Add 5 items
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": f"Message {i}"} for i in range(5)
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
# get_items() with no limit should use session_settings.limit=3
|
||||
retrieved = await session.get_items()
|
||||
assert len(retrieved) == 3
|
||||
# Should get the last 3 items
|
||||
assert retrieved[0].get("content") == "Message 2"
|
||||
assert retrieved[1].get("content") == "Message 3"
|
||||
assert retrieved[2].get("content") == "Message 4"
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_items_explicit_limit_overrides_session_settings():
|
||||
"""Test that explicit limit parameter overrides session_settings."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_override.db"
|
||||
session = SQLiteSession(
|
||||
"explicit_override_test", db_path, session_settings=SessionSettings(limit=5)
|
||||
)
|
||||
|
||||
# Add 10 items
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": f"Message {i}"} for i in range(10)
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
# Explicit limit=2 should override session_settings.limit=5
|
||||
retrieved = await session.get_items(limit=2)
|
||||
assert len(retrieved) == 2
|
||||
assert retrieved[0].get("content") == "Message 8"
|
||||
assert retrieved[1].get("content") == "Message 9"
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_settings_resolve():
|
||||
"""Test SessionSettings.resolve() method."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
base = SessionSettings(limit=100)
|
||||
override = SessionSettings(limit=50)
|
||||
|
||||
final = base.resolve(override)
|
||||
|
||||
assert final.limit == 50 # Override wins
|
||||
assert base.limit == 100 # Original unchanged
|
||||
|
||||
# Resolving with None returns self
|
||||
final_none = base.resolve(None)
|
||||
assert final_none.limit == 100
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runner_with_session_settings_override():
|
||||
"""Test that RunConfig can override session's default settings."""
|
||||
from agents.memory import SessionSettings
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_runner_override.db"
|
||||
|
||||
# Session with default limit=100
|
||||
session = SQLiteSession(
|
||||
"runner_override_test", db_path, session_settings=SessionSettings(limit=100)
|
||||
)
|
||||
|
||||
# Add some history
|
||||
items: list[TResponseInputItem] = [
|
||||
{"role": "user", "content": f"Turn {i}"} for i in range(10)
|
||||
]
|
||||
await session.add_items(items)
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
model.set_next_output([get_text_message("Got it")])
|
||||
|
||||
await Runner.run(
|
||||
agent,
|
||||
"New question",
|
||||
session=session,
|
||||
run_config=RunConfig(
|
||||
session_settings=SessionSettings(limit=2) # Override to 2
|
||||
),
|
||||
)
|
||||
|
||||
# Verify the agent received only the last 2 history items + new question
|
||||
last_input = model.last_turn_args["input"]
|
||||
# Filter out the new "New question" input
|
||||
history_items = [item for item in last_input if item.get("content") != "New question"]
|
||||
# Should have 2 history items (last two from the 10 we added)
|
||||
assert len(history_items) == 2
|
||||
|
||||
session.close()
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Test session_limit parameter functionality via SessionSettings."""
|
||||
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from agents import Agent, RunConfig, SQLiteSession
|
||||
from agents.memory import SessionSettings
|
||||
from tests.fake_model import FakeModel
|
||||
from tests.memory.test_session import run_agent_async
|
||||
from tests.test_responses import get_text_message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_limit_parameter(runner_method):
|
||||
"""Test that session_limit parameter correctly limits conversation history
|
||||
retrieved from session across all Runner methods."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_limit.db"
|
||||
session_id = "limit_test"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# Build up a longer conversation history
|
||||
model.set_next_output([get_text_message("Reply 1")])
|
||||
await run_agent_async(runner_method, agent, "Message 1", session=session)
|
||||
|
||||
model.set_next_output([get_text_message("Reply 2")])
|
||||
await run_agent_async(runner_method, agent, "Message 2", session=session)
|
||||
|
||||
model.set_next_output([get_text_message("Reply 3")])
|
||||
await run_agent_async(runner_method, agent, "Message 3", session=session)
|
||||
|
||||
# Verify we have 6 items in total (3 user + 3 assistant)
|
||||
all_items = await session.get_items()
|
||||
assert len(all_items) == 6
|
||||
|
||||
# Test session_limit via RunConfig - should only get last 2 history items + new input
|
||||
model.set_next_output([get_text_message("Reply 4")])
|
||||
await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
"Message 4",
|
||||
session=session,
|
||||
run_config=RunConfig(session_settings=SessionSettings(limit=2)),
|
||||
)
|
||||
|
||||
# Verify model received limited history
|
||||
last_input = model.last_turn_args["input"]
|
||||
# Should have: 2 history items + 1 new message = 3 total
|
||||
assert len(last_input) == 3
|
||||
# First item should be "Message 3" (not Message 1 or 2)
|
||||
assert last_input[0].get("content") == "Message 3"
|
||||
# Assistant message has content as a list
|
||||
assert last_input[1].get("content")[0]["text"] == "Reply 3"
|
||||
assert last_input[2].get("content") == "Message 4"
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_limit_zero(runner_method):
|
||||
"""Test that session_limit=0 provides no history, only new message."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_limit_zero.db"
|
||||
session_id = "limit_zero_test"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# Build conversation history
|
||||
model.set_next_output([get_text_message("Reply 1")])
|
||||
await run_agent_async(runner_method, agent, "Message 1", session=session)
|
||||
|
||||
model.set_next_output([get_text_message("Reply 2")])
|
||||
await run_agent_async(runner_method, agent, "Message 2", session=session)
|
||||
|
||||
# Test with limit=0 - should get NO history, just new message
|
||||
model.set_next_output([get_text_message("Reply 3")])
|
||||
await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
"Message 3",
|
||||
session=session,
|
||||
run_config=RunConfig(session_settings=SessionSettings(limit=0)),
|
||||
)
|
||||
|
||||
# Verify model received only the new message
|
||||
last_input = model.last_turn_args["input"]
|
||||
assert len(last_input) == 1
|
||||
assert last_input[0].get("content") == "Message 3"
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_limit_none_gets_all_history(runner_method):
|
||||
"""Test that session_limit=None retrieves entire history (default behavior)."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_limit_none.db"
|
||||
session_id = "limit_none_test"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# Build longer conversation
|
||||
for i in range(1, 6):
|
||||
model.set_next_output([get_text_message(f"Reply {i}")])
|
||||
await run_agent_async(runner_method, agent, f"Message {i}", session=session)
|
||||
|
||||
# Verify 10 items in session (5 user + 5 assistant)
|
||||
all_items = await session.get_items()
|
||||
assert len(all_items) == 10
|
||||
|
||||
# Test with session_limit=None (default) - should get all history
|
||||
model.set_next_output([get_text_message("Reply 6")])
|
||||
await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
"Message 6",
|
||||
session=session,
|
||||
run_config=RunConfig(session_settings=SessionSettings(limit=None)),
|
||||
)
|
||||
|
||||
# Verify model received all history + new message
|
||||
last_input = model.last_turn_args["input"]
|
||||
assert len(last_input) == 11 # 10 history + 1 new
|
||||
assert last_input[0].get("content") == "Message 1"
|
||||
assert last_input[-1].get("content") == "Message 6"
|
||||
|
||||
session.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("runner_method", ["run", "run_sync", "run_streamed"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_limit_larger_than_history(runner_method):
|
||||
"""Test that session_limit larger than history size returns all items."""
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
db_path = Path(temp_dir) / "test_limit_large.db"
|
||||
session_id = "limit_large_test"
|
||||
session = SQLiteSession(session_id, db_path)
|
||||
|
||||
model = FakeModel()
|
||||
agent = Agent(name="test", model=model)
|
||||
|
||||
# Build small conversation
|
||||
model.set_next_output([get_text_message("Reply 1")])
|
||||
await run_agent_async(runner_method, agent, "Message 1", session=session)
|
||||
|
||||
# Test with limit=100 (much larger than actual history)
|
||||
model.set_next_output([get_text_message("Reply 2")])
|
||||
await run_agent_async(
|
||||
runner_method,
|
||||
agent,
|
||||
"Message 2",
|
||||
session=session,
|
||||
run_config=RunConfig(session_settings=SessionSettings(limit=100)),
|
||||
)
|
||||
|
||||
# Verify model received all available history + new message
|
||||
last_input = model.last_turn_args["input"]
|
||||
assert len(last_input) == 3 # 2 history + 1 new
|
||||
assert last_input[0].get("content") == "Message 1"
|
||||
# Assistant message has content as a list
|
||||
assert last_input[1].get("content")[0]["text"] == "Reply 1"
|
||||
assert last_input[2].get("content") == "Message 2"
|
||||
|
||||
session.close()
|
||||
@@ -0,0 +1,135 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from agents.items import TResponseInputItem
|
||||
from agents.run_internal.session_persistence import _sanitize_openai_conversation_item
|
||||
|
||||
|
||||
def _sanitize(item: dict[str, Any]) -> dict[str, Any]:
|
||||
return cast(dict[str, Any], _sanitize_openai_conversation_item(cast(TResponseInputItem, item)))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"item_type",
|
||||
[
|
||||
"file_search_call",
|
||||
"web_search_call",
|
||||
"computer_call",
|
||||
"code_interpreter_call",
|
||||
"image_generation_call",
|
||||
"local_shell_call",
|
||||
"local_shell_call_output",
|
||||
"mcp_list_tools",
|
||||
"mcp_approval_request",
|
||||
"mcp_call",
|
||||
"item_reference",
|
||||
],
|
||||
)
|
||||
def test_sanitize_preserves_ids_required_by_openai_conversation_items(item_type: str) -> None:
|
||||
item = {"type": item_type, "id": f"{item_type}_abc123", "status": "completed"}
|
||||
|
||||
sanitized = _sanitize(item)
|
||||
|
||||
assert sanitized["id"] == f"{item_type}_abc123"
|
||||
assert sanitized["type"] == item_type
|
||||
|
||||
|
||||
def test_sanitize_preserves_file_search_call_payload_id() -> None:
|
||||
item = {
|
||||
"type": "file_search_call",
|
||||
"id": "fs_call_abc",
|
||||
"queries": ["latest q3 revenue"],
|
||||
"status": "completed",
|
||||
"results": [{"file_id": "file_1", "filename": "q3.pdf", "score": 0.9, "text": "..."}],
|
||||
}
|
||||
|
||||
sanitized = _sanitize(item)
|
||||
|
||||
assert sanitized["id"] == "fs_call_abc"
|
||||
assert sanitized["queries"] == ["latest q3 revenue"]
|
||||
assert sanitized["status"] == "completed"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"item",
|
||||
[
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_abc",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "hi"}],
|
||||
},
|
||||
{
|
||||
"type": "function_call",
|
||||
"id": "fc_abc",
|
||||
"call_id": "call_abc",
|
||||
"name": "get_weather",
|
||||
"arguments": "{}",
|
||||
},
|
||||
{"type": "function_call_output", "id": "out_abc", "call_id": "call_abc", "output": "{}"},
|
||||
{"type": "computer_call_output", "id": "ccout_abc", "call_id": "call_abc", "output": {}},
|
||||
{"type": "tool_search_call", "id": "ts_abc", "status": "completed"},
|
||||
{"type": "shell_call", "id": "sh_abc", "call_id": "call_abc", "action": {}},
|
||||
],
|
||||
)
|
||||
def test_sanitize_strips_optional_or_policy_controlled_ids(item: dict[str, Any]) -> None:
|
||||
sanitized = _sanitize(item)
|
||||
|
||||
assert "id" not in sanitized
|
||||
assert sanitized["type"] == item["type"]
|
||||
|
||||
|
||||
def test_sanitize_preserves_reasoning_id_for_openai_conversations() -> None:
|
||||
item = {
|
||||
"type": "reasoning",
|
||||
"id": "rs_abc",
|
||||
"summary": [],
|
||||
"content": [],
|
||||
"provider_data": {"server": "metadata"},
|
||||
}
|
||||
|
||||
sanitized = _sanitize(item)
|
||||
|
||||
assert sanitized["id"] == "rs_abc"
|
||||
assert "provider_data" not in sanitized
|
||||
|
||||
|
||||
def test_sanitize_preserves_reasoning_encrypted_content() -> None:
|
||||
item = {
|
||||
"type": "reasoning",
|
||||
"summary": [],
|
||||
"content": [],
|
||||
"encrypted_content": "encrypted",
|
||||
}
|
||||
|
||||
sanitized = _sanitize(item)
|
||||
|
||||
assert sanitized["encrypted_content"] == "encrypted"
|
||||
|
||||
|
||||
def test_sanitize_always_strips_provider_data() -> None:
|
||||
item = {
|
||||
"type": "file_search_call",
|
||||
"id": "fs_keep",
|
||||
"status": "completed",
|
||||
"provider_data": {"model": "gpt-4.1-mini"},
|
||||
}
|
||||
|
||||
sanitized = _sanitize(item)
|
||||
|
||||
assert sanitized["id"] == "fs_keep"
|
||||
assert "provider_data" not in sanitized
|
||||
|
||||
|
||||
def test_sanitize_passes_through_non_dict_items() -> None:
|
||||
class DummyItem:
|
||||
pass
|
||||
|
||||
item = DummyItem()
|
||||
|
||||
sanitized: Any = _sanitize_openai_conversation_item(cast(TResponseInputItem, item))
|
||||
|
||||
assert sanitized is item
|
||||
@@ -0,0 +1,359 @@
|
||||
import json
|
||||
from dataclasses import fields
|
||||
|
||||
from openai.types.shared import Reasoning
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic_core import to_json
|
||||
|
||||
from agents.model_settings import MCPToolChoice, ModelSettings
|
||||
from agents.retry import ModelRetryBackoffSettings, ModelRetrySettings, retry_policies
|
||||
|
||||
|
||||
def verify_serialization(model_settings: ModelSettings) -> None:
|
||||
"""Verify that ModelSettings can be serialized to a JSON string."""
|
||||
json_dict = model_settings.to_json_dict()
|
||||
json_string = json.dumps(json_dict)
|
||||
assert json_string is not None
|
||||
|
||||
|
||||
def test_basic_serialization() -> None:
|
||||
"""Tests whether ModelSettings can be serialized to a JSON string."""
|
||||
|
||||
# First, lets create a ModelSettings instance
|
||||
model_settings = ModelSettings(
|
||||
temperature=0.5,
|
||||
top_p=0.9,
|
||||
max_tokens=100,
|
||||
)
|
||||
|
||||
# Now, lets serialize the ModelSettings instance to a JSON string
|
||||
verify_serialization(model_settings)
|
||||
|
||||
|
||||
def test_mcp_tool_choice_serialization() -> None:
|
||||
"""Tests whether ModelSettings with MCPToolChoice can be serialized to a JSON string."""
|
||||
# First, lets create a ModelSettings instance
|
||||
model_settings = ModelSettings(
|
||||
temperature=0.5,
|
||||
tool_choice=MCPToolChoice(server_label="mcp", name="mcp_tool"),
|
||||
)
|
||||
# Now, lets serialize the ModelSettings instance to a JSON string
|
||||
verify_serialization(model_settings)
|
||||
|
||||
|
||||
def test_all_fields_serialization() -> None:
|
||||
"""Tests whether ModelSettings can be serialized to a JSON string."""
|
||||
|
||||
# First, lets create a ModelSettings instance
|
||||
model_settings = ModelSettings(
|
||||
temperature=0.5,
|
||||
top_p=0.9,
|
||||
frequency_penalty=0.0,
|
||||
presence_penalty=0.0,
|
||||
tool_choice="auto",
|
||||
parallel_tool_calls=True,
|
||||
truncation="auto",
|
||||
max_tokens=100,
|
||||
reasoning=Reasoning(),
|
||||
metadata={"foo": "bar"},
|
||||
store=False,
|
||||
prompt_cache_retention="24h",
|
||||
include_usage=False,
|
||||
response_include=["reasoning.encrypted_content"],
|
||||
top_logprobs=1,
|
||||
verbosity="low",
|
||||
extra_query={"foo": "bar"},
|
||||
extra_body={"foo": "bar"},
|
||||
extra_headers={"foo": "bar"},
|
||||
extra_args={"custom_param": "value", "another_param": 42},
|
||||
retry=ModelRetrySettings(
|
||||
max_retries=2,
|
||||
backoff=ModelRetryBackoffSettings(
|
||||
initial_delay=0.1,
|
||||
max_delay=1.0,
|
||||
multiplier=2.0,
|
||||
jitter=False,
|
||||
),
|
||||
),
|
||||
context_management=[{"type": "compaction", "compact_threshold": 200000}],
|
||||
prompt_cache_options={"mode": "explicit", "ttl": "30m"},
|
||||
)
|
||||
|
||||
# Verify that every single field is set to a non-None value
|
||||
for field in fields(model_settings):
|
||||
assert getattr(model_settings, field.name) is not None, (
|
||||
f"You must set the {field.name} field"
|
||||
)
|
||||
|
||||
# Now, lets serialize the ModelSettings instance to a JSON string
|
||||
verify_serialization(model_settings)
|
||||
|
||||
|
||||
def test_gpt_5_6_reasoning_and_prompt_cache_serialization() -> None:
|
||||
model_settings = ModelSettings(
|
||||
reasoning=Reasoning(mode="pro", effort="max", context="all_turns"),
|
||||
prompt_cache_options={"mode": "explicit", "ttl": "30m"},
|
||||
)
|
||||
|
||||
serialized_reasoning = model_settings.to_json_dict()["reasoning"]
|
||||
assert serialized_reasoning["context"] == "all_turns"
|
||||
assert serialized_reasoning["effort"] == "max"
|
||||
assert serialized_reasoning["mode"] == "pro"
|
||||
assert model_settings.to_traceable_dict()["prompt_cache_options"] == {
|
||||
"mode": "explicit",
|
||||
"ttl": "30m",
|
||||
}
|
||||
|
||||
|
||||
def test_prompt_cache_options_is_appended_to_public_field_order() -> None:
|
||||
field_names = [field.name for field in fields(ModelSettings)]
|
||||
|
||||
assert field_names[-2:] == ["context_management", "prompt_cache_options"]
|
||||
|
||||
|
||||
def test_extra_args_serialization() -> None:
|
||||
"""Test that extra_args are properly serialized."""
|
||||
model_settings = ModelSettings(
|
||||
temperature=0.5,
|
||||
extra_args={"custom_param": "value", "another_param": 42, "nested": {"key": "value"}},
|
||||
)
|
||||
|
||||
json_dict = model_settings.to_json_dict()
|
||||
assert json_dict["extra_args"] == {
|
||||
"custom_param": "value",
|
||||
"another_param": 42,
|
||||
"nested": {"key": "value"},
|
||||
}
|
||||
|
||||
# Verify serialization works
|
||||
verify_serialization(model_settings)
|
||||
|
||||
|
||||
def test_traceable_serialization_omits_request_extras() -> None:
|
||||
model_settings = ModelSettings(
|
||||
temperature=0.5,
|
||||
extra_headers={"Authorization": "Bearer provider-token"},
|
||||
extra_query={"api-key": "query-token"},
|
||||
extra_body={"secret": "body-token"},
|
||||
extra_args={"api_key": "arg-token"},
|
||||
)
|
||||
|
||||
json_dict = model_settings.to_json_dict()
|
||||
assert json_dict["extra_headers"] == {"Authorization": "Bearer provider-token"}
|
||||
assert json_dict["extra_query"] == {"api-key": "query-token"}
|
||||
assert json_dict["extra_body"] == {"secret": "body-token"}
|
||||
assert json_dict["extra_args"] == {"api_key": "arg-token"}
|
||||
|
||||
traceable = model_settings.to_traceable_dict()
|
||||
assert traceable["temperature"] == 0.5
|
||||
assert "extra_headers" not in traceable
|
||||
assert "extra_query" not in traceable
|
||||
assert "extra_body" not in traceable
|
||||
assert "extra_args" not in traceable
|
||||
|
||||
|
||||
def test_extra_args_resolve() -> None:
|
||||
"""Test that extra_args are properly merged in the resolve method."""
|
||||
base_settings = ModelSettings(
|
||||
temperature=0.5, extra_args={"param1": "base_value", "param2": "base_only"}
|
||||
)
|
||||
|
||||
override_settings = ModelSettings(
|
||||
top_p=0.9, extra_args={"param1": "override_value", "param3": "override_only"}
|
||||
)
|
||||
|
||||
resolved = base_settings.resolve(override_settings)
|
||||
|
||||
# Check that regular fields are properly resolved
|
||||
assert resolved.temperature == 0.5 # from base
|
||||
assert resolved.top_p == 0.9 # from override
|
||||
|
||||
# Check that extra_args are properly merged
|
||||
expected_extra_args = {
|
||||
"param1": "override_value", # override wins
|
||||
"param2": "base_only", # from base
|
||||
"param3": "override_only", # from override
|
||||
}
|
||||
assert resolved.extra_args == expected_extra_args
|
||||
|
||||
|
||||
def test_extra_args_resolve_with_none() -> None:
|
||||
"""Test that resolve works properly when one side has None extra_args."""
|
||||
# Base with extra_args, override with None
|
||||
base_settings = ModelSettings(extra_args={"param1": "value1"})
|
||||
override_settings = ModelSettings(temperature=0.8)
|
||||
|
||||
resolved = base_settings.resolve(override_settings)
|
||||
assert resolved.extra_args == {"param1": "value1"}
|
||||
assert resolved.temperature == 0.8
|
||||
|
||||
# Base with None, override with extra_args
|
||||
base_settings = ModelSettings(temperature=0.5)
|
||||
override_settings = ModelSettings(extra_args={"param2": "value2"})
|
||||
|
||||
resolved = base_settings.resolve(override_settings)
|
||||
assert resolved.extra_args == {"param2": "value2"}
|
||||
assert resolved.temperature == 0.5
|
||||
|
||||
|
||||
def test_extra_args_resolve_both_none() -> None:
|
||||
"""Test that resolve works when both sides have None extra_args."""
|
||||
base_settings = ModelSettings(temperature=0.5)
|
||||
override_settings = ModelSettings(top_p=0.9)
|
||||
|
||||
resolved = base_settings.resolve(override_settings)
|
||||
assert resolved.extra_args is None
|
||||
assert resolved.temperature == 0.5
|
||||
assert resolved.top_p == 0.9
|
||||
|
||||
|
||||
def test_pydantic_serialization() -> None:
|
||||
"""Tests whether ModelSettings can be serialized with Pydantic."""
|
||||
|
||||
# First, lets create a ModelSettings instance
|
||||
model_settings = ModelSettings(
|
||||
temperature=0.5,
|
||||
top_p=0.9,
|
||||
frequency_penalty=0.0,
|
||||
presence_penalty=0.0,
|
||||
tool_choice="auto",
|
||||
parallel_tool_calls=True,
|
||||
truncation="auto",
|
||||
max_tokens=100,
|
||||
reasoning=Reasoning(),
|
||||
metadata={"foo": "bar"},
|
||||
store=False,
|
||||
include_usage=False,
|
||||
top_logprobs=1,
|
||||
extra_query={"foo": "bar"},
|
||||
extra_body={"foo": "bar"},
|
||||
extra_headers={"foo": "bar"},
|
||||
extra_args={"custom_param": "value", "another_param": 42},
|
||||
)
|
||||
|
||||
json = to_json(model_settings)
|
||||
deserialized = TypeAdapter(ModelSettings).validate_json(json)
|
||||
|
||||
assert model_settings == deserialized
|
||||
|
||||
|
||||
def test_retry_policy_is_excluded_from_json_dict() -> None:
|
||||
"""Tests whether runtime-only retry policies are omitted from JSON serialization."""
|
||||
|
||||
model_settings = ModelSettings(
|
||||
retry=ModelRetrySettings(
|
||||
max_retries=1,
|
||||
backoff=ModelRetryBackoffSettings(initial_delay=0.1),
|
||||
policy=retry_policies.http_status([429]),
|
||||
)
|
||||
)
|
||||
|
||||
json_dict = model_settings.to_json_dict()
|
||||
assert json_dict["retry"] == {
|
||||
"max_retries": 1,
|
||||
"backoff": {
|
||||
"initial_delay": 0.1,
|
||||
"max_delay": None,
|
||||
"multiplier": None,
|
||||
"jitter": None,
|
||||
},
|
||||
}
|
||||
|
||||
verify_serialization(model_settings)
|
||||
|
||||
|
||||
def test_retry_resolve_deep_merges_backoff() -> None:
|
||||
"""Tests whether retry settings are deep-merged in resolve()."""
|
||||
|
||||
base_settings = ModelSettings(
|
||||
retry=ModelRetrySettings(
|
||||
max_retries=1,
|
||||
backoff=ModelRetryBackoffSettings(initial_delay=0.1, max_delay=1.0),
|
||||
)
|
||||
)
|
||||
override_settings = ModelSettings(
|
||||
retry=ModelRetrySettings(
|
||||
backoff=ModelRetryBackoffSettings(multiplier=3.0, jitter=False),
|
||||
policy=retry_policies.never(),
|
||||
)
|
||||
)
|
||||
|
||||
resolved = base_settings.resolve(override_settings)
|
||||
|
||||
assert resolved.retry is not None
|
||||
assert resolved.retry.max_retries == 1
|
||||
assert resolved.retry.policy is not None
|
||||
assert resolved.retry.backoff == ModelRetryBackoffSettings(
|
||||
initial_delay=0.1,
|
||||
max_delay=1.0,
|
||||
multiplier=3.0,
|
||||
jitter=False,
|
||||
)
|
||||
|
||||
|
||||
def test_retry_policy_is_omitted_from_pydantic_round_trip() -> None:
|
||||
"""Tests whether runtime-only retry policies are omitted from Pydantic serialization."""
|
||||
|
||||
model_settings = ModelSettings(
|
||||
retry=ModelRetrySettings(
|
||||
max_retries=2,
|
||||
backoff=ModelRetryBackoffSettings(initial_delay=0.5),
|
||||
policy=retry_policies.http_status([429]),
|
||||
)
|
||||
)
|
||||
|
||||
serialized = to_json(model_settings)
|
||||
deserialized = TypeAdapter(ModelSettings).validate_json(serialized)
|
||||
|
||||
assert deserialized.retry is not None
|
||||
assert deserialized.retry.max_retries == 2
|
||||
assert deserialized.retry.backoff == ModelRetryBackoffSettings(initial_delay=0.5)
|
||||
assert deserialized.retry.policy is None
|
||||
|
||||
|
||||
def test_retry_backoff_validate_python_accepts_nested_dict_input() -> None:
|
||||
"""Tests whether nested retry/backoff dict input is coerced to dataclasses."""
|
||||
|
||||
deserialized = TypeAdapter(ModelSettings).validate_python(
|
||||
{
|
||||
"retry": {
|
||||
"max_retries": 3,
|
||||
"backoff": {
|
||||
"initial_delay": 0.25,
|
||||
"max_delay": 2.0,
|
||||
"multiplier": 3.0,
|
||||
"jitter": False,
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
assert deserialized.retry is not None
|
||||
assert deserialized.retry.max_retries == 3
|
||||
assert deserialized.retry.backoff == ModelRetryBackoffSettings(
|
||||
initial_delay=0.25,
|
||||
max_delay=2.0,
|
||||
multiplier=3.0,
|
||||
jitter=False,
|
||||
)
|
||||
|
||||
|
||||
def test_retry_backoff_validate_python_preserves_falsey_values() -> None:
|
||||
"""Tests whether falsey-only retry backoff input survives validation and serialization."""
|
||||
|
||||
deserialized = TypeAdapter(ModelRetrySettings).validate_python(
|
||||
{
|
||||
"max_retries": 1,
|
||||
"backoff": {
|
||||
"jitter": False,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
assert deserialized.backoff == ModelRetryBackoffSettings(jitter=False)
|
||||
assert deserialized.to_json_dict()["backoff"] == {
|
||||
"initial_delay": None,
|
||||
"max_delay": None,
|
||||
"multiplier": None,
|
||||
"jitter": False,
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agents import (
|
||||
OpenAIAgentRegistrationConfig,
|
||||
RunConfig,
|
||||
set_default_openai_agent_registration,
|
||||
set_default_openai_harness,
|
||||
)
|
||||
from agents.models.multi_provider import MultiProvider
|
||||
from agents.models.openai_agent_registration import (
|
||||
OPENAI_HARNESS_ID_TRACE_METADATA_KEY,
|
||||
resolve_openai_agent_registration_config,
|
||||
resolve_openai_harness_id_for_model_provider,
|
||||
)
|
||||
from agents.models.openai_provider import OpenAIProvider
|
||||
from agents.run_internal.agent_runner_helpers import resolve_trace_settings
|
||||
from agents.tracing import agent_span, trace
|
||||
|
||||
|
||||
def test_agent_registration_config_precedence(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
|
||||
set_default_openai_agent_registration(
|
||||
OpenAIAgentRegistrationConfig(harness_id="default-harness")
|
||||
)
|
||||
|
||||
try:
|
||||
resolved = resolve_openai_agent_registration_config(
|
||||
OpenAIAgentRegistrationConfig(harness_id="explicit-harness")
|
||||
)
|
||||
finally:
|
||||
set_default_openai_agent_registration(None)
|
||||
|
||||
assert resolved is not None
|
||||
assert resolved.harness_id == "explicit-harness"
|
||||
|
||||
|
||||
def test_agent_registration_uses_default_before_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
|
||||
set_default_openai_agent_registration(
|
||||
OpenAIAgentRegistrationConfig(harness_id="default-harness")
|
||||
)
|
||||
|
||||
try:
|
||||
resolved = resolve_openai_agent_registration_config(None)
|
||||
finally:
|
||||
set_default_openai_agent_registration(None)
|
||||
|
||||
assert resolved is not None
|
||||
assert resolved.harness_id == "default-harness"
|
||||
|
||||
|
||||
def test_agent_registration_uses_env(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
|
||||
|
||||
resolved = resolve_openai_agent_registration_config(None)
|
||||
|
||||
assert resolved is not None
|
||||
assert resolved.harness_id == "env-harness"
|
||||
|
||||
|
||||
def test_set_default_openai_harness(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
|
||||
set_default_openai_harness("helper-harness")
|
||||
|
||||
try:
|
||||
resolved = resolve_openai_agent_registration_config(None)
|
||||
finally:
|
||||
set_default_openai_harness(None)
|
||||
|
||||
assert resolved is not None
|
||||
assert resolved.harness_id == "helper-harness"
|
||||
|
||||
|
||||
def test_agent_registration_disabled_without_config(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.delenv("OPENAI_AGENT_HARNESS_ID", raising=False)
|
||||
|
||||
assert resolve_openai_agent_registration_config(None) is None
|
||||
|
||||
|
||||
def test_agent_registration_provider_constructor_config() -> None:
|
||||
config = OpenAIAgentRegistrationConfig(harness_id="provider-harness")
|
||||
|
||||
openai_provider = OpenAIProvider(agent_registration=config)
|
||||
multi_provider = MultiProvider(openai_agent_registration=config)
|
||||
|
||||
assert openai_provider.agent_registration is not None
|
||||
assert openai_provider.agent_registration.harness_id == "provider-harness"
|
||||
assert multi_provider.openai_provider.agent_registration is not None
|
||||
assert multi_provider.openai_provider.agent_registration.harness_id == "provider-harness"
|
||||
|
||||
|
||||
def test_harness_id_resolves_private_agent_registration() -> None:
|
||||
class Provider:
|
||||
_agent_registration = OpenAIAgentRegistrationConfig(harness_id="private-harness")
|
||||
|
||||
assert resolve_openai_harness_id_for_model_provider(Provider()) == "private-harness"
|
||||
|
||||
|
||||
def test_harness_id_is_added_to_trace_metadata() -> None:
|
||||
provider = OpenAIProvider(
|
||||
agent_registration=OpenAIAgentRegistrationConfig(harness_id="provider-harness")
|
||||
)
|
||||
|
||||
_, _, _, metadata, _ = resolve_trace_settings(
|
||||
run_state=None,
|
||||
run_config=RunConfig(model_provider=provider),
|
||||
)
|
||||
|
||||
assert metadata == {OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "provider-harness"}
|
||||
|
||||
|
||||
def test_harness_id_preserves_explicit_trace_metadata() -> None:
|
||||
provider = OpenAIProvider(
|
||||
agent_registration=OpenAIAgentRegistrationConfig(harness_id="provider-harness")
|
||||
)
|
||||
|
||||
_, _, _, metadata, _ = resolve_trace_settings(
|
||||
run_state=None,
|
||||
run_config=RunConfig(
|
||||
model_provider=provider,
|
||||
trace_metadata={
|
||||
OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "explicit-harness",
|
||||
"source": "test",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
assert metadata == {
|
||||
OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "explicit-harness",
|
||||
"source": "test",
|
||||
}
|
||||
|
||||
|
||||
def test_env_harness_id_is_added_to_trace_metadata(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("OPENAI_AGENT_HARNESS_ID", "env-harness")
|
||||
|
||||
_, _, _, metadata, _ = resolve_trace_settings(
|
||||
run_state=None,
|
||||
run_config=RunConfig(),
|
||||
)
|
||||
|
||||
assert metadata == {OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "env-harness"}
|
||||
|
||||
|
||||
def test_harness_id_trace_metadata_propagates_to_spans() -> None:
|
||||
provider = OpenAIProvider(
|
||||
agent_registration=OpenAIAgentRegistrationConfig(harness_id="provider-harness")
|
||||
)
|
||||
workflow_name, trace_id, group_id, metadata, _ = resolve_trace_settings(
|
||||
run_state=None,
|
||||
run_config=RunConfig(model_provider=provider),
|
||||
)
|
||||
|
||||
with trace(
|
||||
workflow_name=workflow_name,
|
||||
trace_id=trace_id,
|
||||
group_id=group_id,
|
||||
metadata=metadata,
|
||||
):
|
||||
with agent_span(name="agent") as span:
|
||||
assert span.trace_metadata == {OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "provider-harness"}
|
||||
span_export = span.export()
|
||||
assert span_export is not None
|
||||
assert span_export["metadata"] == {
|
||||
OPENAI_HARNESS_ID_TRACE_METADATA_KEY: "provider-harness"
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
"""
|
||||
Test for Anthropic thinking blocks in conversation history.
|
||||
|
||||
This test validates the fix for issue #1704:
|
||||
- Thinking blocks are properly preserved from Anthropic responses
|
||||
- Reasoning items are stored in session but not sent back in conversation history
|
||||
- Non-reasoning models are unaffected
|
||||
- Token usage is not increased for non-reasoning scenarios
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
from openai.types.chat import ChatCompletionMessageToolCall
|
||||
from openai.types.chat.chat_completion_message_tool_call import Function
|
||||
|
||||
from agents.extensions.models.litellm_model import InternalChatCompletionMessage
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
|
||||
|
||||
def create_mock_anthropic_response_with_thinking() -> InternalChatCompletionMessage:
|
||||
"""Create a mock Anthropic response with thinking blocks (like real response)."""
|
||||
message = InternalChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="I'll check the weather in Paris for you.",
|
||||
reasoning_content="I need to call the weather function for Paris",
|
||||
thinking_blocks=[
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": "I need to call the weather function for Paris",
|
||||
"signature": "EqMDCkYIBxgCKkBAFZO8EyZwN1hiLctq0YjZnP0KeKgprr+C0PzgDv4GSggnFwrPQHIZ9A5s+paH+DrQBI1+Vnfq3mLAU5lJnoetEgzUEWx/Cv1022ieAvcaDCXdmg1XkMK0tZ8uCCIwURYAAX0uf2wFdnWt9n8whkhmy8ARQD5G2za4R8X5vTqBq8jpJ15T3c1Jcf3noKMZKooCWFVf0/W5VQqpZTgwDkqyTau7XraS+u48YlmJGSfyWMPO8snFLMZLGaGmVJgHfEI5PILhOEuX/R2cEeLuC715f51LMVuxTNzlOUV/037JV6P2ten7D66FnWU9JJMMJJov+DjMb728yQFHwHz4roBJ5ePHaaFP6mDwpqYuG/hai6pVv2TAK1IdKUui/oXrYtU+0gxb6UF2kS1bspqDuN++R8JdL7CMSU5l28pQ8TsH1TpVF4jZpsFbp1Du4rQIULFsCFFg+Edf9tPgyKZOq6xcskIjT7oylAPO37/jhdNknDq2S82PaSKtke3ViOigtM5uJfG521ZscBJQ1K3kwoI/repIdV9PatjOYdsYAQ==", # noqa: E501
|
||||
}
|
||||
],
|
||||
)
|
||||
return message
|
||||
|
||||
|
||||
def test_converter_skips_reasoning_items():
|
||||
"""
|
||||
Unit test to verify that reasoning items are skipped when converting items to messages.
|
||||
"""
|
||||
# Create test items including a reasoning item
|
||||
test_items: list[dict[str, Any]] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{
|
||||
"id": "reasoning_123",
|
||||
"type": "reasoning",
|
||||
"summary": [{"text": "User said hello", "type": "summary_text"}],
|
||||
},
|
||||
{
|
||||
"id": "msg_123",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "Hi there!"}],
|
||||
"status": "completed",
|
||||
},
|
||||
]
|
||||
|
||||
# Convert to messages
|
||||
messages = Converter.items_to_messages(test_items) # type: ignore[arg-type]
|
||||
|
||||
# Should have user message and assistant message, but no reasoning content
|
||||
assert len(messages) == 2
|
||||
assert messages[0]["role"] == "user"
|
||||
assert messages[1]["role"] == "assistant"
|
||||
|
||||
# Verify no thinking blocks in assistant message
|
||||
assistant_msg = messages[1]
|
||||
content = assistant_msg.get("content")
|
||||
if isinstance(content, list):
|
||||
for part in content:
|
||||
assert part.get("type") != "thinking"
|
||||
|
||||
|
||||
def test_reasoning_items_preserved_in_message_conversion():
|
||||
"""
|
||||
Test that reasoning content and thinking blocks are properly extracted
|
||||
from Anthropic responses and stored in reasoning items.
|
||||
"""
|
||||
# Create mock message with thinking blocks
|
||||
mock_message = create_mock_anthropic_response_with_thinking()
|
||||
|
||||
# Convert to output items
|
||||
output_items = Converter.message_to_output_items(mock_message)
|
||||
|
||||
# Should have reasoning item, message item, and tool call items
|
||||
reasoning_items = [
|
||||
item for item in output_items if hasattr(item, "type") and item.type == "reasoning"
|
||||
]
|
||||
assert len(reasoning_items) == 1
|
||||
|
||||
reasoning_item = reasoning_items[0]
|
||||
assert reasoning_item.summary[0].text == "I need to call the weather function for Paris"
|
||||
|
||||
# Verify thinking blocks are stored if we preserve them
|
||||
if (
|
||||
hasattr(reasoning_item, "content")
|
||||
and reasoning_item.content
|
||||
and len(reasoning_item.content) > 0
|
||||
):
|
||||
thinking_block = reasoning_item.content[0]
|
||||
assert thinking_block.type == "reasoning_text"
|
||||
assert thinking_block.text == "I need to call the weather function for Paris"
|
||||
|
||||
|
||||
def test_anthropic_thinking_blocks_with_tool_calls():
|
||||
"""
|
||||
Test for models with extended thinking and interleaved thinking with tool calls.
|
||||
|
||||
This test verifies the Anthropic's API's requirements for thinking blocks
|
||||
to be the first content in assistant messages when reasoning is enabled and tool
|
||||
calls are present.
|
||||
"""
|
||||
# Create a message with reasoning, thinking blocks and tool calls
|
||||
message = InternalChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="I'll check the weather for you.",
|
||||
reasoning_content="The user wants weather information, I need to call the weather function",
|
||||
thinking_blocks=[
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": (
|
||||
"The user is asking about weather. "
|
||||
"Let me use the weather tool to get this information."
|
||||
),
|
||||
"signature": "TestSignature123",
|
||||
},
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": ("We should use the city Tokyo as the city."),
|
||||
"signature": "TestSignature456",
|
||||
},
|
||||
],
|
||||
tool_calls=[
|
||||
ChatCompletionMessageToolCall(
|
||||
id="call_123",
|
||||
type="function",
|
||||
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
# Step 1: Convert message to output items
|
||||
output_items = Converter.message_to_output_items(message)
|
||||
|
||||
# Verify reasoning item exists and contains thinking blocks
|
||||
reasoning_items = [
|
||||
item for item in output_items if hasattr(item, "type") and item.type == "reasoning"
|
||||
]
|
||||
assert len(reasoning_items) == 1, "Should have exactly two reasoning items"
|
||||
|
||||
reasoning_item = reasoning_items[0]
|
||||
|
||||
# Verify thinking text is stored in content
|
||||
assert hasattr(reasoning_item, "content") and reasoning_item.content, (
|
||||
"Reasoning item should have content"
|
||||
)
|
||||
assert reasoning_item.content[0].type == "reasoning_text", (
|
||||
"Content should be reasoning_text type"
|
||||
)
|
||||
|
||||
# Verify signature is stored in encrypted_content
|
||||
assert hasattr(reasoning_item, "encrypted_content"), (
|
||||
"Reasoning item should have encrypted_content"
|
||||
)
|
||||
assert reasoning_item.encrypted_content == "TestSignature123\nTestSignature456", (
|
||||
"Signature should be preserved"
|
||||
)
|
||||
|
||||
# Verify tool calls are present
|
||||
tool_call_items = [
|
||||
item for item in output_items if hasattr(item, "type") and item.type == "function_call"
|
||||
]
|
||||
assert len(tool_call_items) == 1, "Should have exactly one tool call"
|
||||
|
||||
# Step 2: Convert output items back to messages
|
||||
# Convert items to dicts for the converter (simulating serialization/deserialization)
|
||||
items_as_dicts: list[dict[str, Any]] = []
|
||||
for item in output_items:
|
||||
if hasattr(item, "model_dump"):
|
||||
items_as_dicts.append(item.model_dump())
|
||||
else:
|
||||
items_as_dicts.append(cast(dict[str, Any], item))
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
items_as_dicts, # type: ignore[arg-type]
|
||||
model="anthropic/claude-4-opus",
|
||||
preserve_thinking_blocks=True,
|
||||
)
|
||||
|
||||
# Find the assistant message with tool calls
|
||||
assistant_messages = [
|
||||
msg for msg in messages if msg.get("role") == "assistant" and msg.get("tool_calls")
|
||||
]
|
||||
assert len(assistant_messages) == 1, "Should have exactly one assistant message with tool calls"
|
||||
|
||||
assistant_msg = assistant_messages[0]
|
||||
|
||||
# Content must start with thinking blocks, not text
|
||||
content = assistant_msg.get("content")
|
||||
assert content is not None, "Assistant message should have content"
|
||||
|
||||
assert isinstance(content, list) and len(content) > 0, (
|
||||
"Assistant message content should be a non-empty list"
|
||||
)
|
||||
|
||||
first_content = content[0]
|
||||
assert first_content.get("type") == "thinking", (
|
||||
f"First content must be 'thinking' type for Anthropic compatibility, "
|
||||
f"but got '{first_content.get('type')}'"
|
||||
)
|
||||
expected_thinking = (
|
||||
"The user is asking about weather. Let me use the weather tool to get this information."
|
||||
)
|
||||
assert first_content.get("thinking") == expected_thinking, (
|
||||
"Thinking content should be preserved"
|
||||
)
|
||||
# Signature should also be preserved
|
||||
assert first_content.get("signature") == "TestSignature123", (
|
||||
"Signature should be preserved in thinking block"
|
||||
)
|
||||
|
||||
second_content = content[1]
|
||||
assert second_content.get("type") == "thinking", (
|
||||
f"Second content must be 'thinking' type for Anthropic compatibility, "
|
||||
f"but got '{second_content.get('type')}'"
|
||||
)
|
||||
expected_thinking = "We should use the city Tokyo as the city."
|
||||
assert second_content.get("thinking") == expected_thinking, (
|
||||
"Thinking content should be preserved"
|
||||
)
|
||||
# Signature should also be preserved
|
||||
assert second_content.get("signature") == "TestSignature456", (
|
||||
"Signature should be preserved in thinking block"
|
||||
)
|
||||
|
||||
last_content = content[2]
|
||||
assert last_content.get("type") == "text", (
|
||||
f"First content must be 'text' type but got '{last_content.get('type')}'"
|
||||
)
|
||||
expected_text = "I'll check the weather for you."
|
||||
assert last_content.get("text") == expected_text, "Content text should be preserved"
|
||||
|
||||
# Verify tool calls are preserved
|
||||
tool_calls = assistant_msg.get("tool_calls", [])
|
||||
assert len(cast(list[Any], tool_calls)) == 1, "Tool calls should be preserved"
|
||||
assert cast(list[Any], tool_calls)[0]["function"]["name"] == "get_weather"
|
||||
|
||||
|
||||
def test_items_to_messages_preserves_positional_bool_arguments():
|
||||
"""
|
||||
Preserve positional compatibility for the released items_to_messages signature.
|
||||
"""
|
||||
message = InternalChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="I'll check the weather for you.",
|
||||
reasoning_content="The user wants weather information, I need to call the weather function",
|
||||
thinking_blocks=[
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": (
|
||||
"The user is asking about weather. "
|
||||
"Let me use the weather tool to get this information."
|
||||
),
|
||||
"signature": "TestSignature123",
|
||||
}
|
||||
],
|
||||
tool_calls=[
|
||||
ChatCompletionMessageToolCall(
|
||||
id="call_123",
|
||||
type="function",
|
||||
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
output_items = Converter.message_to_output_items(message)
|
||||
items_as_dicts: list[dict[str, Any]] = []
|
||||
for item in output_items:
|
||||
if hasattr(item, "model_dump"):
|
||||
items_as_dicts.append(item.model_dump())
|
||||
else:
|
||||
items_as_dicts.append(cast(dict[str, Any], item))
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
items_as_dicts, # type: ignore[arg-type]
|
||||
"anthropic/claude-4-opus",
|
||||
True,
|
||||
True,
|
||||
)
|
||||
|
||||
assistant_messages = [
|
||||
msg for msg in messages if msg.get("role") == "assistant" and msg.get("tool_calls")
|
||||
]
|
||||
assert len(assistant_messages) == 1, "Should have exactly one assistant message with tool calls"
|
||||
|
||||
assistant_msg = assistant_messages[0]
|
||||
content = assistant_msg.get("content")
|
||||
assert isinstance(content, list) and len(content) > 0, (
|
||||
"Positional bool arguments should still preserve thinking blocks"
|
||||
)
|
||||
assert content[0].get("type") == "thinking", (
|
||||
"The third positional argument must continue to map to preserve_thinking_blocks"
|
||||
)
|
||||
|
||||
|
||||
def test_anthropic_thinking_blocks_without_tool_calls():
|
||||
"""
|
||||
Test for models with extended thinking WITHOUT tool calls.
|
||||
|
||||
This test verifies that thinking blocks are properly attached to assistant
|
||||
messages even when there are no tool calls (fixes issue #2195).
|
||||
"""
|
||||
# Create a message with reasoning and thinking blocks but NO tool calls
|
||||
message = InternalChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="The weather in Paris is sunny with a temperature of 22°C.",
|
||||
reasoning_content="The user wants to know about the weather in Paris.",
|
||||
thinking_blocks=[
|
||||
{
|
||||
"type": "thinking",
|
||||
"thinking": "Let me think about the weather in Paris.",
|
||||
"signature": "TestSignatureNoTools123",
|
||||
}
|
||||
],
|
||||
tool_calls=None, # No tool calls
|
||||
)
|
||||
|
||||
# Step 1: Convert message to output items
|
||||
output_items = Converter.message_to_output_items(message)
|
||||
|
||||
# Verify reasoning item exists and contains thinking blocks
|
||||
reasoning_items = [
|
||||
item for item in output_items if hasattr(item, "type") and item.type == "reasoning"
|
||||
]
|
||||
assert len(reasoning_items) == 1, "Should have exactly one reasoning item"
|
||||
|
||||
reasoning_item = reasoning_items[0]
|
||||
|
||||
# Verify thinking text is stored in content
|
||||
assert hasattr(reasoning_item, "content") and reasoning_item.content, (
|
||||
"Reasoning item should have content"
|
||||
)
|
||||
assert reasoning_item.content[0].type == "reasoning_text", (
|
||||
"Content should be reasoning_text type"
|
||||
)
|
||||
assert reasoning_item.content[0].text == "Let me think about the weather in Paris.", (
|
||||
"Thinking text should be preserved"
|
||||
)
|
||||
|
||||
# Verify signature is stored in encrypted_content
|
||||
assert hasattr(reasoning_item, "encrypted_content"), (
|
||||
"Reasoning item should have encrypted_content"
|
||||
)
|
||||
assert reasoning_item.encrypted_content == "TestSignatureNoTools123", (
|
||||
"Signature should be preserved"
|
||||
)
|
||||
|
||||
# Verify message item exists
|
||||
message_items = [
|
||||
item for item in output_items if hasattr(item, "type") and item.type == "message"
|
||||
]
|
||||
assert len(message_items) == 1, "Should have exactly one message item"
|
||||
|
||||
# Step 2: Convert output items back to messages with preserve_thinking_blocks=True
|
||||
items_as_dicts: list[dict[str, Any]] = []
|
||||
for item in output_items:
|
||||
if hasattr(item, "model_dump"):
|
||||
items_as_dicts.append(item.model_dump())
|
||||
else:
|
||||
items_as_dicts.append(cast(dict[str, Any], item))
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
items_as_dicts, # type: ignore[arg-type]
|
||||
model="anthropic/claude-4-opus",
|
||||
preserve_thinking_blocks=True,
|
||||
)
|
||||
|
||||
# Should have one assistant message
|
||||
assistant_messages = [msg for msg in messages if msg.get("role") == "assistant"]
|
||||
assert len(assistant_messages) == 1, "Should have exactly one assistant message"
|
||||
|
||||
assistant_msg = assistant_messages[0]
|
||||
|
||||
# Content must start with thinking blocks even WITHOUT tool calls
|
||||
content = assistant_msg.get("content")
|
||||
assert content is not None, "Assistant message should have content"
|
||||
assert isinstance(content, list), (
|
||||
f"Assistant message content should be a list when thinking blocks are present, "
|
||||
f"but got {type(content)}"
|
||||
)
|
||||
assert len(content) >= 2, (
|
||||
f"Assistant message should have at least 2 content items "
|
||||
f"(thinking + text), got {len(content)}"
|
||||
)
|
||||
|
||||
# First content should be thinking block
|
||||
first_content = content[0]
|
||||
assert first_content.get("type") == "thinking", (
|
||||
f"First content must be 'thinking' type for Anthropic compatibility, "
|
||||
f"but got '{first_content.get('type')}'"
|
||||
)
|
||||
assert first_content.get("thinking") == "Let me think about the weather in Paris.", (
|
||||
"Thinking content should be preserved"
|
||||
)
|
||||
assert first_content.get("signature") == "TestSignatureNoTools123", (
|
||||
"Signature should be preserved in thinking block"
|
||||
)
|
||||
|
||||
# Second content should be text
|
||||
second_content = content[1]
|
||||
assert second_content.get("type") == "text", (
|
||||
f"Second content must be 'text' type, but got '{second_content.get('type')}'"
|
||||
)
|
||||
assert (
|
||||
second_content.get("text") == "The weather in Paris is sunny with a temperature of 22°C."
|
||||
), "Text content should be preserved"
|
||||
@@ -0,0 +1,904 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
import types as pytypes
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
import pytest
|
||||
from openai.types.chat import (
|
||||
ChatCompletion,
|
||||
ChatCompletionChunk,
|
||||
ChatCompletionMessage,
|
||||
ChatCompletionMessageFunctionToolCall,
|
||||
)
|
||||
from openai.types.chat.chat_completion import Choice
|
||||
from openai.types.chat.chat_completion_chunk import ChoiceDelta
|
||||
from openai.types.completion_usage import CompletionUsage, PromptTokensDetails
|
||||
from openai.types.responses import Response, ResponseCompletedEvent, ResponseOutputMessage
|
||||
from openai.types.responses.response_error_event import ResponseErrorEvent
|
||||
from openai.types.responses.response_failed_event import ResponseFailedEvent
|
||||
from openai.types.responses.response_incomplete_event import ResponseIncompleteEvent
|
||||
from openai.types.responses.response_output_text import ResponseOutputText
|
||||
from openai.types.responses.response_usage import (
|
||||
InputTokensDetails,
|
||||
OutputTokensDetails,
|
||||
ResponseUsage,
|
||||
)
|
||||
from pydantic import BaseModel
|
||||
|
||||
from agents import (
|
||||
Agent,
|
||||
Handoff,
|
||||
ModelBehaviorError,
|
||||
ModelSettings,
|
||||
ModelTracing,
|
||||
Tool,
|
||||
TResponseInputItem,
|
||||
__version__,
|
||||
)
|
||||
from agents.exceptions import UserError
|
||||
from agents.models.chatcmpl_helpers import HEADERS_OVERRIDE
|
||||
from agents.models.fake_id import FAKE_RESPONSES_ID
|
||||
|
||||
|
||||
class FakeAnyLLMProvider:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
supports_responses: bool,
|
||||
chat_response: Any | None = None,
|
||||
responses_response: Any | None = None,
|
||||
) -> None:
|
||||
self.SUPPORTS_RESPONSES = supports_responses
|
||||
self.chat_response = chat_response
|
||||
self.responses_response = responses_response
|
||||
self.chat_calls: list[dict[str, Any]] = []
|
||||
self.responses_calls: list[dict[str, Any]] = []
|
||||
self.private_responses_calls: list[dict[str, Any]] = []
|
||||
|
||||
async def acompletion(self, **kwargs: Any) -> Any:
|
||||
self.chat_calls.append(kwargs)
|
||||
return self.chat_response
|
||||
|
||||
async def aresponses(self, **kwargs: Any) -> Any:
|
||||
self.responses_calls.append(kwargs)
|
||||
return self.responses_response
|
||||
|
||||
async def _aresponses(self, params: Any, **kwargs: Any) -> Any:
|
||||
self.private_responses_calls.append({"params": params, "kwargs": kwargs})
|
||||
return self.responses_response
|
||||
|
||||
|
||||
def _import_any_llm_module(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
provider: FakeAnyLLMProvider,
|
||||
) -> tuple[Any, list[dict[str, Any]]]:
|
||||
create_calls: list[dict[str, Any]] = []
|
||||
|
||||
class FakeAnyLLMFactory:
|
||||
@staticmethod
|
||||
def create(provider_name: str, api_key: str | None = None, api_base: str | None = None):
|
||||
create_calls.append(
|
||||
{
|
||||
"provider_name": provider_name,
|
||||
"api_key": api_key,
|
||||
"api_base": api_base,
|
||||
}
|
||||
)
|
||||
return provider
|
||||
|
||||
fake_any_llm: Any = pytypes.ModuleType("any_llm")
|
||||
fake_any_llm.AnyLLM = FakeAnyLLMFactory
|
||||
|
||||
sys.modules.pop("agents.extensions.models.any_llm_model", None)
|
||||
monkeypatch.setitem(sys.modules, "any_llm", fake_any_llm)
|
||||
|
||||
module = importlib.import_module("agents.extensions.models.any_llm_model")
|
||||
monkeypatch.setattr(module, "AnyLLM", FakeAnyLLMFactory, raising=True)
|
||||
return module, create_calls
|
||||
|
||||
|
||||
def _chat_completion(text: str) -> ChatCompletion:
|
||||
return ChatCompletion(
|
||||
id="chatcmpl_123",
|
||||
created=0,
|
||||
model="fake-model",
|
||||
object="chat.completion",
|
||||
choices=[
|
||||
Choice(
|
||||
index=0,
|
||||
finish_reason="stop",
|
||||
message=ChatCompletionMessage(role="assistant", content=text),
|
||||
)
|
||||
],
|
||||
usage=CompletionUsage(
|
||||
completion_tokens=5,
|
||||
prompt_tokens=7,
|
||||
total_tokens=12,
|
||||
prompt_tokens_details=PromptTokensDetails.model_validate(
|
||||
{"cached_tokens": 2, "cache_write_tokens": 4}
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _responses_output(text: str) -> list[Any]:
|
||||
return [
|
||||
ResponseOutputMessage(
|
||||
id="msg_123",
|
||||
role="assistant",
|
||||
status="completed",
|
||||
type="message",
|
||||
content=[
|
||||
ResponseOutputText(
|
||||
text=text,
|
||||
type="output_text",
|
||||
annotations=[],
|
||||
logprobs=[],
|
||||
)
|
||||
],
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _response(text: str, response_id: str = "resp_123") -> Response:
|
||||
return Response(
|
||||
id=response_id,
|
||||
created_at=123,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=_responses_output(text),
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
usage=ResponseUsage(
|
||||
input_tokens=11,
|
||||
output_tokens=13,
|
||||
total_tokens=24,
|
||||
input_tokens_details=InputTokensDetails.model_validate(
|
||||
{"cache_write_tokens": 0, "cached_tokens": 0}
|
||||
),
|
||||
output_tokens_details=OutputTokensDetails(reasoning_tokens=0),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _chat_completion_with_tool_call(*, thought_signature: str) -> ChatCompletion:
|
||||
return ChatCompletion(
|
||||
id="chatcmpl_tool_123",
|
||||
created=0,
|
||||
model="fake-model",
|
||||
object="chat.completion",
|
||||
choices=[
|
||||
Choice(
|
||||
index=0,
|
||||
finish_reason="tool_calls",
|
||||
message=ChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="Calling a tool.",
|
||||
tool_calls=[
|
||||
ChatCompletionMessageFunctionToolCall.model_validate(
|
||||
{
|
||||
"id": "call_123",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_weather",
|
||||
"arguments": '{"city":"Paris"}',
|
||||
},
|
||||
"extra_content": {
|
||||
"google": {"thought_signature": thought_signature}
|
||||
},
|
||||
}
|
||||
)
|
||||
],
|
||||
),
|
||||
)
|
||||
],
|
||||
usage=CompletionUsage(
|
||||
completion_tokens=5,
|
||||
prompt_tokens=7,
|
||||
total_tokens=12,
|
||||
prompt_tokens_details=PromptTokensDetails(cached_tokens=0),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class GenericChatCompletionPayload(BaseModel):
|
||||
id: str
|
||||
created: int
|
||||
model: str
|
||||
object: str
|
||||
choices: list[Any]
|
||||
usage: Any
|
||||
|
||||
|
||||
async def _empty_chat_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
if False:
|
||||
yield ChatCompletionChunk(
|
||||
id="chunk_123",
|
||||
created=0,
|
||||
model="fake-model",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason=None)],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("override_ua", [None, "test_user_agent"])
|
||||
async def test_user_agent_header_any_llm_chat(override_ua: str | None, monkeypatch) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
||||
expected_ua = override_ua or f"Agents/Python {__version__}"
|
||||
|
||||
if override_ua is not None:
|
||||
token = HEADERS_OVERRIDE.set({"User-Agent": override_ua})
|
||||
else:
|
||||
token = None
|
||||
try:
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
finally:
|
||||
if token is not None:
|
||||
HEADERS_OVERRIDE.reset(token)
|
||||
|
||||
assert provider.chat_calls[0]["extra_headers"]["User-Agent"] == expected_ua
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_chat_path_is_used_when_responses_are_unsupported(monkeypatch) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
||||
module, create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini", api_key="router-key")
|
||||
response = await model.get_response(
|
||||
system_instructions="You are terse.",
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id="resp_prev",
|
||||
conversation_id="conv_123",
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
assert create_calls == [
|
||||
{
|
||||
"provider_name": "openrouter",
|
||||
"api_key": "router-key",
|
||||
"api_base": None,
|
||||
}
|
||||
]
|
||||
assert len(provider.chat_calls) == 1
|
||||
assert provider.responses_calls == []
|
||||
assert provider.chat_calls[0]["model"] == "openai/gpt-5.4-mini"
|
||||
assert response.response_id is None
|
||||
assert response.output[0].content[0].text == "Hello"
|
||||
assert response.usage.input_tokens_details.cached_tokens == 2
|
||||
assert getattr(response.usage.input_tokens_details, "cache_write_tokens", None) == 4
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"chat_response",
|
||||
[
|
||||
pytest.param(_chat_completion("Hello").model_dump(), id="dict"),
|
||||
pytest.param(
|
||||
GenericChatCompletionPayload.model_validate(_chat_completion("Hello").model_dump()),
|
||||
id="basemodel",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_any_llm_chat_path_normalizes_non_stream_payloads(
|
||||
monkeypatch,
|
||||
chat_response: Any,
|
||||
) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=chat_response)
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
||||
response = await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
assert response.response_id is None
|
||||
assert response.output[0].content[0].text == "Hello"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_chat_path_preserves_gemini_tool_call_metadata(monkeypatch) -> None:
|
||||
provider = FakeAnyLLMProvider(
|
||||
supports_responses=False,
|
||||
chat_response=_chat_completion_with_tool_call(thought_signature="sig_123"),
|
||||
)
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="gemini/gemini-2.0-flash")
|
||||
response = await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
function_calls = [
|
||||
item for item in response.output if getattr(item, "type", None) == "function_call"
|
||||
]
|
||||
assert len(function_calls) == 1
|
||||
provider_data = function_calls[0].model_dump()["provider_data"]
|
||||
assert provider_data["model"] == "gemini/gemini-2.0-flash"
|
||||
assert provider_data["response_id"] == "chatcmpl_tool_123"
|
||||
assert provider_data["thought_signature"] == "sig_123"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_responses_path_is_used_when_supported(monkeypatch) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
|
||||
module, create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="gpt-5.4-mini", api_key="openai-key")
|
||||
response = await model.get_response(
|
||||
system_instructions="You are terse.",
|
||||
input="hi",
|
||||
model_settings=ModelSettings(store=True),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id="resp_prev",
|
||||
conversation_id="conv_123",
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
assert create_calls == [
|
||||
{
|
||||
"provider_name": "openai",
|
||||
"api_key": "openai-key",
|
||||
"api_base": None,
|
||||
}
|
||||
]
|
||||
assert provider.chat_calls == []
|
||||
assert provider.responses_calls == []
|
||||
assert len(provider.private_responses_calls) == 1
|
||||
params = provider.private_responses_calls[0]["params"]
|
||||
kwargs = provider.private_responses_calls[0]["kwargs"]
|
||||
assert params.model == "gpt-5.4-mini"
|
||||
assert params.previous_response_id == "resp_prev"
|
||||
assert params.conversation == "conv_123"
|
||||
assert kwargs["extra_headers"]["User-Agent"] == f"Agents/Python {__version__}"
|
||||
assert response.response_id == "resp_123"
|
||||
assert response.output[0].content[0].text == "Hello"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_can_force_chat_completions_when_responses_are_supported(monkeypatch) -> None:
|
||||
provider = FakeAnyLLMProvider(
|
||||
supports_responses=True,
|
||||
chat_response=_chat_completion("Hello from chat"),
|
||||
responses_response=_response("Hello from responses"),
|
||||
)
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openai/gpt-4.1-mini", api="chat_completions")
|
||||
response = await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id="resp_prev",
|
||||
conversation_id="conv_123",
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
assert len(provider.chat_calls) == 1
|
||||
assert provider.responses_calls == []
|
||||
assert response.response_id is None
|
||||
assert response.output[0].content[0].text == "Hello from chat"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_forced_responses_errors_when_provider_does_not_support_it(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_chat_completion("Hello"))
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openrouter/openai/gpt-4.1-mini", api="responses")
|
||||
with pytest.raises(UserError, match="does not support the Responses API"):
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_stream_uses_chat_handler_when_responses_are_unsupported(monkeypatch) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=False, chat_response=_empty_chat_stream())
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
completed = ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=_response("Hello from stream"),
|
||||
sequence_number=1,
|
||||
)
|
||||
|
||||
async def fake_handle_stream(response, stream, model=None):
|
||||
assert model == "openrouter/openai/gpt-5.4-mini"
|
||||
async for _chunk in stream:
|
||||
pass
|
||||
yield completed
|
||||
|
||||
monkeypatch.setattr(module.ChatCmplStreamHandler, "handle_stream", fake_handle_stream)
|
||||
|
||||
model = AnyLLMModel(model="openrouter/openai/gpt-5.4-mini")
|
||||
events = [
|
||||
event
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
]
|
||||
|
||||
assert [event.type for event in events] == ["response.completed"]
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_stream_passthrough_uses_responses_when_supported(monkeypatch) -> None:
|
||||
async def response_stream() -> AsyncIterator[ResponseCompletedEvent]:
|
||||
yield ResponseCompletedEvent(
|
||||
type="response.completed",
|
||||
response=_response("Hello from responses stream"),
|
||||
sequence_number=1,
|
||||
)
|
||||
|
||||
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
||||
events = [
|
||||
event
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id="resp_prev",
|
||||
conversation_id="conv_123",
|
||||
prompt=None,
|
||||
)
|
||||
]
|
||||
|
||||
assert [event.type for event in events] == ["response.completed"]
|
||||
assert provider.responses_calls == []
|
||||
assert provider.private_responses_calls[0]["params"].previous_response_id == "resp_prev"
|
||||
assert provider.private_responses_calls[0]["params"].conversation == "conv_123"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("terminal_event_type", "terminal_event_cls"),
|
||||
[
|
||||
("response.incomplete", ResponseIncompleteEvent),
|
||||
("response.failed", ResponseFailedEvent),
|
||||
],
|
||||
)
|
||||
async def test_any_llm_responses_stream_rejects_failed_terminal_events(
|
||||
monkeypatch,
|
||||
terminal_event_type: str,
|
||||
terminal_event_cls: type[Any],
|
||||
) -> None:
|
||||
async def response_stream() -> AsyncIterator[Any]:
|
||||
yield terminal_event_cls(
|
||||
type=terminal_event_type,
|
||||
response=_response("partial", response_id="resp-terminal"),
|
||||
sequence_number=1,
|
||||
)
|
||||
|
||||
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
||||
events = []
|
||||
with pytest.raises(ModelBehaviorError, match=terminal_event_type):
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
events.append(event)
|
||||
|
||||
assert len(events) == 1
|
||||
assert events[0].type == terminal_event_type
|
||||
assert events[0].response.id == "resp-terminal"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_responses_stream_rejects_error_event(monkeypatch) -> None:
|
||||
async def response_stream() -> AsyncIterator[ResponseErrorEvent]:
|
||||
yield ResponseErrorEvent(
|
||||
type="error",
|
||||
code="invalid_request_error",
|
||||
message="bad request",
|
||||
param=None,
|
||||
sequence_number=1,
|
||||
)
|
||||
|
||||
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=response_stream())
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
||||
events = []
|
||||
with pytest.raises(ModelBehaviorError, match="invalid_request_error"):
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
events.append(event)
|
||||
|
||||
assert len(events) == 1
|
||||
assert events[0].type == "error"
|
||||
assert events[0].code == "invalid_request_error"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_responses_path_passes_transport_kwargs_via_private_provider_api(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(
|
||||
extra_headers={"X-Test-Header": "test"},
|
||||
extra_query={"trace": "1"},
|
||||
extra_body={"foo": "bar"},
|
||||
),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
assert provider.responses_calls == []
|
||||
assert len(provider.private_responses_calls) == 1
|
||||
call = provider.private_responses_calls[0]
|
||||
assert call["kwargs"]["extra_headers"]["X-Test-Header"] == "test"
|
||||
assert call["kwargs"]["extra_query"] == {"trace": "1"}
|
||||
assert call["kwargs"]["extra_body"] == {"foo": "bar"}
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_prompt_requests_fail_fast(monkeypatch) -> None:
|
||||
provider = FakeAnyLLMProvider(supports_responses=True, responses_response=_response("Hello"))
|
||||
module, _create_calls = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
||||
with pytest.raises(Exception, match="prompt-managed requests"):
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt={"id": "pmpt_123"},
|
||||
)
|
||||
|
||||
|
||||
def test_any_llm_responses_input_sanitizer_strips_none_fields_from_reasoning_items() -> None:
|
||||
pytest.importorskip(
|
||||
"any_llm",
|
||||
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
||||
)
|
||||
from agents.extensions.models.any_llm_model import AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="openai/gpt-5.4-mini")
|
||||
raw_input = [
|
||||
{
|
||||
"id": "rid1",
|
||||
"summary": [{"text": "why", "type": "summary_text"}],
|
||||
"type": "reasoning",
|
||||
"content": [{"type": "reasoning_text", "text": "thinking"}],
|
||||
"status": None,
|
||||
"encrypted_content": None,
|
||||
}
|
||||
]
|
||||
|
||||
cleaned = model._sanitize_any_llm_responses_input(raw_input)
|
||||
|
||||
assert cleaned == [
|
||||
{
|
||||
"id": "rid1",
|
||||
"summary": [{"text": "why", "type": "summary_text"}],
|
||||
"type": "reasoning",
|
||||
"content": [{"type": "reasoning_text", "text": "thinking"}],
|
||||
}
|
||||
]
|
||||
|
||||
ResponsesParams = importlib.import_module("any_llm.types.responses").ResponsesParams
|
||||
params = ResponsesParams(model="dummy", input=cleaned)
|
||||
assert isinstance(params.input, list)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_any_llm_responses_path_sanitizes_replayed_items_before_validation() -> None:
|
||||
pytest.importorskip(
|
||||
"any_llm",
|
||||
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
||||
)
|
||||
from agents.extensions.models.any_llm_model import AnyLLMModel
|
||||
|
||||
class ValidatingProvider:
|
||||
SUPPORTS_RESPONSES = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.private_responses_calls: list[dict[str, Any]] = []
|
||||
|
||||
async def aresponses(self, **kwargs: Any) -> Any:
|
||||
raise AssertionError("public aresponses path should not be used in this test")
|
||||
|
||||
async def _aresponses(self, params: Any, **kwargs: Any) -> Response:
|
||||
self.private_responses_calls.append({"params": params, "kwargs": kwargs})
|
||||
return _response("Hello from sanitized replay")
|
||||
|
||||
class TestAnyLLMModel(AnyLLMModel):
|
||||
def __init__(self, provider: ValidatingProvider) -> None:
|
||||
super().__init__(model="openai/gpt-5.4-mini", api="responses")
|
||||
self._provider = provider
|
||||
|
||||
def _get_provider(self) -> Any:
|
||||
return self._provider
|
||||
|
||||
provider = ValidatingProvider()
|
||||
model = TestAnyLLMModel(provider)
|
||||
tools: list[Tool] = []
|
||||
handoffs: list[Handoff[Any, Agent[Any]]] = []
|
||||
stream_flag: Literal[False] = False
|
||||
|
||||
replay_input = cast(
|
||||
list[TResponseInputItem],
|
||||
[
|
||||
{"role": "user", "content": "What's the weather in Tokyo?"},
|
||||
{
|
||||
"id": FAKE_RESPONSES_ID,
|
||||
"summary": [
|
||||
{"text": "I should call the weather tool first.", "type": "summary_text"}
|
||||
],
|
||||
"type": "reasoning",
|
||||
"content": [{"type": "reasoning_text", "text": "thinking"}],
|
||||
"status": None,
|
||||
"provider_data": {"model": "anthropic/fake-responses-model"},
|
||||
},
|
||||
{
|
||||
"id": FAKE_RESPONSES_ID,
|
||||
"arguments": '{"city": "Tokyo"}',
|
||||
"call_id": "call_weather_123",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"status": None,
|
||||
"provider_data": {"model": "anthropic/fake-responses-model"},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_123",
|
||||
"output": "The weather in Tokyo is sunny and 22°C.",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
response = await model._fetch_responses_response(
|
||||
system_instructions=None,
|
||||
input=replay_input,
|
||||
model_settings=ModelSettings(),
|
||||
tools=tools,
|
||||
output_schema=None,
|
||||
handoffs=handoffs,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
stream=stream_flag,
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
assert response.id == "resp_123"
|
||||
assert len(provider.private_responses_calls) == 1
|
||||
params = provider.private_responses_calls[0]["params"]
|
||||
assert params.input == [
|
||||
{"role": "user", "content": "What's the weather in Tokyo?"},
|
||||
{
|
||||
"arguments": '{"city": "Tokyo"}',
|
||||
"call_id": "call_weather_123",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_123",
|
||||
"output": "The weather in Tokyo is sunny and 22°C.",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_any_llm_provider_passes_api_override() -> None:
|
||||
pytest.importorskip(
|
||||
"any_llm",
|
||||
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
||||
)
|
||||
from agents.extensions.models.any_llm_model import AnyLLMModel
|
||||
from agents.extensions.models.any_llm_provider import AnyLLMProvider
|
||||
|
||||
provider = AnyLLMProvider(api="chat_completions")
|
||||
model = provider.get_model("openai/gpt-4.1-mini")
|
||||
|
||||
assert isinstance(model, AnyLLMModel)
|
||||
assert model.api == "chat_completions"
|
||||
|
||||
|
||||
def test_any_llm_reasoning_objects_prefer_content_attributes_over_iterable_pairs() -> None:
|
||||
pytest.importorskip(
|
||||
"any_llm",
|
||||
reason="`any-llm-sdk` is only available when the optional dependency is installed.",
|
||||
)
|
||||
from any_llm.types.completion import Reasoning
|
||||
|
||||
from agents.extensions.models.any_llm_model import _extract_any_llm_reasoning_text
|
||||
|
||||
delta = pytypes.SimpleNamespace(reasoning=Reasoning(content="用户"))
|
||||
|
||||
assert _extract_any_llm_reasoning_text(delta) == "用户"
|
||||
|
||||
|
||||
def test_any_llm_split_does_not_duplicate_content_or_thinking(monkeypatch) -> None:
|
||||
"""Splitting multi-tool assistant messages must not duplicate text/thinking blocks.
|
||||
|
||||
Anthropic's extended thinking API rejects requests that include the same signed
|
||||
thinking block more than once, and duplicated assistant text corrupts conversation
|
||||
history. Only the first split should retain content, thinking_blocks, and
|
||||
reasoning_content; subsequent splits should carry the tool_call alone.
|
||||
"""
|
||||
provider = FakeAnyLLMProvider(supports_responses=False)
|
||||
module, _ = _import_any_llm_module(monkeypatch, provider)
|
||||
AnyLLMModel = module.AnyLLMModel
|
||||
|
||||
model = AnyLLMModel(model="anthropic/claude-3-5-sonnet")
|
||||
messages: list[Any] = [
|
||||
{"role": "user", "content": "Search both"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Looking up both queries.",
|
||||
"thinking_blocks": [{"type": "thinking", "thinking": "plan", "signature": "sig_abc"}],
|
||||
"reasoning_content": "internal plan",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "s", "arguments": "{}"},
|
||||
},
|
||||
{
|
||||
"id": "call_2",
|
||||
"type": "function",
|
||||
"function": {"name": "s", "arguments": "{}"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "ok1"},
|
||||
{"role": "tool", "tool_call_id": "call_2", "content": "ok2"},
|
||||
]
|
||||
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
assistants = [m for m in result if m.get("role") == "assistant"]
|
||||
assert len(assistants) == 2
|
||||
# First split keeps the shared fields.
|
||||
assert assistants[0].get("content") == "Looking up both queries."
|
||||
assert "thinking_blocks" in assistants[0]
|
||||
assert "reasoning_content" in assistants[0]
|
||||
# Second split must NOT duplicate them.
|
||||
assert "content" not in assistants[1]
|
||||
assert "thinking_blocks" not in assistants[1]
|
||||
assert "reasoning_content" not in assistants[1]
|
||||
# Tool calls are still split one-per-message.
|
||||
assert assistants[0]["tool_calls"][0]["id"] == "call_1"
|
||||
assert assistants[1]["tool_calls"][0]["id"] == "call_2"
|
||||
@@ -0,0 +1,361 @@
|
||||
from typing import Any
|
||||
|
||||
import litellm
|
||||
import pytest
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionMessageToolCall,
|
||||
Choices,
|
||||
Function,
|
||||
Message,
|
||||
ModelResponse,
|
||||
Usage,
|
||||
)
|
||||
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
from agents.models.interface import ModelTracing
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_deepseek_reasoning_content_preserved_in_tool_calls(monkeypatch):
|
||||
"""
|
||||
Ensure DeepSeek reasoning_content is preserved when converting items to messages.
|
||||
|
||||
DeepSeek requires reasoning_content field in assistant messages with tool_calls.
|
||||
This test verifies that reasoning content from reasoning items is correctly
|
||||
extracted and added to assistant messages during conversion.
|
||||
"""
|
||||
# Capture the messages sent to the model
|
||||
captured_calls: list[dict[str, Any]] = []
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured_calls.append({"model": model, "messages": messages, **kwargs})
|
||||
|
||||
# First call: model returns reasoning_content + tool_call
|
||||
if len(captured_calls) == 1:
|
||||
tool_call = ChatCompletionMessageToolCall(
|
||||
id="call_123",
|
||||
type="function",
|
||||
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
|
||||
)
|
||||
msg = Message(
|
||||
role="assistant",
|
||||
content=None,
|
||||
tool_calls=[tool_call],
|
||||
)
|
||||
# DeepSeek adds reasoning_content to the message
|
||||
msg.reasoning_content = "Let me think about getting the weather for Tokyo..."
|
||||
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
|
||||
|
||||
# Second call: model returns final response
|
||||
msg = Message(role="assistant", content="The weather in Tokyo is sunny.")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
model = LitellmModel(model="deepseek/deepseek-reasoner")
|
||||
|
||||
# First call: get the tool call response
|
||||
first_response = await model.get_response(
|
||||
system_instructions="You are a helpful assistant.",
|
||||
input="What's the weather in Tokyo?",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[], # We'll simulate the tool response manually
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
)
|
||||
|
||||
assert len(first_response.output) >= 1
|
||||
|
||||
input_items: list[Any] = []
|
||||
input_items.append({"role": "user", "content": "What's the weather in Tokyo?"})
|
||||
|
||||
for item in first_response.output:
|
||||
if hasattr(item, "model_dump"):
|
||||
input_items.append(item.model_dump())
|
||||
else:
|
||||
input_items.append(item)
|
||||
|
||||
input_items.append(
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_123",
|
||||
"output": "The weather in Tokyo is sunny.",
|
||||
}
|
||||
)
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
input_items,
|
||||
model="deepseek/deepseek-reasoner",
|
||||
)
|
||||
|
||||
assistant_messages_with_tool_calls = [
|
||||
m
|
||||
for m in messages
|
||||
if isinstance(m, dict) and m.get("role") == "assistant" and m.get("tool_calls")
|
||||
]
|
||||
|
||||
assert len(assistant_messages_with_tool_calls) > 0
|
||||
assistant_msg = assistant_messages_with_tool_calls[0]
|
||||
assert "reasoning_content" in assistant_msg
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_deepseek_reasoning_content_in_multi_turn_conversation(monkeypatch):
|
||||
"""
|
||||
Verify reasoning_content is included in assistant messages during multi-turn conversations.
|
||||
|
||||
When DeepSeek returns reasoning_content with tool_calls, subsequent API calls must
|
||||
include the reasoning_content field in the assistant message to avoid 400 errors.
|
||||
"""
|
||||
captured_calls: list[dict[str, Any]] = []
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured_calls.append({"model": model, "messages": messages, **kwargs})
|
||||
|
||||
# First call: model returns reasoning_content + tool_call
|
||||
if len(captured_calls) == 1:
|
||||
tool_call = ChatCompletionMessageToolCall(
|
||||
id="call_weather_123",
|
||||
type="function",
|
||||
function=Function(name="get_weather", arguments='{"city": "Tokyo"}'),
|
||||
)
|
||||
msg = Message(
|
||||
role="assistant",
|
||||
content=None,
|
||||
tool_calls=[tool_call],
|
||||
)
|
||||
# DeepSeek adds reasoning_content
|
||||
msg.reasoning_content = "I need to get the weather for Tokyo first."
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
|
||||
|
||||
# Second call: check if reasoning_content was in the request
|
||||
# In real DeepSeek API, this would fail with 400 if reasoning_content is missing
|
||||
msg = Message(
|
||||
role="assistant", content="Based on my findings, the weather in Tokyo is sunny."
|
||||
)
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(100, 50, 150))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
model = LitellmModel(model="deepseek/deepseek-reasoner")
|
||||
|
||||
# First call
|
||||
first_response = await model.get_response(
|
||||
system_instructions="You are a helpful assistant.",
|
||||
input="What's the weather in Tokyo?",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
)
|
||||
|
||||
input_items: list[Any] = []
|
||||
input_items.append({"role": "user", "content": "What's the weather in Tokyo?"})
|
||||
|
||||
for item in first_response.output:
|
||||
if hasattr(item, "model_dump"):
|
||||
input_items.append(item.model_dump())
|
||||
else:
|
||||
input_items.append(item)
|
||||
|
||||
input_items.append(
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_123",
|
||||
"output": "The weather in Tokyo is sunny and 22°C.",
|
||||
}
|
||||
)
|
||||
|
||||
await model.get_response(
|
||||
system_instructions="You are a helpful assistant.",
|
||||
input=input_items,
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
)
|
||||
|
||||
assert len(captured_calls) == 2
|
||||
|
||||
second_call_messages = captured_calls[1]["messages"]
|
||||
|
||||
assistant_with_tools = None
|
||||
for msg in second_call_messages:
|
||||
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
assistant_with_tools = msg
|
||||
break
|
||||
|
||||
assert assistant_with_tools is not None
|
||||
assert "reasoning_content" in assistant_with_tools
|
||||
|
||||
|
||||
def test_deepseek_reasoning_content_with_openai_chatcompletions_path():
|
||||
"""
|
||||
Verify reasoning_content works when using OpenAIChatCompletionsModel.
|
||||
|
||||
This ensures the fix works for both LiteLLM and OpenAI ChatCompletions code paths.
|
||||
"""
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
|
||||
input_items: list[Any] = [
|
||||
{"role": "user", "content": "What's the weather in Paris?"},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [{"text": "I need to check the weather in Paris.", "type": "summary_text"}],
|
||||
"type": "reasoning",
|
||||
"content": None,
|
||||
"encrypted_content": None,
|
||||
"status": None,
|
||||
"provider_data": {"model": "deepseek-reasoner", "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"arguments": '{"city": "Paris"}',
|
||||
"call_id": "call_weather_456",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"id": "__fake_id__",
|
||||
"status": None,
|
||||
"provider_data": {"model": "deepseek-reasoner"},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_456",
|
||||
"output": "The weather in Paris is cloudy and 15°C.",
|
||||
},
|
||||
]
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
input_items,
|
||||
model="deepseek-reasoner",
|
||||
)
|
||||
|
||||
assistant_with_tools = None
|
||||
for msg in messages:
|
||||
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
assistant_with_tools = msg
|
||||
break
|
||||
|
||||
assert assistant_with_tools is not None
|
||||
assert "reasoning_content" in assistant_with_tools
|
||||
# Use type: ignore since reasoning_content is a dynamic field not in OpenAI's TypedDict
|
||||
assert assistant_with_tools["reasoning_content"] == "I need to check the weather in Paris." # type: ignore[typeddict-item]
|
||||
|
||||
|
||||
def test_reasoning_content_from_other_provider_not_attached_to_deepseek():
|
||||
"""
|
||||
Verify reasoning_content from non-DeepSeek providers is NOT attached to DeepSeek messages.
|
||||
|
||||
When switching models mid-conversation (e.g., from Claude to DeepSeek), reasoning items
|
||||
that originated from Claude should not have their summaries attached as reasoning_content
|
||||
to DeepSeek assistant messages, as this would leak unrelated reasoning and may trigger
|
||||
DeepSeek 400 errors.
|
||||
"""
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
|
||||
input_items: list[Any] = [
|
||||
{"role": "user", "content": "What's the weather in Paris?"},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [{"text": "Claude's reasoning about the weather.", "type": "summary_text"}],
|
||||
"type": "reasoning",
|
||||
"content": None,
|
||||
"encrypted_content": None,
|
||||
"status": None,
|
||||
# this one came from Claude, not DeepSeek
|
||||
"provider_data": {"model": "claude-sonnet-4-20250514", "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"arguments": '{"city": "Paris"}',
|
||||
"call_id": "call_weather_789",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"id": "__fake_id__",
|
||||
"status": None,
|
||||
"provider_data": {"model": "claude-sonnet-4-20250514"},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_789",
|
||||
"output": "The weather in Paris is cloudy.",
|
||||
},
|
||||
]
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
input_items,
|
||||
model="deepseek-reasoner",
|
||||
)
|
||||
|
||||
assistant_with_tools = None
|
||||
for msg in messages:
|
||||
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
assistant_with_tools = msg
|
||||
break
|
||||
|
||||
assert assistant_with_tools is not None
|
||||
# reasoning_content should NOT be present since the reasoning came from Claude, not DeepSeek
|
||||
assert "reasoning_content" not in assistant_with_tools
|
||||
|
||||
|
||||
def test_reasoning_content_without_provider_data_attached_for_backward_compat():
|
||||
"""
|
||||
Verify reasoning_content from items without provider_data is attached for backward compat.
|
||||
|
||||
For older items that don't have provider_data (before provider tracking was added),
|
||||
we should still attach reasoning_content to maintain backward compatibility.
|
||||
"""
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
|
||||
# Reasoning item without provider_data (older format)
|
||||
input_items: list[Any] = [
|
||||
{"role": "user", "content": "What's the weather in Tokyo?"},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [{"text": "Reasoning without provider info.", "type": "summary_text"}],
|
||||
"type": "reasoning",
|
||||
"content": None,
|
||||
"encrypted_content": None,
|
||||
"status": None,
|
||||
# No provider_data
|
||||
},
|
||||
{
|
||||
"arguments": '{"city": "Tokyo"}',
|
||||
"call_id": "call_weather_101",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"id": "__fake_id__",
|
||||
"status": None,
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_101",
|
||||
"output": "The weather in Tokyo is sunny.",
|
||||
},
|
||||
]
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
input_items,
|
||||
model="deepseek-reasoner",
|
||||
)
|
||||
|
||||
assistant_with_tools = None
|
||||
for msg in messages:
|
||||
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
assistant_with_tools = msg
|
||||
break
|
||||
|
||||
assert assistant_with_tools is not None
|
||||
# reasoning_content SHOULD be present for backward compatibility
|
||||
assert "reasoning_content" in assistant_with_tools
|
||||
assert assistant_with_tools["reasoning_content"] == "Reasoning without provider info." # type: ignore[typeddict-item]
|
||||
@@ -0,0 +1,201 @@
|
||||
import os
|
||||
from typing import Literal
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from openai.types.shared.reasoning import Reasoning
|
||||
|
||||
from agents import Agent
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models import (
|
||||
get_default_model,
|
||||
get_default_model_settings,
|
||||
gpt_5_reasoning_settings_required,
|
||||
is_gpt_5_default,
|
||||
)
|
||||
|
||||
|
||||
def _gpt_5_default_settings(
|
||||
reasoning_effort: Literal["none", "low", "medium"] | None,
|
||||
) -> ModelSettings:
|
||||
if reasoning_effort is None:
|
||||
return ModelSettings(verbosity="low")
|
||||
return ModelSettings(reasoning=Reasoning(effort=reasoning_effort), verbosity="low")
|
||||
|
||||
|
||||
def test_default_model_is_gpt_5_4_mini():
|
||||
assert get_default_model() == "gpt-5.4-mini"
|
||||
assert is_gpt_5_default() is True
|
||||
assert gpt_5_reasoning_settings_required(get_default_model()) is True
|
||||
assert get_default_model_settings() == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5.4"})
|
||||
def test_is_gpt_5_default_with_real_model_name():
|
||||
assert get_default_model() == "gpt-5.4"
|
||||
assert is_gpt_5_default() is True
|
||||
|
||||
|
||||
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-4.1"})
|
||||
def test_is_gpt_5_default_returns_false_for_non_gpt_5_default_model():
|
||||
assert get_default_model() == "gpt-4.1"
|
||||
assert is_gpt_5_default() is False
|
||||
|
||||
|
||||
def test_gpt_5_reasoning_settings_required_detects_gpt_5_models_while_ignoring_chat_latest():
|
||||
assert gpt_5_reasoning_settings_required("gpt-5") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.1") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.2") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.2-codex") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.2-pro") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.4-pro") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.5") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5-mini") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5-nano") is True
|
||||
assert gpt_5_reasoning_settings_required("gpt-5-chat-latest") is False
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.1-chat-latest") is False
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.2-chat-latest") is False
|
||||
assert gpt_5_reasoning_settings_required("gpt-5.3-chat-latest") is False
|
||||
|
||||
|
||||
def test_gpt_5_reasoning_settings_required_returns_false_for_non_gpt_5_models():
|
||||
assert gpt_5_reasoning_settings_required("gpt-4.1") is False
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_1_models():
|
||||
assert get_default_model_settings("gpt-5.1") == _gpt_5_default_settings("none")
|
||||
assert get_default_model_settings("gpt-5.1-2025-11-13") == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_2_models():
|
||||
assert get_default_model_settings("gpt-5.2") == _gpt_5_default_settings("none")
|
||||
assert get_default_model_settings("gpt-5.2-2025-12-11") == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_3_codex_models():
|
||||
assert get_default_model_settings("gpt-5.3-codex") == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_4_models():
|
||||
assert get_default_model_settings("gpt-5.4") == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_4_snapshot_families():
|
||||
assert get_default_model_settings("gpt-5.4-2026-03-05") == _gpt_5_default_settings("none")
|
||||
assert get_default_model_settings("gpt-5.4-mini-2026-03-17") == _gpt_5_default_settings("none")
|
||||
assert get_default_model_settings("gpt-5.4-nano-2026-03-17") == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_4_mini_and_nano():
|
||||
assert get_default_model_settings("gpt-5.4-mini") == _gpt_5_default_settings("none")
|
||||
assert get_default_model_settings("gpt-5.4-nano") == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_5_models():
|
||||
assert get_default_model_settings("gpt-5.5") == _gpt_5_default_settings("none")
|
||||
assert get_default_model_settings("gpt-5.5-2026-04-23") == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"])
|
||||
def test_get_default_model_settings_returns_none_reasoning_defaults_for_gpt_5_6_models(
|
||||
model: str,
|
||||
):
|
||||
assert get_default_model_settings(model) == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["gpt-5.6", "gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"])
|
||||
def test_agent_uses_gpt_5_6_model_settings_from_default_model_env(
|
||||
model: str, monkeypatch: pytest.MonkeyPatch
|
||||
):
|
||||
monkeypatch.setenv("OPENAI_DEFAULT_MODEL", model.upper())
|
||||
|
||||
agent = Agent(name="test")
|
||||
|
||||
assert get_default_model() == model
|
||||
assert agent.model is None
|
||||
assert agent.model_settings == _gpt_5_default_settings("none")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_low_reasoning_defaults_for_base_gpt_5():
|
||||
assert get_default_model_settings("gpt-5") == _gpt_5_default_settings("low")
|
||||
assert get_default_model_settings("gpt-5-2025-08-07") == _gpt_5_default_settings("low")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_low_reasoning_defaults_for_gpt_5_2_codex():
|
||||
assert get_default_model_settings("gpt-5.2-codex") == _gpt_5_default_settings("low")
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_medium_reasoning_defaults_for_gpt_5_pro_models():
|
||||
assert get_default_model_settings("gpt-5.2-pro") == _gpt_5_default_settings("medium")
|
||||
assert get_default_model_settings("gpt-5.2-pro-2025-12-11") == _gpt_5_default_settings("medium")
|
||||
assert get_default_model_settings("gpt-5.4-pro") == _gpt_5_default_settings("medium")
|
||||
assert get_default_model_settings("gpt-5.4-pro-2026-03-05") == _gpt_5_default_settings("medium")
|
||||
|
||||
|
||||
def test_get_default_model_settings_omits_reasoning_for_unconfirmed_gpt_5_variants():
|
||||
assert get_default_model_settings("gpt-5-mini") == _gpt_5_default_settings(None)
|
||||
assert get_default_model_settings("gpt-5-mini-2025-08-07") == _gpt_5_default_settings(None)
|
||||
assert get_default_model_settings("gpt-5-nano") == _gpt_5_default_settings(None)
|
||||
assert get_default_model_settings("gpt-5-nano-2025-08-07") == _gpt_5_default_settings(None)
|
||||
assert get_default_model_settings("gpt-5.1-codex") == _gpt_5_default_settings(None)
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_empty_settings_for_gpt_5_chat_latest_aliases():
|
||||
assert get_default_model_settings("gpt-5-chat-latest") == ModelSettings()
|
||||
assert get_default_model_settings("gpt-5.1-chat-latest") == ModelSettings()
|
||||
assert get_default_model_settings("gpt-5.2-chat-latest") == ModelSettings()
|
||||
assert get_default_model_settings("gpt-5.3-chat-latest") == ModelSettings()
|
||||
|
||||
|
||||
def test_get_default_model_settings_returns_empty_settings_for_non_gpt_5_models():
|
||||
assert get_default_model_settings("gpt-4.1") == ModelSettings()
|
||||
|
||||
|
||||
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5"})
|
||||
def test_agent_uses_gpt_5_default_model_settings():
|
||||
"""Agent should inherit GPT-5 default model settings."""
|
||||
agent = Agent(name="test")
|
||||
assert agent.model is None
|
||||
assert agent.model_settings.reasoning.effort == "low" # type: ignore[union-attr]
|
||||
assert agent.model_settings.verbosity == "low"
|
||||
|
||||
|
||||
def test_agent_uses_model_specific_settings_for_explicit_gpt_5_models():
|
||||
"""Agent should not apply the fallback model's GPT-5 settings to explicit GPT-5 models."""
|
||||
agent = Agent(name="test", model="gpt-5")
|
||||
assert agent.model == "gpt-5"
|
||||
assert agent.model_settings == get_default_model_settings("gpt-5")
|
||||
assert agent.model_settings.reasoning.effort == "low" # type: ignore[union-attr]
|
||||
|
||||
|
||||
def test_agent_uses_empty_settings_for_explicit_non_gpt_5_models():
|
||||
"""Agent should not apply GPT-5 defaults to explicit non-GPT-5 models."""
|
||||
agent = Agent(name="test", model="gpt-4.1")
|
||||
assert agent.model == "gpt-4.1"
|
||||
assert agent.model_settings == ModelSettings()
|
||||
|
||||
|
||||
def test_agent_clone_recomputes_implicit_settings_when_model_changes():
|
||||
"""Agent.clone should keep implicit model settings aligned with the cloned model."""
|
||||
agent = Agent(name="test", model="gpt-5")
|
||||
cloned = agent.clone(model="gpt-5.4-mini")
|
||||
assert cloned.model == "gpt-5.4-mini"
|
||||
assert cloned.model_settings == get_default_model_settings("gpt-5.4-mini")
|
||||
assert cloned.model_settings.reasoning.effort == "none" # type: ignore[union-attr]
|
||||
|
||||
|
||||
def test_agent_clone_preserves_explicit_settings_when_model_changes():
|
||||
"""Agent.clone should not recompute model settings that were explicitly customized."""
|
||||
model_settings = ModelSettings(temperature=0.3)
|
||||
agent = Agent(name="test", model="gpt-5", model_settings=model_settings)
|
||||
cloned = agent.clone(model="gpt-5.4-mini")
|
||||
assert cloned.model == "gpt-5.4-mini"
|
||||
assert cloned.model_settings == model_settings
|
||||
|
||||
|
||||
@patch.dict(os.environ, {"OPENAI_DEFAULT_MODEL": "gpt-5"})
|
||||
def test_agent_resets_model_settings_for_non_gpt_5_models():
|
||||
"""Agent should reset default GPT-5 settings when using a non-GPT-5 model."""
|
||||
agent = Agent(name="test", model="gpt-4.1")
|
||||
assert agent.model == "gpt-4.1"
|
||||
assert agent.model_settings == ModelSettings()
|
||||
@@ -0,0 +1,354 @@
|
||||
"""Tests for the extended thinking message order bug fix in LitellmModel."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
from openai.types.chat import ChatCompletionMessageParam
|
||||
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
|
||||
|
||||
class TestExtendedThinkingMessageOrder:
|
||||
"""Test the _fix_tool_message_ordering method."""
|
||||
|
||||
def test_basic_reordering_tool_result_before_call(self):
|
||||
"""Test that a tool result appearing before its tool call gets reordered correctly."""
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "tool", "tool_call_id": "call_123", "content": "Result for call_123"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_123",
|
||||
"type": "function",
|
||||
"function": {"name": "test", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "Thanks"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
# Should reorder to: user, assistant+tool_call, tool_result, user
|
||||
assert len(result) == 4
|
||||
assert result[0]["role"] == "user"
|
||||
assert result[1]["role"] == "assistant"
|
||||
assert result[1]["tool_calls"][0]["id"] == "call_123" # type: ignore
|
||||
assert result[2]["role"] == "tool"
|
||||
assert result[2]["tool_call_id"] == "call_123"
|
||||
assert result[3]["role"] == "user"
|
||||
|
||||
def test_consecutive_tool_calls_get_separated(self):
|
||||
"""Test that consecutive assistant messages with tool calls get properly paired with results.""" # noqa: E501
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "test1", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_2",
|
||||
"type": "function",
|
||||
"function": {"name": "test2", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "Result 1"},
|
||||
{"role": "tool", "tool_call_id": "call_2", "content": "Result 2"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
# Should pair each tool call with its result immediately
|
||||
assert len(result) == 5
|
||||
assert result[0]["role"] == "user"
|
||||
assert result[1]["role"] == "assistant"
|
||||
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
|
||||
assert result[2]["role"] == "tool"
|
||||
assert result[2]["tool_call_id"] == "call_1"
|
||||
assert result[3]["role"] == "assistant"
|
||||
assert result[3]["tool_calls"][0]["id"] == "call_2" # type: ignore
|
||||
assert result[4]["role"] == "tool"
|
||||
assert result[4]["tool_call_id"] == "call_2"
|
||||
|
||||
def test_unmatched_tool_results_preserved(self):
|
||||
"""Test that tool results without matching tool calls are preserved."""
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "test", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "Matched result"},
|
||||
{"role": "tool", "tool_call_id": "call_orphan", "content": "Orphaned result"},
|
||||
{"role": "user", "content": "End"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
# Should preserve the orphaned tool result
|
||||
assert len(result) == 5
|
||||
assert result[0]["role"] == "user"
|
||||
assert result[1]["role"] == "assistant"
|
||||
assert result[2]["role"] == "tool"
|
||||
assert result[2]["tool_call_id"] == "call_1"
|
||||
assert result[3]["role"] == "tool" # Orphaned result preserved
|
||||
assert result[3]["tool_call_id"] == "call_orphan"
|
||||
assert result[4]["role"] == "user"
|
||||
|
||||
def test_tool_calls_without_results_preserved(self):
|
||||
"""Test that tool calls without results are still included."""
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "test", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "user", "content": "End"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
# Should preserve the tool call even without a result
|
||||
assert len(result) == 3
|
||||
assert result[0]["role"] == "user"
|
||||
assert result[1]["role"] == "assistant"
|
||||
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
|
||||
assert result[2]["role"] == "user"
|
||||
|
||||
def test_correctly_ordered_messages_unchanged(self):
|
||||
"""Test that correctly ordered messages remain in the same order."""
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "test", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "Result"},
|
||||
{"role": "assistant", "content": "Done"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
# Should remain exactly the same
|
||||
assert len(result) == 4
|
||||
assert result[0]["role"] == "user"
|
||||
assert result[1]["role"] == "assistant"
|
||||
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
|
||||
assert result[2]["role"] == "tool"
|
||||
assert result[2]["tool_call_id"] == "call_1"
|
||||
assert result[3]["role"] == "assistant"
|
||||
|
||||
def test_multiple_tool_calls_single_message(self):
|
||||
"""Test assistant message with multiple tool calls gets split properly."""
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "test1", "arguments": "{}"},
|
||||
},
|
||||
{
|
||||
"id": "call_2",
|
||||
"type": "function",
|
||||
"function": {"name": "test2", "arguments": "{}"},
|
||||
},
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "Result 1"},
|
||||
{"role": "tool", "tool_call_id": "call_2", "content": "Result 2"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
# Should split the multi-tool message and pair each properly
|
||||
assert len(result) == 5
|
||||
assert result[0]["role"] == "user"
|
||||
assert result[1]["role"] == "assistant"
|
||||
assert len(result[1]["tool_calls"]) == 1 # type: ignore
|
||||
assert result[1]["tool_calls"][0]["id"] == "call_1" # type: ignore
|
||||
assert result[2]["role"] == "tool"
|
||||
assert result[2]["tool_call_id"] == "call_1"
|
||||
assert result[3]["role"] == "assistant"
|
||||
assert len(result[3]["tool_calls"]) == 1 # type: ignore
|
||||
assert result[3]["tool_calls"][0]["id"] == "call_2" # type: ignore
|
||||
assert result[4]["role"] == "tool"
|
||||
assert result[4]["tool_call_id"] == "call_2"
|
||||
|
||||
def test_split_does_not_duplicate_content_or_thinking(self):
|
||||
"""Splitting multi-tool assistant messages must not duplicate text/thinking blocks.
|
||||
|
||||
Anthropic's extended thinking API rejects requests that include the same signed
|
||||
thinking block more than once, and duplicated assistant text corrupts conversation
|
||||
history. Only the first split should retain content, thinking_blocks, and
|
||||
reasoning_content; subsequent splits should carry the tool_call alone.
|
||||
"""
|
||||
# Build the assistant message via cast so mypy doesn't reject the
|
||||
# extra keys (`thinking_blocks`, `reasoning_content`) which are not
|
||||
# part of the upstream ChatCompletionAssistantMessageParam TypedDict
|
||||
# but are surfaced by litellm for Anthropic extended thinking.
|
||||
assistant_msg = cast(
|
||||
ChatCompletionMessageParam,
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Looking up both queries.",
|
||||
"thinking_blocks": [
|
||||
{"type": "thinking", "thinking": "plan", "signature": "sig_abc"}
|
||||
],
|
||||
"reasoning_content": "internal plan",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "s", "arguments": "{}"},
|
||||
},
|
||||
{
|
||||
"id": "call_2",
|
||||
"type": "function",
|
||||
"function": {"name": "s", "arguments": "{}"},
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Search both"},
|
||||
assistant_msg,
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "ok1"},
|
||||
{"role": "tool", "tool_call_id": "call_2", "content": "ok2"},
|
||||
]
|
||||
|
||||
model = LitellmModel("claude-3-5-sonnet")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
assistants = [cast(dict[str, Any], m) for m in result if m.get("role") == "assistant"]
|
||||
assert len(assistants) == 2
|
||||
# First split keeps the shared fields.
|
||||
assert assistants[0].get("content") == "Looking up both queries."
|
||||
assert "thinking_blocks" in assistants[0]
|
||||
assert "reasoning_content" in assistants[0]
|
||||
# Second split must NOT duplicate them.
|
||||
assert "content" not in assistants[1]
|
||||
assert "thinking_blocks" not in assistants[1]
|
||||
assert "reasoning_content" not in assistants[1]
|
||||
# Tool calls are still split one-per-message.
|
||||
assert assistants[0]["tool_calls"][0]["id"] == "call_1"
|
||||
assert assistants[1]["tool_calls"][0]["id"] == "call_2"
|
||||
|
||||
def test_empty_messages_list(self):
|
||||
"""Test that empty message list is handled correctly."""
|
||||
messages: list[ChatCompletionMessageParam] = []
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
assert result == []
|
||||
|
||||
def test_no_tool_messages(self):
|
||||
"""Test that messages without tool calls are left unchanged."""
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hi there"},
|
||||
{"role": "user", "content": "How are you?"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
assert result == messages
|
||||
|
||||
def test_complex_mixed_scenario(self):
|
||||
"""Test a complex scenario with various message types and orderings."""
|
||||
messages: list[ChatCompletionMessageParam] = [
|
||||
{"role": "user", "content": "Start"},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_out_of_order",
|
||||
"content": "Out of order result",
|
||||
}, # This comes before its call
|
||||
{"role": "assistant", "content": "Regular response"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_out_of_order",
|
||||
"type": "function",
|
||||
"function": {"name": "test", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_normal",
|
||||
"type": "function",
|
||||
"function": {"name": "test2", "arguments": "{}"},
|
||||
}
|
||||
],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_normal", "content": "Normal result"},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_orphan",
|
||||
"content": "Orphaned result",
|
||||
}, # No matching call
|
||||
{"role": "user", "content": "End"},
|
||||
]
|
||||
|
||||
model = LitellmModel("test-model")
|
||||
result = model._fix_tool_message_ordering(messages)
|
||||
|
||||
# Should reorder properly while preserving all messages
|
||||
assert len(result) == 8
|
||||
assert result[0]["role"] == "user" # Start
|
||||
assert result[1]["role"] == "assistant" # Regular response
|
||||
assert result[2]["role"] == "assistant" # call_out_of_order
|
||||
assert result[2]["tool_calls"][0]["id"] == "call_out_of_order" # type: ignore
|
||||
assert result[3]["role"] == "tool" # Out of order result (now properly paired)
|
||||
assert result[3]["tool_call_id"] == "call_out_of_order"
|
||||
assert result[4]["role"] == "assistant" # call_normal
|
||||
assert result[4]["tool_calls"][0]["id"] == "call_normal" # type: ignore
|
||||
assert result[5]["role"] == "tool" # Normal result
|
||||
assert result[5]["tool_call_id"] == "call_normal"
|
||||
assert result[6]["role"] == "tool" # Orphaned result (preserved)
|
||||
assert result[6]["tool_call_id"] == "call_orphan"
|
||||
assert result[7]["role"] == "user" # End
|
||||
@@ -0,0 +1,126 @@
|
||||
"""
|
||||
Test for Gemini thought signatures in function calling.
|
||||
|
||||
Validates that thought signatures are preserved through the bidirectional roundtrip:
|
||||
- Gemini chatcmpl message → response item → back to message
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from openai.types.chat.chat_completion_message_tool_call import Function
|
||||
|
||||
from agents.extensions.models.litellm_model import InternalChatCompletionMessage, InternalToolCall
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
|
||||
|
||||
def test_gemini_thought_signature_roundtrip():
|
||||
"""Test that thought signatures are preserved from Gemini responses to messages."""
|
||||
|
||||
# Create mock Gemini response with thought signature in new extra_content structure
|
||||
class MockToolCall(InternalToolCall):
|
||||
def __init__(self):
|
||||
super().__init__(
|
||||
id="call_123",
|
||||
type="function",
|
||||
function=Function(name="get_weather", arguments='{"city": "Paris"}'),
|
||||
extra_content={"google": {"thought_signature": "test_signature_abc"}},
|
||||
)
|
||||
|
||||
message = InternalChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="I'll check the weather.",
|
||||
reasoning_content="",
|
||||
tool_calls=[MockToolCall()],
|
||||
)
|
||||
|
||||
# Step 1: Convert to items
|
||||
provider_data = {"model": "gemini/gemini-3-pro", "response_id": "gemini-response-id-123"}
|
||||
|
||||
items = Converter.message_to_output_items(message, provider_data=provider_data)
|
||||
|
||||
func_calls = [item for item in items if hasattr(item, "type") and item.type == "function_call"]
|
||||
assert len(func_calls) == 1
|
||||
|
||||
# Verify thought_signature is stored in items with our provider_data structure
|
||||
func_call_dict = func_calls[0].model_dump()
|
||||
|
||||
assert func_call_dict["provider_data"]["model"] == "gemini/gemini-3-pro"
|
||||
assert func_call_dict["provider_data"]["response_id"] == "gemini-response-id-123"
|
||||
assert func_call_dict["provider_data"]["thought_signature"] == "test_signature_abc"
|
||||
|
||||
# Step 2: Convert back to messages
|
||||
items_as_dicts = [item.model_dump() for item in items]
|
||||
messages = Converter.items_to_messages(
|
||||
[{"role": "user", "content": "test"}] + items_as_dicts,
|
||||
model="gemini/gemini-3-pro",
|
||||
)
|
||||
|
||||
# Verify thought_signature is restored in extra_content format
|
||||
assistant_msg = [msg for msg in messages if msg.get("role") == "assistant"][0]
|
||||
tool_call = assistant_msg["tool_calls"][0] # type: ignore[index, typeddict-item]
|
||||
assert tool_call["extra_content"]["google"]["thought_signature"] == "test_signature_abc"
|
||||
|
||||
|
||||
def test_gemini_multiple_tool_calls_with_thought_signatures():
|
||||
"""Test multiple tool calls each preserve their own thought signatures."""
|
||||
tool_call_1 = InternalToolCall(
|
||||
id="call_1",
|
||||
type="function",
|
||||
function=Function(name="func_a", arguments='{"x": 1}'),
|
||||
extra_content={"google": {"thought_signature": "sig_aaa"}},
|
||||
)
|
||||
tool_call_2 = InternalToolCall(
|
||||
id="call_2",
|
||||
type="function",
|
||||
function=Function(name="func_b", arguments='{"y": 2}'),
|
||||
extra_content={"google": {"thought_signature": "sig_bbb"}},
|
||||
)
|
||||
|
||||
message = InternalChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="Calling two functions.",
|
||||
reasoning_content="",
|
||||
tool_calls=[tool_call_1, tool_call_2],
|
||||
)
|
||||
|
||||
provider_data = {"model": "gemini/gemini-3-pro"}
|
||||
items = Converter.message_to_output_items(message, provider_data=provider_data)
|
||||
|
||||
func_calls = [i for i in items if hasattr(i, "type") and i.type == "function_call"]
|
||||
assert len(func_calls) == 2
|
||||
|
||||
assert func_calls[0].model_dump()["provider_data"]["thought_signature"] == "sig_aaa"
|
||||
assert func_calls[1].model_dump()["provider_data"]["thought_signature"] == "sig_bbb"
|
||||
|
||||
|
||||
def test_gemini_thought_signature_items_to_messages():
|
||||
"""Test that items_to_messages restores extra_content from provider_data for Gemini."""
|
||||
|
||||
# Create a function call item with provider_data containing thought_signature
|
||||
func_call_item = {
|
||||
"id": "fake-id",
|
||||
"call_id": "call_restore",
|
||||
"name": "restore_func",
|
||||
"arguments": '{"test": true}',
|
||||
"type": "function_call",
|
||||
"provider_data": {
|
||||
"model": "gemini/gemini-3-pro",
|
||||
"response_id": "gemini-response-id-123",
|
||||
"thought_signature": "restored_sig_xyz",
|
||||
},
|
||||
}
|
||||
|
||||
items = [{"role": "user", "content": "test"}, func_call_item]
|
||||
messages = Converter.items_to_messages(items, model="gemini/gemini-3-pro") # type: ignore[arg-type]
|
||||
|
||||
# Find the assistant message with tool_calls
|
||||
assistant_msgs = [m for m in messages if m.get("role") == "assistant"]
|
||||
assert len(assistant_msgs) == 1
|
||||
|
||||
tool_calls: list[dict[str, Any]] = assistant_msgs[0].get("tool_calls", []) # type: ignore[assignment]
|
||||
assert len(tool_calls) == 1
|
||||
|
||||
# Verify extra_content is restored in Google format
|
||||
assert tool_calls[0]["extra_content"]["google"]["thought_signature"] == "restored_sig_xyz"
|
||||
@@ -0,0 +1,210 @@
|
||||
"""
|
||||
Test for Gemini thought signatures in streaming function calls.
|
||||
|
||||
Validates that thought signatures are captured from streaming chunks
|
||||
and included in the final function call events.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
from openai.types.chat import ChatCompletionChunk
|
||||
from openai.types.chat.chat_completion_chunk import (
|
||||
Choice,
|
||||
ChoiceDelta,
|
||||
ChoiceDeltaToolCall,
|
||||
ChoiceDeltaToolCallFunction,
|
||||
)
|
||||
from openai.types.responses import Response
|
||||
|
||||
from agents.models.chatcmpl_stream_handler import ChatCmplStreamHandler
|
||||
|
||||
# ========== Helper Functions ==========
|
||||
|
||||
|
||||
def create_tool_call_delta(
|
||||
index: int,
|
||||
tool_call_id: str | None = None,
|
||||
function_name: str | None = None,
|
||||
arguments: str | None = None,
|
||||
provider_specific_fields: dict[str, Any] | None = None,
|
||||
extra_content: dict[str, Any] | None = None,
|
||||
) -> ChoiceDeltaToolCall:
|
||||
"""Create a tool call delta for streaming."""
|
||||
function = ChoiceDeltaToolCallFunction(
|
||||
name=function_name,
|
||||
arguments=arguments,
|
||||
)
|
||||
|
||||
delta = ChoiceDeltaToolCall(
|
||||
index=index,
|
||||
id=tool_call_id,
|
||||
type="function" if tool_call_id else None,
|
||||
function=function,
|
||||
)
|
||||
|
||||
# Add provider_specific_fields (litellm format)
|
||||
if provider_specific_fields:
|
||||
delta_any = cast(Any, delta)
|
||||
delta_any.provider_specific_fields = provider_specific_fields
|
||||
|
||||
# Add extra_content (Google chatcmpl format)
|
||||
if extra_content:
|
||||
delta_any = cast(Any, delta)
|
||||
delta_any.extra_content = extra_content
|
||||
|
||||
return delta
|
||||
|
||||
|
||||
def create_chunk(
|
||||
tool_calls: list[ChoiceDeltaToolCall] | None = None,
|
||||
content: str | None = None,
|
||||
include_usage: bool = False,
|
||||
) -> ChatCompletionChunk:
|
||||
"""Create a ChatCompletionChunk for testing."""
|
||||
delta = ChoiceDelta(
|
||||
content=content,
|
||||
role="assistant" if content or tool_calls else None,
|
||||
tool_calls=tool_calls,
|
||||
)
|
||||
|
||||
chunk = ChatCompletionChunk(
|
||||
id="chunk-id-123",
|
||||
created=1,
|
||||
model="gemini/gemini-3-pro",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=delta, finish_reason=None)],
|
||||
)
|
||||
|
||||
if include_usage:
|
||||
from openai.types.completion_usage import CompletionUsage
|
||||
|
||||
chunk.usage = CompletionUsage(
|
||||
completion_tokens=10,
|
||||
prompt_tokens=5,
|
||||
total_tokens=15,
|
||||
)
|
||||
|
||||
return chunk
|
||||
|
||||
|
||||
def create_final_chunk() -> ChatCompletionChunk:
|
||||
"""Create a final chunk with finish_reason='tool_calls'."""
|
||||
return ChatCompletionChunk(
|
||||
id="chunk-id-456",
|
||||
created=1,
|
||||
model="gemini/gemini-3-pro",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="tool_calls")],
|
||||
)
|
||||
|
||||
|
||||
async def create_fake_stream(
|
||||
chunks: list[ChatCompletionChunk],
|
||||
) -> AsyncIterator[ChatCompletionChunk]:
|
||||
"""Create an async iterator from chunks."""
|
||||
for chunk in chunks:
|
||||
yield chunk
|
||||
|
||||
|
||||
def create_mock_response() -> Response:
|
||||
"""Create a mock Response object."""
|
||||
return Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="gemini/gemini-3-pro",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="auto",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
|
||||
|
||||
# ========== Tests ==========
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_captures_litellmprovider_specific_fields_thought_signature():
|
||||
"""Test streaming captures thought_signature from litellm's provider_specific_fields."""
|
||||
chunks = [
|
||||
create_chunk(
|
||||
tool_calls=[
|
||||
create_tool_call_delta(
|
||||
index=0,
|
||||
tool_call_id="call_stream_1",
|
||||
function_name="get_weather",
|
||||
provider_specific_fields={"thought_signature": "litellm_sig_123"},
|
||||
)
|
||||
]
|
||||
),
|
||||
create_chunk(tool_calls=[create_tool_call_delta(index=0, arguments='{"city": "Tokyo"}')]),
|
||||
create_final_chunk(),
|
||||
]
|
||||
|
||||
response = create_mock_response()
|
||||
stream = create_fake_stream(chunks)
|
||||
|
||||
events = []
|
||||
async for event in ChatCmplStreamHandler.handle_stream(
|
||||
response,
|
||||
stream, # type: ignore[arg-type]
|
||||
model="gemini/gemini-3-pro",
|
||||
):
|
||||
events.append(event)
|
||||
|
||||
# Find function call done event
|
||||
done_events = [e for e in events if e.type == "response.output_item.done"]
|
||||
func_done = [
|
||||
e for e in done_events if hasattr(e.item, "type") and e.item.type == "function_call"
|
||||
]
|
||||
assert len(func_done) == 1
|
||||
|
||||
provider_data = func_done[0].item.model_dump().get("provider_data", {})
|
||||
assert provider_data.get("thought_signature") == "litellm_sig_123"
|
||||
assert provider_data["model"] == "gemini/gemini-3-pro"
|
||||
assert provider_data["response_id"] == "chunk-id-123"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_captures_google_extra_content_thought_signature():
|
||||
"""Test streaming captures thought_signature from Google's extra_content format."""
|
||||
chunks = [
|
||||
create_chunk(
|
||||
tool_calls=[
|
||||
create_tool_call_delta(
|
||||
index=0,
|
||||
tool_call_id="call_stream_2",
|
||||
function_name="search",
|
||||
extra_content={"google": {"thought_signature": "google_sig_456"}},
|
||||
)
|
||||
]
|
||||
),
|
||||
create_chunk(tool_calls=[create_tool_call_delta(index=0, arguments='{"query": "test"}')]),
|
||||
create_final_chunk(),
|
||||
]
|
||||
|
||||
response = create_mock_response()
|
||||
stream = create_fake_stream(chunks)
|
||||
|
||||
events = []
|
||||
async for event in ChatCmplStreamHandler.handle_stream(
|
||||
response,
|
||||
stream, # type: ignore[arg-type]
|
||||
model="gemini/gemini-3-pro",
|
||||
):
|
||||
events.append(event)
|
||||
|
||||
done_events = [e for e in events if e.type == "response.output_item.done"]
|
||||
func_done = [
|
||||
e for e in done_events if hasattr(e.item, "type") and e.item.type == "function_call"
|
||||
]
|
||||
assert len(func_done) == 1
|
||||
|
||||
provider_data = func_done[0].item.model_dump().get("provider_data", {})
|
||||
assert provider_data.get("thought_signature") == "google_sig_456"
|
||||
assert provider_data["model"] == "gemini/gemini-3-pro"
|
||||
assert provider_data["response_id"] == "chunk-id-123"
|
||||
@@ -0,0 +1,317 @@
|
||||
import httpx
|
||||
import litellm
|
||||
import pytest
|
||||
from httpx import Headers, Response
|
||||
from litellm.exceptions import RateLimitError
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
from openai import APIConnectionError
|
||||
from openai.types.chat.chat_completion import ChatCompletion, Choice
|
||||
from openai.types.chat.chat_completion_message import ChatCompletionMessage
|
||||
from openai.types.completion_usage import CompletionUsage
|
||||
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models._retry_runtime import provider_managed_retries_disabled
|
||||
from agents.models.interface import ModelTracing
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from agents.retry import ModelRetryAdviceRequest, ModelRetrySettings
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_litellm_kwargs_forwarded(monkeypatch):
|
||||
"""
|
||||
Test that kwargs from ModelSettings are forwarded to litellm.acompletion.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="test response")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
settings = ModelSettings(
|
||||
temperature=0.5,
|
||||
extra_args={
|
||||
"custom_param": "custom_value",
|
||||
"seed": 42,
|
||||
"stop": ["END"],
|
||||
"logit_bias": {123: -100},
|
||||
},
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="test input",
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
)
|
||||
|
||||
# Verify that all kwargs were passed through
|
||||
assert captured["custom_param"] == "custom_value"
|
||||
assert captured["seed"] == 42
|
||||
assert captured["stop"] == ["END"]
|
||||
assert captured["logit_bias"] == {123: -100}
|
||||
|
||||
# Verify regular parameters are still passed
|
||||
assert captured["temperature"] == 0.5
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_chatcompletions_kwargs_forwarded(monkeypatch):
|
||||
"""
|
||||
Test that kwargs from ModelSettings are forwarded to OpenAI chat completions API.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class MockChatCompletions:
|
||||
async def create(self, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = ChatCompletionMessage(role="assistant", content="test response")
|
||||
choice = Choice(index=0, message=msg, finish_reason="stop")
|
||||
return ChatCompletion(
|
||||
id="test-id",
|
||||
created=0,
|
||||
model="gpt-4",
|
||||
object="chat.completion",
|
||||
choices=[choice],
|
||||
usage=CompletionUsage(completion_tokens=5, prompt_tokens=10, total_tokens=15),
|
||||
)
|
||||
|
||||
class MockChat:
|
||||
def __init__(self):
|
||||
self.completions = MockChatCompletions()
|
||||
|
||||
class MockClient:
|
||||
def __init__(self):
|
||||
self.chat = MockChat()
|
||||
self.base_url = "https://api.openai.com/v1"
|
||||
|
||||
settings = ModelSettings(
|
||||
temperature=0.7,
|
||||
extra_args={
|
||||
"seed": 123,
|
||||
"logit_bias": {456: 10},
|
||||
"stop": ["STOP", "END"],
|
||||
"user": "test-user",
|
||||
},
|
||||
)
|
||||
|
||||
mock_client = MockClient()
|
||||
model = OpenAIChatCompletionsModel(model="gpt-4", openai_client=mock_client) # type: ignore
|
||||
|
||||
await model.get_response(
|
||||
system_instructions="Test system",
|
||||
input="test input",
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
# Verify that all kwargs were passed through
|
||||
assert captured["seed"] == 123
|
||||
assert captured["logit_bias"] == {456: 10}
|
||||
assert captured["stop"] == ["STOP", "END"]
|
||||
assert captured["user"] == "test-user"
|
||||
|
||||
# Verify regular parameters are still passed
|
||||
assert captured["temperature"] == 0.7
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_kwargs_handling(monkeypatch):
|
||||
"""
|
||||
Test that empty or None kwargs are handled gracefully.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="test response")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
# Test with None kwargs
|
||||
settings_none = ModelSettings(temperature=0.5, extra_args=None)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="test input",
|
||||
model_settings=settings_none,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
# Should work without error and include regular parameters
|
||||
assert captured["temperature"] == 0.5
|
||||
|
||||
# Test with empty dict
|
||||
captured.clear()
|
||||
settings_empty = ModelSettings(temperature=0.3, extra_args={})
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="test input",
|
||||
model_settings=settings_empty,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
# Should work without error and include regular parameters
|
||||
assert captured["temperature"] == 0.3
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_effort_falls_back_to_extra_args(monkeypatch):
|
||||
"""
|
||||
Ensure reasoning_effort from extra_args is promoted when reasoning settings are missing.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="test response")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
# GitHub issue context: https://github.com/openai/openai-agents-python/issues/1764.
|
||||
settings = ModelSettings(
|
||||
extra_args={"reasoning_effort": "none", "custom_param": "custom_value"}
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="test input",
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert captured["reasoning_effort"] == "none"
|
||||
assert captured["custom_param"] == "custom_value"
|
||||
assert settings.extra_args == {"reasoning_effort": "none", "custom_param": "custom_value"}
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_litellm_retry_settings_do_not_leak_and_disable_provider_retries_on_runner_retry(
|
||||
monkeypatch,
|
||||
):
|
||||
"""Runner retries should disable LiteLLM's own retries without forwarding SDK retry config."""
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="test response")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
settings = ModelSettings(
|
||||
retry=ModelRetrySettings(
|
||||
max_retries=2,
|
||||
backoff={"initial_delay": 0.25, "jitter": False},
|
||||
),
|
||||
extra_args={"max_retries": 7, "num_retries": 6, "custom_param": "custom_value"},
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
with provider_managed_retries_disabled(True):
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="test input",
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
)
|
||||
|
||||
assert settings.retry is not None
|
||||
assert settings.retry.backoff is not None
|
||||
assert captured["custom_param"] == "custom_value"
|
||||
assert captured["max_retries"] == 0
|
||||
assert captured["num_retries"] == 0
|
||||
assert "retry" not in captured
|
||||
|
||||
|
||||
def test_litellm_get_retry_advice_uses_response_headers() -> None:
|
||||
"""LiteLLM retry advice should expose OpenAI-compatible retry headers."""
|
||||
|
||||
model = LitellmModel(model="test-model")
|
||||
error = RateLimitError(
|
||||
message="rate limited",
|
||||
llm_provider="openai",
|
||||
model="gpt-4o-mini",
|
||||
response=Response(
|
||||
status_code=429,
|
||||
headers=Headers({"x-should-retry": "true", "retry-after-ms": "250"}),
|
||||
),
|
||||
)
|
||||
|
||||
advice = model.get_retry_advice(
|
||||
ModelRetryAdviceRequest(
|
||||
error=error,
|
||||
attempt=1,
|
||||
stream=False,
|
||||
)
|
||||
)
|
||||
|
||||
assert advice is not None
|
||||
assert advice.suggested is True
|
||||
assert advice.retry_after == 0.25
|
||||
|
||||
|
||||
def test_litellm_get_retry_advice_keeps_stateful_transport_failures_ambiguous() -> None:
|
||||
model = LitellmModel(model="test-model")
|
||||
error = APIConnectionError(
|
||||
message="connection error",
|
||||
request=httpx.Request("POST", "https://api.openai.com/v1/responses"),
|
||||
)
|
||||
|
||||
advice = model.get_retry_advice(
|
||||
ModelRetryAdviceRequest(
|
||||
error=error,
|
||||
attempt=1,
|
||||
stream=False,
|
||||
previous_response_id="resp_prev",
|
||||
)
|
||||
)
|
||||
|
||||
assert advice is not None
|
||||
assert advice.suggested is True
|
||||
assert advice.replay_safety is None
|
||||
@@ -0,0 +1,697 @@
|
||||
from collections.abc import AsyncIterator
|
||||
|
||||
import pytest
|
||||
from openai.types.chat.chat_completion_chunk import (
|
||||
ChatCompletionChunk,
|
||||
Choice,
|
||||
ChoiceDelta,
|
||||
ChoiceDeltaToolCall,
|
||||
ChoiceDeltaToolCallFunction,
|
||||
)
|
||||
from openai.types.completion_usage import (
|
||||
CompletionTokensDetails,
|
||||
CompletionUsage,
|
||||
PromptTokensDetails,
|
||||
)
|
||||
from openai.types.responses import (
|
||||
Response,
|
||||
ResponseCompletedEvent,
|
||||
ResponseContentPartAddedEvent,
|
||||
ResponseFunctionToolCall,
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputRefusal,
|
||||
ResponseOutputText,
|
||||
ResponseReasoningItem,
|
||||
ResponseRefusalDeltaEvent,
|
||||
)
|
||||
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
from agents.extensions.models.litellm_provider import LitellmProvider
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import ModelTracing
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_yields_events_for_text_content(monkeypatch) -> None:
|
||||
"""
|
||||
Validate that `stream_response` emits the correct sequence of events when
|
||||
streaming a simple assistant message consisting of plain text content.
|
||||
We simulate two chunks of text returned from the chat completion stream.
|
||||
"""
|
||||
# Create two chunks that will be emitted by the fake stream.
|
||||
chunk1 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(content="He"))],
|
||||
)
|
||||
# Mark last chunk with usage so stream_response knows this is final.
|
||||
chunk2 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(content="llo"))],
|
||||
usage=CompletionUsage(
|
||||
completion_tokens=5,
|
||||
prompt_tokens=7,
|
||||
total_tokens=12,
|
||||
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=2),
|
||||
prompt_tokens_details=PromptTokensDetails(cached_tokens=6),
|
||||
),
|
||||
)
|
||||
|
||||
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
for c in (chunk1, chunk2):
|
||||
yield c
|
||||
|
||||
# Patch _fetch_response to inject our fake stream
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
# `_fetch_response` is expected to return a Response skeleton and the async stream
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, fake_stream()
|
||||
|
||||
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
|
||||
model = LitellmProvider().get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
# We expect a response.created, then a response.output_item.added, content part added,
|
||||
# two content delta events (for "He" and "llo"), a content part done, the assistant message
|
||||
# output_item.done, and finally response.completed.
|
||||
# There should be 8 events in total.
|
||||
assert len(output_events) == 8
|
||||
# First event indicates creation.
|
||||
assert output_events[0].type == "response.created"
|
||||
# The output item added and content part added events should mark the assistant message.
|
||||
assert output_events[1].type == "response.output_item.added"
|
||||
assert output_events[2].type == "response.content_part.added"
|
||||
# Two text delta events.
|
||||
assert output_events[3].type == "response.output_text.delta"
|
||||
assert output_events[3].delta == "He"
|
||||
assert output_events[4].type == "response.output_text.delta"
|
||||
assert output_events[4].delta == "llo"
|
||||
# After streaming, the content part and item should be marked done.
|
||||
assert output_events[5].type == "response.content_part.done"
|
||||
assert output_events[6].type == "response.output_item.done"
|
||||
# Last event indicates completion of the stream.
|
||||
assert output_events[7].type == "response.completed"
|
||||
# The completed response should have one output message with full text.
|
||||
completed_resp = output_events[7].response
|
||||
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
|
||||
assert isinstance(completed_resp.output[0].content[0], ResponseOutputText)
|
||||
assert completed_resp.output[0].content[0].text == "Hello"
|
||||
|
||||
assert completed_resp.usage, "usage should not be None"
|
||||
assert completed_resp.usage.input_tokens == 7
|
||||
assert completed_resp.usage.output_tokens == 5
|
||||
assert completed_resp.usage.total_tokens == 12
|
||||
assert completed_resp.usage.input_tokens_details.cached_tokens == 6
|
||||
assert completed_resp.usage.output_tokens_details.reasoning_tokens == 2
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_yields_events_for_refusal_content(monkeypatch) -> None:
|
||||
"""
|
||||
Validate that when the model streams a refusal string instead of normal content,
|
||||
`stream_response` emits the appropriate sequence of events including
|
||||
`response.refusal.delta` events for each chunk of the refusal message and
|
||||
constructs a completed assistant message with a `ResponseOutputRefusal` part.
|
||||
"""
|
||||
# Simulate refusal text coming in two pieces, like content but using the `refusal`
|
||||
# field on the delta rather than `content`.
|
||||
chunk1 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(refusal="No"))],
|
||||
)
|
||||
chunk2 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(refusal="Thanks"))],
|
||||
usage=CompletionUsage(completion_tokens=2, prompt_tokens=2, total_tokens=4),
|
||||
)
|
||||
|
||||
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
for c in (chunk1, chunk2):
|
||||
yield c
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, fake_stream()
|
||||
|
||||
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
|
||||
model = LitellmProvider().get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
# Expect sequence similar to text: created, output_item.added, content part added,
|
||||
# two refusal delta events, content part done, output_item.done, completed.
|
||||
assert len(output_events) == 8
|
||||
assert output_events[0].type == "response.created"
|
||||
assert output_events[1].type == "response.output_item.added"
|
||||
assert output_events[2].type == "response.content_part.added"
|
||||
assert output_events[3].type == "response.refusal.delta"
|
||||
assert output_events[3].delta == "No"
|
||||
assert output_events[4].type == "response.refusal.delta"
|
||||
assert output_events[4].delta == "Thanks"
|
||||
assert output_events[5].type == "response.content_part.done"
|
||||
assert output_events[6].type == "response.output_item.done"
|
||||
assert output_events[7].type == "response.completed"
|
||||
completed_resp = output_events[7].response
|
||||
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
|
||||
refusal_part = completed_resp.output[0].content[0]
|
||||
assert isinstance(refusal_part, ResponseOutputRefusal)
|
||||
assert refusal_part.refusal == "NoThanks"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_yields_events_for_tool_call(monkeypatch) -> None:
|
||||
"""
|
||||
Validate that `stream_response` emits the correct sequence of events when
|
||||
the model is streaming a function/tool call instead of plain text.
|
||||
The function call will be split across two chunks.
|
||||
"""
|
||||
# Simulate a single tool call with complete function name in first chunk
|
||||
# and arguments split across chunks (reflecting real API behavior)
|
||||
tool_call_delta1 = ChoiceDeltaToolCall(
|
||||
index=0,
|
||||
id="tool-id",
|
||||
function=ChoiceDeltaToolCallFunction(name="my_func", arguments="arg1"),
|
||||
type="function",
|
||||
)
|
||||
tool_call_delta2 = ChoiceDeltaToolCall(
|
||||
index=0,
|
||||
id="tool-id",
|
||||
function=ChoiceDeltaToolCallFunction(name=None, arguments="arg2"),
|
||||
type="function",
|
||||
)
|
||||
chunk1 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta1]))],
|
||||
)
|
||||
chunk2 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta2]))],
|
||||
usage=CompletionUsage(completion_tokens=1, prompt_tokens=1, total_tokens=2),
|
||||
)
|
||||
|
||||
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
for c in (chunk1, chunk2):
|
||||
yield c
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, fake_stream()
|
||||
|
||||
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
|
||||
model = LitellmProvider().get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
# Sequence should be: response.created, then after loop we expect function call-related events:
|
||||
# one response.output_item.added for function call, a response.function_call_arguments.delta,
|
||||
# a response.output_item.done, and finally response.completed.
|
||||
assert output_events[0].type == "response.created"
|
||||
# The next three events are about the tool call.
|
||||
assert output_events[1].type == "response.output_item.added"
|
||||
# The added item should be a ResponseFunctionToolCall.
|
||||
added_fn = output_events[1].item
|
||||
assert isinstance(added_fn, ResponseFunctionToolCall)
|
||||
assert added_fn.name == "my_func" # Name should be complete from first chunk
|
||||
assert added_fn.arguments == "" # Arguments start empty
|
||||
assert output_events[2].type == "response.function_call_arguments.delta"
|
||||
assert output_events[2].delta == "arg1" # First argument chunk
|
||||
assert output_events[3].type == "response.function_call_arguments.delta"
|
||||
assert output_events[3].delta == "arg2" # Second argument chunk
|
||||
assert output_events[4].type == "response.output_item.done"
|
||||
assert output_events[5].type == "response.completed"
|
||||
# Final function call should have complete arguments
|
||||
final_fn = output_events[4].item
|
||||
assert isinstance(final_fn, ResponseFunctionToolCall)
|
||||
assert final_fn.name == "my_func"
|
||||
assert final_fn.arguments == "arg1arg2"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_yields_real_time_function_call_arguments(monkeypatch) -> None:
|
||||
"""
|
||||
Validate that LiteLLM `stream_response` also emits function call arguments in real-time
|
||||
as they are received, ensuring consistent behavior across model providers.
|
||||
"""
|
||||
# Simulate realistic chunks: name first, then arguments incrementally
|
||||
tool_call_delta1 = ChoiceDeltaToolCall(
|
||||
index=0,
|
||||
id="litellm-call-456",
|
||||
function=ChoiceDeltaToolCallFunction(name="generate_code", arguments=""),
|
||||
type="function",
|
||||
)
|
||||
tool_call_delta2 = ChoiceDeltaToolCall(
|
||||
index=0,
|
||||
function=ChoiceDeltaToolCallFunction(arguments='{"language": "'),
|
||||
type="function",
|
||||
)
|
||||
tool_call_delta3 = ChoiceDeltaToolCall(
|
||||
index=0,
|
||||
function=ChoiceDeltaToolCallFunction(arguments='python", "task": "'),
|
||||
type="function",
|
||||
)
|
||||
tool_call_delta4 = ChoiceDeltaToolCall(
|
||||
index=0,
|
||||
function=ChoiceDeltaToolCallFunction(arguments='hello world"}'),
|
||||
type="function",
|
||||
)
|
||||
|
||||
chunk1 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta1]))],
|
||||
)
|
||||
chunk2 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta2]))],
|
||||
)
|
||||
chunk3 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta3]))],
|
||||
)
|
||||
chunk4 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(tool_calls=[tool_call_delta4]))],
|
||||
usage=CompletionUsage(completion_tokens=1, prompt_tokens=1, total_tokens=2),
|
||||
)
|
||||
|
||||
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
for c in (chunk1, chunk2, chunk3, chunk4):
|
||||
yield c
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, fake_stream()
|
||||
|
||||
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
|
||||
model = LitellmProvider().get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
|
||||
# Extract events by type
|
||||
function_args_delta_events = [
|
||||
e for e in output_events if e.type == "response.function_call_arguments.delta"
|
||||
]
|
||||
output_item_added_events = [e for e in output_events if e.type == "response.output_item.added"]
|
||||
|
||||
# Verify we got real-time streaming (3 argument delta events)
|
||||
assert len(function_args_delta_events) == 3
|
||||
assert len(output_item_added_events) == 1
|
||||
|
||||
# Verify the deltas were streamed correctly
|
||||
expected_deltas = ['{"language": "', 'python", "task": "', 'hello world"}']
|
||||
for i, delta_event in enumerate(function_args_delta_events):
|
||||
assert delta_event.delta == expected_deltas[i]
|
||||
|
||||
# Verify function call metadata
|
||||
added_event = output_item_added_events[0]
|
||||
assert isinstance(added_event.item, ResponseFunctionToolCall)
|
||||
assert added_event.item.name == "generate_code"
|
||||
assert added_event.item.call_id == "litellm-call-456"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_synthesizes_refusal_on_content_filter(monkeypatch) -> None:
|
||||
"""A stream that terminates with finish_reason == "content_filter" and no
|
||||
emitted content (as Anthropic-on-Bedrock does via LiteLLM) must synthesize a
|
||||
ResponseOutputRefusal so the completed response carries an explicit refusal
|
||||
rather than an empty assistant turn.
|
||||
|
||||
Mirrors the real Bedrock chunk shape: an empty-string content delta followed
|
||||
by a terminal content_filter chunk with no content. The empty "" delta must
|
||||
not open a text content part; the synthesized refusal must be the only
|
||||
content part, at the same index in the stream and in response.completed.
|
||||
"""
|
||||
chunk1 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(role="assistant", content=""))],
|
||||
)
|
||||
chunk2 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")],
|
||||
usage=CompletionUsage(
|
||||
completion_tokens=0,
|
||||
prompt_tokens=7,
|
||||
total_tokens=7,
|
||||
),
|
||||
)
|
||||
|
||||
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
for c in (chunk1, chunk2):
|
||||
yield c
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, fake_stream()
|
||||
|
||||
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
|
||||
model = LitellmProvider().get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
|
||||
types = [e.type for e in output_events]
|
||||
# Coherent refusal sequence: the message + refusal part are opened, a refusal
|
||||
# delta is emitted, and the parts/message are closed before completion.
|
||||
assert "response.output_item.added" in types
|
||||
assert "response.content_part.added" in types
|
||||
assert "response.refusal.delta" in types
|
||||
assert types[-1] == "response.completed"
|
||||
assert "response.output_item.done" in types
|
||||
|
||||
# The refusal delta carries a non-empty message.
|
||||
refusal_deltas = [e for e in output_events if e.type == "response.refusal.delta"]
|
||||
assert refusal_deltas and refusal_deltas[0].delta
|
||||
|
||||
# Event coherence: the assistant message is announced exactly once, and every
|
||||
# content part that is opened is also closed.
|
||||
assert types.count("response.output_item.added") == 1
|
||||
assert types.count("response.content_part.added") == types.count("response.content_part.done")
|
||||
|
||||
# The empty "" content delta must NOT open a text content part: no text part
|
||||
# events and no output_text.delta are emitted at all.
|
||||
assert "response.output_text.delta" not in types
|
||||
added_parts = [e for e in output_events if e.type == "response.content_part.added"]
|
||||
assert len(added_parts) == 1
|
||||
assert isinstance(added_parts[0].part, ResponseOutputRefusal)
|
||||
|
||||
# The completed response contains exactly one content part: the refusal.
|
||||
completed_event = output_events[-1]
|
||||
assert isinstance(completed_event, ResponseCompletedEvent)
|
||||
completed_resp = completed_event.response
|
||||
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
|
||||
assert len(completed_resp.output[0].content) == 1
|
||||
refusal_part = completed_resp.output[0].content[0]
|
||||
assert isinstance(refusal_part, ResponseOutputRefusal)
|
||||
assert refusal_part.refusal
|
||||
|
||||
# The refusal's streamed content_index matches its position in the completed
|
||||
# response (0), so raw-event replay and the final response stay aligned.
|
||||
assert added_parts[0].content_index == 0
|
||||
assert refusal_deltas[0].content_index == 0
|
||||
done_parts = [e for e in output_events if e.type == "response.content_part.done"]
|
||||
assert len(done_parts) == 1
|
||||
assert done_parts[0].content_index == 0
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_content_filter_does_not_clobber_text(monkeypatch) -> None:
|
||||
"""A content_filter finish_reason that arrives AFTER real text was streamed
|
||||
must not synthesize a refusal (the text stands)."""
|
||||
chunk1 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(content="answer"))],
|
||||
)
|
||||
chunk2 = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")],
|
||||
usage=CompletionUsage(completion_tokens=1, prompt_tokens=7, total_tokens=8),
|
||||
)
|
||||
|
||||
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
for c in (chunk1, chunk2):
|
||||
yield c
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, fake_stream()
|
||||
|
||||
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
|
||||
model = LitellmProvider().get_model("gpt-4")
|
||||
output_events = [
|
||||
event
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
]
|
||||
|
||||
assert "response.refusal.delta" not in [e.type for e in output_events]
|
||||
completed_event = output_events[-1]
|
||||
assert isinstance(completed_event, ResponseCompletedEvent)
|
||||
completed_resp = completed_event.response
|
||||
assert isinstance(completed_resp.output[0], ResponseOutputMessage)
|
||||
assert isinstance(completed_resp.output[0].content[0], ResponseOutputText)
|
||||
assert completed_resp.output[0].content[0].text == "answer"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_content_filter_refusal_after_reasoning(monkeypatch) -> None:
|
||||
"""A content_filter turn preceded by reasoning must still place the
|
||||
synthesized refusal at content_index 0 of the assistant message. Reasoning
|
||||
is a *separate* output item (it shifts the message's output_index, not its
|
||||
content_index), so the refusal — the sole content part — stays at
|
||||
content_index 0 in both the stream and response.completed."""
|
||||
reasoning_delta = ChoiceDelta(role="assistant", content=None)
|
||||
# reasoning_content is a provider extra field the handler reads via hasattr.
|
||||
reasoning_delta.reasoning_content = "thinking..." # type: ignore[attr-defined]
|
||||
chunk_reasoning = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=reasoning_delta)],
|
||||
)
|
||||
chunk_empty = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(content=""))],
|
||||
)
|
||||
chunk_filter = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="fake",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=ChoiceDelta(), finish_reason="content_filter")],
|
||||
usage=CompletionUsage(completion_tokens=0, prompt_tokens=7, total_tokens=7),
|
||||
)
|
||||
|
||||
async def fake_stream() -> AsyncIterator[ChatCompletionChunk]:
|
||||
for c in (chunk_reasoning, chunk_empty, chunk_filter):
|
||||
yield c
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, fake_stream()
|
||||
|
||||
monkeypatch.setattr(LitellmModel, "_fetch_response", patched_fetch_response)
|
||||
model = LitellmProvider().get_model("gpt-4")
|
||||
output_events = [
|
||||
event
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
]
|
||||
|
||||
# A reasoning item was produced as a separate output item.
|
||||
completed_event = output_events[-1]
|
||||
assert isinstance(completed_event, ResponseCompletedEvent)
|
||||
completed_resp = completed_event.response
|
||||
assert isinstance(completed_resp.output[0], ResponseReasoningItem)
|
||||
assistant_msg = completed_resp.output[1]
|
||||
assert isinstance(assistant_msg, ResponseOutputMessage)
|
||||
# The refusal is the sole content part of the assistant message, at index 0.
|
||||
assert len(assistant_msg.content) == 1
|
||||
assert isinstance(assistant_msg.content[0], ResponseOutputRefusal)
|
||||
|
||||
# The assistant message's output_index is 1 (after the reasoning item), and
|
||||
# every refusal event uses that output_index and content_index 0 — matching
|
||||
# the refusal's position in response.completed.
|
||||
added = [
|
||||
e
|
||||
for e in output_events
|
||||
if isinstance(e, ResponseContentPartAddedEvent)
|
||||
and isinstance(e.part, ResponseOutputRefusal)
|
||||
]
|
||||
deltas = [e for e in output_events if isinstance(e, ResponseRefusalDeltaEvent)]
|
||||
assert len(added) == 1
|
||||
assert added[0].content_index == 0
|
||||
assert added[0].output_index == 1
|
||||
assert deltas and all(d.content_index == 0 and d.output_index == 1 for d in deltas)
|
||||
# The empty "" delta still opens no text part.
|
||||
assert "response.output_text.delta" not in [e.type for e in output_events]
|
||||
@@ -0,0 +1,89 @@
|
||||
import litellm
|
||||
import pytest
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal
|
||||
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import ModelTracing
|
||||
|
||||
|
||||
async def _get_response(monkeypatch, *, finish_reason, content):
|
||||
"""Drive get_response against a mocked litellm completion and return the items."""
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
msg = Message(role="assistant", content=content)
|
||||
choice = Choices(index=0, finish_reason=finish_reason, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
model = LitellmModel(model="test-model")
|
||||
return await model.get_response(
|
||||
system_instructions=None,
|
||||
input=[],
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_content_filter_finish_reason_surfaces_refusal(monkeypatch):
|
||||
"""A content-filter block (empty message, finish_reason=content_filter) must
|
||||
become an explicit ResponseOutputRefusal, not zero output items.
|
||||
|
||||
Some providers (e.g. Anthropic on Amazon Bedrock) signal a safety block only
|
||||
via ``finish_reason == "content_filter"`` with an empty message and no
|
||||
``refusal`` field; without this the turn is indistinguishable from an empty
|
||||
response and drives agent loops into fruitless retries.
|
||||
"""
|
||||
resp = await _get_response(monkeypatch, finish_reason="content_filter", content="")
|
||||
|
||||
refusals = [
|
||||
content
|
||||
for item in resp.output
|
||||
if isinstance(item, ResponseOutputMessage)
|
||||
for content in item.content
|
||||
if isinstance(content, ResponseOutputRefusal)
|
||||
]
|
||||
assert refusals, f"expected a refusal item, got: {resp.output}"
|
||||
assert refusals[0].refusal # non-empty message
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_content_filter_does_not_clobber_real_content(monkeypatch):
|
||||
"""A content_filter finish_reason that still carries text is left alone — we
|
||||
only synthesize a refusal when the message is genuinely empty."""
|
||||
resp = await _get_response(
|
||||
monkeypatch, finish_reason="content_filter", content="here is the answer"
|
||||
)
|
||||
|
||||
refusals = [
|
||||
content
|
||||
for item in resp.output
|
||||
if isinstance(item, ResponseOutputMessage)
|
||||
for content in item.content
|
||||
if isinstance(content, ResponseOutputRefusal)
|
||||
]
|
||||
assert not refusals, "should not synthesize a refusal when content is present"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_normal_stop_is_unaffected(monkeypatch):
|
||||
"""A normal completion is unchanged — no spurious refusal."""
|
||||
resp = await _get_response(monkeypatch, finish_reason="stop", content="all good")
|
||||
|
||||
refusals = [
|
||||
content
|
||||
for item in resp.output
|
||||
if isinstance(item, ResponseOutputMessage)
|
||||
for content in item.content
|
||||
if isinstance(content, ResponseOutputRefusal)
|
||||
]
|
||||
assert not refusals
|
||||
@@ -0,0 +1,265 @@
|
||||
import logging
|
||||
|
||||
import litellm
|
||||
import pytest
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import ModelTracing
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_extra_body_is_forwarded(monkeypatch):
|
||||
"""
|
||||
Forward `extra_body` via LiteLLM's dedicated kwarg.
|
||||
|
||||
This ensures that provider-specific request fields stay nested under `extra_body`
|
||||
so LiteLLM can merge them into the upstream request body itself.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="ok")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
settings = ModelSettings(
|
||||
temperature=0.1, extra_body={"cached_content": "some_cache", "foo": 123}
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=[],
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert captured["extra_body"] == {"cached_content": "some_cache", "foo": 123}
|
||||
assert "cached_content" not in captured
|
||||
assert "foo" not in captured
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_extra_body_reasoning_effort_is_promoted(monkeypatch):
|
||||
"""
|
||||
Ensure reasoning_effort from extra_body is promoted to the top-level parameter.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="ok")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
# GitHub issue context: https://github.com/openai/openai-agents-python/issues/1764.
|
||||
settings = ModelSettings(
|
||||
extra_body={"reasoning_effort": "none", "cached_content": "some_cache"}
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=[],
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert captured["reasoning_effort"] == "none"
|
||||
assert captured["extra_body"] == {"cached_content": "some_cache"}
|
||||
assert settings.extra_body == {"reasoning_effort": "none", "cached_content": "some_cache"}
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_reasoning_effort_prefers_model_settings(monkeypatch):
|
||||
"""
|
||||
Verify explicit ModelSettings.reasoning takes precedence over extra_body entries.
|
||||
"""
|
||||
from openai.types.shared import Reasoning
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="ok")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
settings = ModelSettings(
|
||||
reasoning=Reasoning(effort="low"),
|
||||
extra_body={"reasoning_effort": "high"},
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=[],
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
# reasoning_effort is string when no summary is provided (backward compatible)
|
||||
assert captured["reasoning_effort"] == "low"
|
||||
assert "extra_body" not in captured
|
||||
assert settings.extra_body == {"reasoning_effort": "high"}
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_extra_body_reasoning_effort_overrides_extra_args(monkeypatch):
|
||||
"""
|
||||
Ensure extra_body reasoning_effort wins over extra_args when both are provided.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="ok")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
# GitHub issue context: https://github.com/openai/openai-agents-python/issues/1764.
|
||||
settings = ModelSettings(
|
||||
extra_body={"reasoning_effort": "none"},
|
||||
extra_args={"reasoning_effort": "low", "custom_param": "custom"},
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=[],
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert captured["reasoning_effort"] == "none"
|
||||
assert captured["custom_param"] == "custom"
|
||||
assert "extra_body" not in captured
|
||||
assert settings.extra_args == {"reasoning_effort": "low", "custom_param": "custom"}
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_extra_body_metadata_stays_nested(monkeypatch):
|
||||
"""
|
||||
Keep extra_body metadata nested even when top-level metadata is also set.
|
||||
|
||||
LiteLLM resolves top-level metadata and extra_body separately. Flattening the nested
|
||||
metadata dict loses the caller's intended request shape for OpenAI-compatible proxies.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="ok")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
settings = ModelSettings(
|
||||
metadata={"sdk": "agents"},
|
||||
extra_body={
|
||||
"metadata": {"trace_user_id": "user-123", "generation_id": "gen-456"},
|
||||
"cached_content": "some_cache",
|
||||
},
|
||||
)
|
||||
model = LitellmModel(model="test-model")
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=[],
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert captured["metadata"] == {"sdk": "agents"}
|
||||
assert captured["extra_body"] == {
|
||||
"metadata": {"trace_user_id": "user-123", "generation_id": "gen-456"},
|
||||
"cached_content": "some_cache",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model_name",
|
||||
[
|
||||
"openai/gpt-5-mini",
|
||||
"anthropic/claude-sonnet-4-5",
|
||||
"gemini/gemini-2.5-pro",
|
||||
],
|
||||
)
|
||||
async def test_reasoning_summary_uses_scalar_effort_and_warns(
|
||||
monkeypatch, caplog: pytest.LogCaptureFixture, model_name: str
|
||||
):
|
||||
"""
|
||||
Ensure reasoning.summary does not change the LiteLLM chat-completions argument shape.
|
||||
|
||||
LitellmModel should continue to pass a scalar reasoning_effort value and warn that summary
|
||||
is ignored on this path, regardless of the provider encoded in the model string.
|
||||
"""
|
||||
from openai.types.shared import Reasoning
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = Message(role="assistant", content="ok")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
settings = ModelSettings(
|
||||
reasoning=Reasoning(effort="medium", summary="auto"),
|
||||
)
|
||||
model = LitellmModel(model=model_name)
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="openai.agents"):
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=[],
|
||||
model_settings=settings,
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assert captured["reasoning_effort"] == "medium"
|
||||
warning_messages = [
|
||||
record.message
|
||||
for record in caplog.records
|
||||
if "does not forward Reasoning.summary" in record.message
|
||||
]
|
||||
assert len(warning_messages) == 1
|
||||
@@ -0,0 +1,29 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("litellm")
|
||||
|
||||
|
||||
def test_litellm_logging_patch_env_var_controls_application(monkeypatch):
|
||||
"""Assert the serializer patch only applies when the env var is enabled."""
|
||||
litellm_logging = importlib.import_module("litellm.litellm_core_utils.litellm_logging")
|
||||
litellm_model = importlib.import_module("agents.extensions.models.litellm_model")
|
||||
|
||||
monkeypatch.delenv("OPENAI_AGENTS_ENABLE_LITELLM_SERIALIZER_PATCH", raising=False)
|
||||
litellm_logging = importlib.reload(litellm_logging)
|
||||
importlib.reload(litellm_model)
|
||||
|
||||
assert hasattr(
|
||||
litellm_logging,
|
||||
"_extract_response_obj_and_hidden_params",
|
||||
), "LiteLLM removed _extract_response_obj_and_hidden_params; revisit warning patch."
|
||||
assert getattr(litellm_logging, "_openai_agents_patched_serializer_warnings", False) is False
|
||||
|
||||
monkeypatch.setenv("OPENAI_AGENTS_ENABLE_LITELLM_SERIALIZER_PATCH", "true")
|
||||
litellm_logging = importlib.reload(litellm_logging)
|
||||
importlib.reload(litellm_model)
|
||||
|
||||
assert getattr(litellm_logging, "_openai_agents_patched_serializer_warnings", False) is True
|
||||
@@ -0,0 +1,89 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from agents import ModelSettings, ModelTracing, __version__
|
||||
from agents.models.chatcmpl_helpers import HEADERS_OVERRIDE
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("override_ua", [None, "test_user_agent"])
|
||||
async def test_user_agent_header_litellm(override_ua: str | None, monkeypatch):
|
||||
called_kwargs: dict[str, Any] = {}
|
||||
expected_ua = override_ua or f"Agents/Python {__version__}"
|
||||
|
||||
import importlib
|
||||
import sys
|
||||
import types as pytypes
|
||||
|
||||
litellm_fake: Any = pytypes.ModuleType("litellm")
|
||||
|
||||
class DummyMessage:
|
||||
role = "assistant"
|
||||
content = "Hello"
|
||||
tool_calls: list[Any] | None = None
|
||||
|
||||
def get(self, _key, _default=None):
|
||||
return None
|
||||
|
||||
def model_dump(self):
|
||||
return {"role": self.role, "content": self.content}
|
||||
|
||||
class Choices: # noqa: N801 - mimic litellm naming
|
||||
def __init__(self):
|
||||
self.message = DummyMessage()
|
||||
|
||||
class DummyModelResponse:
|
||||
def __init__(self):
|
||||
self.choices = [Choices()]
|
||||
|
||||
async def acompletion(**kwargs):
|
||||
nonlocal called_kwargs
|
||||
called_kwargs = kwargs
|
||||
return DummyModelResponse()
|
||||
|
||||
utils_ns = pytypes.SimpleNamespace()
|
||||
utils_ns.Choices = Choices
|
||||
utils_ns.ModelResponse = DummyModelResponse
|
||||
|
||||
litellm_types = pytypes.SimpleNamespace(
|
||||
utils=utils_ns,
|
||||
llms=pytypes.SimpleNamespace(openai=pytypes.SimpleNamespace(ChatCompletionAnnotation=dict)),
|
||||
)
|
||||
litellm_fake.acompletion = acompletion
|
||||
litellm_fake.types = litellm_types
|
||||
|
||||
monkeypatch.setitem(sys.modules, "litellm", litellm_fake)
|
||||
|
||||
litellm_mod = importlib.import_module("agents.extensions.models.litellm_model")
|
||||
monkeypatch.setattr(litellm_mod, "litellm", litellm_fake, raising=True)
|
||||
LitellmModel = litellm_mod.LitellmModel
|
||||
|
||||
model = LitellmModel(model="gpt-4")
|
||||
|
||||
if override_ua is not None:
|
||||
token = HEADERS_OVERRIDE.set({"User-Agent": override_ua})
|
||||
else:
|
||||
token = None
|
||||
try:
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input="hi",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
finally:
|
||||
if token is not None:
|
||||
HEADERS_OVERRIDE.reset(token)
|
||||
|
||||
assert "extra_headers" in called_kwargs
|
||||
assert called_kwargs["extra_headers"]["User-Agent"] == expected_ua
|
||||
@@ -0,0 +1,209 @@
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
|
||||
from agents import (
|
||||
Agent,
|
||||
MultiProvider,
|
||||
OpenAIResponsesModel,
|
||||
OpenAIResponsesWSModel,
|
||||
RunConfig,
|
||||
UserError,
|
||||
)
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
from agents.models.multi_provider import MultiProviderMap
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from agents.run_internal.run_loop import get_model
|
||||
|
||||
|
||||
def test_no_prefix_is_openai():
|
||||
agent = Agent(model="gpt-4o", instructions="", name="test")
|
||||
model = get_model(agent, RunConfig())
|
||||
assert isinstance(model, OpenAIResponsesModel)
|
||||
|
||||
|
||||
def test_openai_prefix_is_openai():
|
||||
agent = Agent(model="openai/gpt-4o", instructions="", name="test")
|
||||
model = get_model(agent, RunConfig())
|
||||
assert isinstance(model, OpenAIResponsesModel)
|
||||
|
||||
|
||||
def test_litellm_prefix_is_litellm():
|
||||
agent = Agent(model="litellm/foo/bar", instructions="", name="test")
|
||||
model = get_model(agent, RunConfig())
|
||||
assert isinstance(model, LitellmModel)
|
||||
|
||||
|
||||
def test_any_llm_prefix_uses_any_llm_provider(monkeypatch):
|
||||
import sys
|
||||
import types as pytypes
|
||||
|
||||
captured_model: dict[str, Any] = {}
|
||||
|
||||
class FakeAnyLLMModel:
|
||||
pass
|
||||
|
||||
class FakeAnyLLMProvider:
|
||||
def get_model(self, model_name):
|
||||
captured_model["value"] = model_name
|
||||
return FakeAnyLLMModel()
|
||||
|
||||
fake_module: Any = pytypes.ModuleType("agents.extensions.models.any_llm_provider")
|
||||
fake_module.AnyLLMProvider = FakeAnyLLMProvider
|
||||
monkeypatch.setitem(sys.modules, "agents.extensions.models.any_llm_provider", fake_module)
|
||||
|
||||
agent = Agent(model="any-llm/openrouter/openai/gpt-5.4-mini", instructions="", name="test")
|
||||
model = get_model(agent, RunConfig())
|
||||
assert isinstance(model, FakeAnyLLMModel)
|
||||
assert captured_model["value"] == "openrouter/openai/gpt-5.4-mini"
|
||||
|
||||
|
||||
def test_no_prefix_can_use_openai_responses_websocket():
|
||||
agent = Agent(model="gpt-4o", instructions="", name="test")
|
||||
model = get_model(
|
||||
agent,
|
||||
RunConfig(model_provider=MultiProvider(openai_use_responses_websocket=True)),
|
||||
)
|
||||
assert isinstance(model, OpenAIResponsesWSModel)
|
||||
|
||||
|
||||
def test_openai_prefix_can_use_openai_responses_websocket():
|
||||
agent = Agent(model="openai/gpt-4o", instructions="", name="test")
|
||||
model = get_model(
|
||||
agent,
|
||||
RunConfig(model_provider=MultiProvider(openai_use_responses_websocket=True)),
|
||||
)
|
||||
assert isinstance(model, OpenAIResponsesWSModel)
|
||||
|
||||
|
||||
def test_multi_provider_passes_websocket_base_url_to_openai_provider(monkeypatch):
|
||||
captured_kwargs = {}
|
||||
|
||||
class FakeOpenAIProvider:
|
||||
def __init__(self, **kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
|
||||
def get_model(self, model_name):
|
||||
raise AssertionError("This test only verifies constructor passthrough.")
|
||||
|
||||
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
|
||||
|
||||
MultiProvider(openai_websocket_base_url="wss://proxy.example.test/v1")
|
||||
assert captured_kwargs["websocket_base_url"] == "wss://proxy.example.test/v1"
|
||||
|
||||
|
||||
def test_multi_provider_forwards_openai_buffer_streamed_tool_calls_to_chat_model():
|
||||
provider = MultiProvider(
|
||||
openai_client=cast(Any, object()),
|
||||
openai_use_responses=False,
|
||||
openai_buffer_streamed_tool_calls=True,
|
||||
)
|
||||
|
||||
model = provider.get_model("gpt-4o")
|
||||
|
||||
assert isinstance(model, OpenAIChatCompletionsModel)
|
||||
assert model._buffer_streamed_tool_calls is True
|
||||
|
||||
|
||||
def test_openai_prefix_defaults_to_alias_mode(monkeypatch):
|
||||
captured_model: dict[str, Any] = {}
|
||||
|
||||
class FakeOpenAIProvider:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
def get_model(self, model_name):
|
||||
captured_model["value"] = model_name
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
|
||||
|
||||
provider = MultiProvider()
|
||||
provider.get_model("openai/gpt-4o")
|
||||
assert captured_model["value"] == "gpt-4o"
|
||||
|
||||
|
||||
def test_openai_prefix_can_be_preserved_as_literal_model_id(monkeypatch):
|
||||
captured_model: dict[str, Any] = {}
|
||||
|
||||
class FakeOpenAIProvider:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
def get_model(self, model_name):
|
||||
captured_model["value"] = model_name
|
||||
return object()
|
||||
|
||||
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
|
||||
|
||||
provider = MultiProvider(openai_prefix_mode="model_id")
|
||||
provider.get_model("openai/gpt-4o")
|
||||
assert captured_model["value"] == "openai/gpt-4o"
|
||||
|
||||
|
||||
def test_unknown_prefix_defaults_to_error():
|
||||
provider = MultiProvider()
|
||||
|
||||
with pytest.raises(UserError, match="Unknown prefix: openrouter"):
|
||||
provider.get_model("openrouter/openai/gpt-4o")
|
||||
|
||||
|
||||
def test_unknown_prefix_can_be_preserved_for_openai_compatible_model_ids(monkeypatch):
|
||||
captured_model: dict[str, Any] = {}
|
||||
captured_result: dict[str, Any] = {}
|
||||
|
||||
class FakeOpenAIProvider:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
def get_model(self, model_name):
|
||||
captured_model["value"] = model_name
|
||||
fake_model = object()
|
||||
captured_result["value"] = fake_model
|
||||
return fake_model
|
||||
|
||||
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
|
||||
|
||||
provider = MultiProvider(unknown_prefix_mode="model_id")
|
||||
result = provider.get_model("openrouter/openai/gpt-4o")
|
||||
assert result is captured_result["value"]
|
||||
assert captured_model["value"] == "openrouter/openai/gpt-4o"
|
||||
|
||||
|
||||
def test_provider_map_entries_override_openai_prefix_mode(monkeypatch):
|
||||
captured_model: dict[str, Any] = {}
|
||||
|
||||
class FakeCustomProvider:
|
||||
def get_model(self, model_name):
|
||||
captured_model["value"] = model_name
|
||||
return object()
|
||||
|
||||
class FakeOpenAIProvider:
|
||||
def __init__(self, **kwargs):
|
||||
pass
|
||||
|
||||
def get_model(self, model_name):
|
||||
raise AssertionError("Expected the explicit provider_map entry to win.")
|
||||
|
||||
monkeypatch.setattr("agents.models.multi_provider.OpenAIProvider", FakeOpenAIProvider)
|
||||
|
||||
provider_map = MultiProviderMap()
|
||||
provider_map.add_provider("openai", cast(Any, FakeCustomProvider()))
|
||||
|
||||
provider = MultiProvider(
|
||||
provider_map=provider_map,
|
||||
openai_prefix_mode="model_id",
|
||||
)
|
||||
provider.get_model("openai/gpt-4o")
|
||||
assert captured_model["value"] == "gpt-4o"
|
||||
|
||||
|
||||
def test_multi_provider_rejects_invalid_prefix_modes():
|
||||
bad_openai_prefix_mode: Any = "invalid"
|
||||
bad_unknown_prefix_mode: Any = "invalid"
|
||||
|
||||
with pytest.raises(UserError, match="openai_prefix_mode"):
|
||||
MultiProvider(openai_prefix_mode=bad_openai_prefix_mode)
|
||||
|
||||
with pytest.raises(UserError, match="unknown_prefix_mode"):
|
||||
MultiProvider(unknown_prefix_mode=bad_unknown_prefix_mode)
|
||||
@@ -0,0 +1,186 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Iterator
|
||||
from typing import Any, cast
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from openai import omit
|
||||
from openai.types.chat.chat_completion import ChatCompletion
|
||||
|
||||
from agents import (
|
||||
ModelSettings,
|
||||
ModelTracing,
|
||||
OpenAIChatCompletionsModel,
|
||||
OpenAIResponsesModel,
|
||||
generation_span,
|
||||
)
|
||||
from agents.models import (
|
||||
openai_chatcompletions as chat_module,
|
||||
openai_responses as responses_module,
|
||||
)
|
||||
|
||||
|
||||
class _SingleUseIterable:
|
||||
"""Helper iterable that raises if iterated more than once."""
|
||||
|
||||
def __init__(self, values: list[object]) -> None:
|
||||
self._values = list(values)
|
||||
self.iterations = 0
|
||||
|
||||
def __iter__(self) -> Iterator[object]:
|
||||
if self.iterations:
|
||||
raise RuntimeError("Iterable should have been materialized exactly once.")
|
||||
self.iterations += 1
|
||||
yield from self._values
|
||||
|
||||
|
||||
def _force_materialization(value: object) -> None:
|
||||
if isinstance(value, dict):
|
||||
for nested in value.values():
|
||||
_force_materialization(nested)
|
||||
elif isinstance(value, list):
|
||||
for nested in value:
|
||||
_force_materialization(nested)
|
||||
elif isinstance(value, Iterable) and not isinstance(value, str | bytes | bytearray):
|
||||
list(value)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_chat_completions_materializes_iterator_payload(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
message_iter = _SingleUseIterable([{"type": "text", "text": "hi"}])
|
||||
tool_iter = _SingleUseIterable([{"type": "string"}])
|
||||
|
||||
chat_converter = cast(Any, chat_module).Converter
|
||||
|
||||
monkeypatch.setattr(
|
||||
chat_converter,
|
||||
"items_to_messages",
|
||||
classmethod(lambda _cls, _input, **kwargs: [{"role": "user", "content": message_iter}]),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
chat_converter,
|
||||
"tool_to_openai",
|
||||
classmethod(
|
||||
lambda _cls, _tool: {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "dummy",
|
||||
"parameters": {"properties": tool_iter},
|
||||
},
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
captured_kwargs: dict[str, Any] = {}
|
||||
|
||||
class DummyCompletions:
|
||||
async def create(self, **kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
_force_materialization(kwargs["messages"])
|
||||
if kwargs["tools"] is not omit:
|
||||
_force_materialization(kwargs["tools"])
|
||||
return ChatCompletion(
|
||||
id="dummy-id",
|
||||
created=0,
|
||||
model="gpt-4",
|
||||
object="chat.completion",
|
||||
choices=[],
|
||||
usage=None,
|
||||
)
|
||||
|
||||
class DummyClient:
|
||||
def __init__(self) -> None:
|
||||
self.chat = type("_Chat", (), {"completions": DummyCompletions()})()
|
||||
self.base_url = httpx.URL("http://example.test")
|
||||
|
||||
model = OpenAIChatCompletionsModel(model="gpt-4", openai_client=DummyClient()) # type: ignore[arg-type]
|
||||
|
||||
with generation_span(disabled=True) as span:
|
||||
await cast(Any, model)._fetch_response(
|
||||
system_instructions=None,
|
||||
input="ignored",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[object()],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
span=span,
|
||||
tracing=ModelTracing.DISABLED,
|
||||
stream=False,
|
||||
)
|
||||
|
||||
assert message_iter.iterations == 1
|
||||
assert tool_iter.iterations == 1
|
||||
assert isinstance(captured_kwargs["messages"][0]["content"], list)
|
||||
assert isinstance(captured_kwargs["tools"][0]["function"]["parameters"]["properties"], list)
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_materializes_iterator_payload(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
input_iter = _SingleUseIterable([{"type": "input_text", "text": "hello"}])
|
||||
tool_iter = _SingleUseIterable([{"type": "string"}])
|
||||
|
||||
responses_item_helpers = cast(Any, responses_module).ItemHelpers
|
||||
responses_converter = cast(Any, responses_module).Converter
|
||||
|
||||
monkeypatch.setattr(
|
||||
responses_item_helpers,
|
||||
"input_to_new_input_list",
|
||||
classmethod(lambda _cls, _input: [{"role": "user", "content": input_iter}]),
|
||||
)
|
||||
|
||||
converted_tools = responses_module.ConvertedTools(
|
||||
tools=[
|
||||
cast(
|
||||
Any,
|
||||
{
|
||||
"type": "function",
|
||||
"name": "dummy",
|
||||
"parameters": {"properties": tool_iter},
|
||||
},
|
||||
)
|
||||
],
|
||||
includes=[],
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
responses_converter,
|
||||
"convert_tools",
|
||||
classmethod(lambda _cls, _tools, _handoffs, **_kwargs: converted_tools),
|
||||
)
|
||||
|
||||
captured_kwargs: dict[str, Any] = {}
|
||||
|
||||
class DummyResponses:
|
||||
async def create(self, **kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
_force_materialization(kwargs["input"])
|
||||
_force_materialization(kwargs["tools"])
|
||||
return object()
|
||||
|
||||
class DummyClient:
|
||||
def __init__(self) -> None:
|
||||
self.responses = DummyResponses()
|
||||
|
||||
model = OpenAIResponsesModel(model="gpt-4.1", openai_client=DummyClient()) # type: ignore[arg-type]
|
||||
|
||||
await cast(Any, model)._fetch_response(
|
||||
system_instructions=None,
|
||||
input="ignored",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
stream=False,
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
assert input_iter.iterations == 1
|
||||
assert tool_iter.iterations == 1
|
||||
assert isinstance(captured_kwargs["input"][0]["content"], list)
|
||||
assert isinstance(captured_kwargs["tools"][0]["parameters"]["properties"], list)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,756 @@
|
||||
# Copyright (c) OpenAI
|
||||
#
|
||||
# Licensed under the MIT License.
|
||||
# See LICENSE file in the project root for full license information.
|
||||
|
||||
"""
|
||||
Unit tests for the internal `Converter` class defined in
|
||||
`agents.models.openai_chatcompletions`. The converter is responsible for
|
||||
translating between internal "item" structures (e.g., `ResponseOutputMessage`
|
||||
and related types from `openai.types.responses`) and the ChatCompletion message
|
||||
structures defined by the OpenAI client library.
|
||||
|
||||
These tests exercise both conversion directions:
|
||||
|
||||
- `Converter.message_to_output_items` turns a `ChatCompletionMessage` (as
|
||||
returned by the OpenAI API) into a list of `ResponseOutputItem` instances.
|
||||
|
||||
- `Converter.items_to_messages` takes in either a simple string prompt, or a
|
||||
list of input/output items such as `ResponseOutputMessage` and
|
||||
`ResponseFunctionToolCallParam` dicts, and constructs a list of
|
||||
`ChatCompletionMessageParam` dicts suitable for sending back to the API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
import pytest
|
||||
from openai import omit
|
||||
from openai.types.chat import ChatCompletionMessage, ChatCompletionMessageFunctionToolCall
|
||||
from openai.types.chat.chat_completion_message_custom_tool_call import (
|
||||
ChatCompletionMessageCustomToolCall,
|
||||
Custom,
|
||||
)
|
||||
from openai.types.chat.chat_completion_message_tool_call import Function
|
||||
from openai.types.responses import (
|
||||
ResponseFunctionToolCall,
|
||||
ResponseFunctionToolCallParam,
|
||||
ResponseInputAudioParam,
|
||||
ResponseInputTextParam,
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputRefusal,
|
||||
ResponseOutputText,
|
||||
)
|
||||
from openai.types.responses.response_input_item_param import FunctionCallOutput
|
||||
|
||||
from agents.agent_output import AgentOutputSchema
|
||||
from agents.exceptions import UserError
|
||||
from agents.items import TResponseInputItem
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
from agents.models.fake_id import FAKE_RESPONSES_ID
|
||||
|
||||
|
||||
def test_message_to_output_items_with_text_only():
|
||||
"""
|
||||
Make sure a simple ChatCompletionMessage with string content is converted
|
||||
into a single ResponseOutputMessage containing one ResponseOutputText.
|
||||
"""
|
||||
msg = ChatCompletionMessage(role="assistant", content="Hello")
|
||||
items = Converter.message_to_output_items(msg)
|
||||
# Expect exactly one output item (the message)
|
||||
assert len(items) == 1
|
||||
message_item = cast(ResponseOutputMessage, items[0])
|
||||
assert message_item.id == FAKE_RESPONSES_ID
|
||||
assert message_item.role == "assistant"
|
||||
assert message_item.type == "message"
|
||||
assert message_item.status == "completed"
|
||||
# Message content should have exactly one text part with the same text.
|
||||
assert len(message_item.content) == 1
|
||||
text_part = cast(ResponseOutputText, message_item.content[0])
|
||||
assert text_part.type == "output_text"
|
||||
assert text_part.text == "Hello"
|
||||
|
||||
|
||||
def test_message_to_output_items_with_refusal():
|
||||
"""
|
||||
Make sure a message with a refusal string produces a ResponseOutputMessage
|
||||
with a ResponseOutputRefusal content part.
|
||||
"""
|
||||
msg = ChatCompletionMessage(role="assistant", refusal="I'm sorry")
|
||||
items = Converter.message_to_output_items(msg)
|
||||
assert len(items) == 1
|
||||
message_item = cast(ResponseOutputMessage, items[0])
|
||||
assert len(message_item.content) == 1
|
||||
refusal_part = cast(ResponseOutputRefusal, message_item.content[0])
|
||||
assert refusal_part.type == "refusal"
|
||||
assert refusal_part.refusal == "I'm sorry"
|
||||
|
||||
|
||||
def test_message_to_output_items_with_tool_call():
|
||||
"""
|
||||
If the ChatCompletionMessage contains one or more tool_calls, they should
|
||||
be reflected as separate `ResponseFunctionToolCall` items appended after
|
||||
the message item.
|
||||
"""
|
||||
tool_call = ChatCompletionMessageFunctionToolCall(
|
||||
id="tool1",
|
||||
type="function",
|
||||
function=Function(name="myfn", arguments='{"x":1}'),
|
||||
)
|
||||
msg = ChatCompletionMessage(role="assistant", content="Hi", tool_calls=[tool_call])
|
||||
items = Converter.message_to_output_items(msg)
|
||||
# Should produce a message item followed by one function tool call item
|
||||
assert len(items) == 2
|
||||
message_item = cast(ResponseOutputMessage, items[0])
|
||||
assert isinstance(message_item, ResponseOutputMessage)
|
||||
fn_call_item = cast(ResponseFunctionToolCall, items[1])
|
||||
assert fn_call_item.id == FAKE_RESPONSES_ID
|
||||
assert fn_call_item.call_id == tool_call.id
|
||||
assert fn_call_item.name == tool_call.function.name
|
||||
assert fn_call_item.arguments == tool_call.function.arguments
|
||||
assert fn_call_item.type == "function_call"
|
||||
|
||||
|
||||
def test_message_to_output_items_with_custom_tool_call_keeps_default_compatibility():
|
||||
"""Custom tool calls should keep the default Chat Completions behavior."""
|
||||
tool_call = ChatCompletionMessageCustomToolCall(
|
||||
id="tool1",
|
||||
type="custom",
|
||||
custom=Custom(name="raw_tool", input="payload"),
|
||||
)
|
||||
msg = ChatCompletionMessage(role="assistant", tool_calls=[tool_call])
|
||||
|
||||
assert Converter.message_to_output_items(msg) == []
|
||||
|
||||
|
||||
def test_message_to_output_items_with_custom_tool_call_raises_in_strict_mode():
|
||||
"""Strict validation should fail explicitly instead of dropping custom tool calls."""
|
||||
tool_call = ChatCompletionMessageCustomToolCall(
|
||||
id="tool1",
|
||||
type="custom",
|
||||
custom=Custom(name="raw_tool", input="payload"),
|
||||
)
|
||||
msg = ChatCompletionMessage(role="assistant", tool_calls=[tool_call])
|
||||
|
||||
with pytest.raises(UserError, match="Custom tool calls are not supported"):
|
||||
Converter.message_to_output_items(msg, strict_feature_validation=True)
|
||||
|
||||
|
||||
def test_message_to_output_items_with_mixed_custom_tool_call_raises_in_strict_mode():
|
||||
"""Strict validation should not partially hide an unsupported custom tool call."""
|
||||
function_tool_call = ChatCompletionMessageFunctionToolCall(
|
||||
id="function-tool",
|
||||
type="function",
|
||||
function=Function(name="myfn", arguments='{"x":1}'),
|
||||
)
|
||||
custom_tool_call = ChatCompletionMessageCustomToolCall(
|
||||
id="custom-tool",
|
||||
type="custom",
|
||||
custom=Custom(name="raw_tool", input="payload"),
|
||||
)
|
||||
msg = ChatCompletionMessage(
|
||||
role="assistant",
|
||||
tool_calls=[function_tool_call, custom_tool_call],
|
||||
)
|
||||
|
||||
with pytest.raises(UserError, match="Custom tool calls are not supported"):
|
||||
Converter.message_to_output_items(msg, strict_feature_validation=True)
|
||||
|
||||
|
||||
def test_items_to_messages_with_string_user_content():
|
||||
"""
|
||||
A simple string as the items argument should be converted into a user
|
||||
message param dict with the same content.
|
||||
"""
|
||||
result = Converter.items_to_messages("Ask me anything")
|
||||
assert isinstance(result, list)
|
||||
assert len(result) == 1
|
||||
msg = result[0]
|
||||
assert msg["role"] == "user"
|
||||
assert msg["content"] == "Ask me anything"
|
||||
|
||||
|
||||
def test_items_to_messages_with_easy_input_message():
|
||||
"""
|
||||
Given an easy input message dict (just role/content), the converter should
|
||||
produce the appropriate ChatCompletionMessageParam with the same content.
|
||||
"""
|
||||
items: list[TResponseInputItem] = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "How are you?",
|
||||
}
|
||||
]
|
||||
messages = Converter.items_to_messages(items)
|
||||
assert len(messages) == 1
|
||||
out = messages[0]
|
||||
assert out["role"] == "user"
|
||||
# For simple string inputs, the converter returns the content as a bare string
|
||||
assert out["content"] == "How are you?"
|
||||
|
||||
|
||||
def test_items_to_messages_accepts_raw_chat_completions_user_content_parts():
|
||||
"""
|
||||
Raw Chat Completions content parts should be accepted as aliases for the SDK's
|
||||
canonical input content shapes.
|
||||
"""
|
||||
items: list[TResponseInputItem] = [
|
||||
# Cast the fixture because mypy cannot infer this raw chat-style dict as a specific
|
||||
# member of the TResponseInputItem TypedDict union on its own.
|
||||
cast(
|
||||
TResponseInputItem,
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "https://example.com/image.png",
|
||||
"detail": "high",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
messages = Converter.items_to_messages(items)
|
||||
|
||||
assert len(messages) == 1
|
||||
message = messages[0]
|
||||
assert message["role"] == "user"
|
||||
assert message["content"] == [
|
||||
{"type": "text", "text": "What is in this image?"},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "https://example.com/image.png",
|
||||
"detail": "high",
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_items_to_messages_with_output_message_and_function_call():
|
||||
"""
|
||||
Given a sequence of one ResponseOutputMessageParam followed by a
|
||||
ResponseFunctionToolCallParam, the converter should produce a single
|
||||
ChatCompletionAssistantMessageParam that includes both the assistant's
|
||||
textual content and a populated `tool_calls` reflecting the function call.
|
||||
"""
|
||||
# Construct output message param dict with two content parts.
|
||||
output_text: ResponseOutputText = ResponseOutputText(
|
||||
text="Part 1",
|
||||
type="output_text",
|
||||
annotations=[],
|
||||
logprobs=[],
|
||||
)
|
||||
refusal: ResponseOutputRefusal = ResponseOutputRefusal(
|
||||
refusal="won't do that",
|
||||
type="refusal",
|
||||
)
|
||||
resp_msg: ResponseOutputMessage = ResponseOutputMessage(
|
||||
id="42",
|
||||
type="message",
|
||||
role="assistant",
|
||||
status="completed",
|
||||
content=[output_text, refusal],
|
||||
)
|
||||
# Construct a function call item dict (as if returned from model)
|
||||
func_item: ResponseFunctionToolCallParam = {
|
||||
"id": "99",
|
||||
"call_id": "abc",
|
||||
"name": "math",
|
||||
"arguments": "{}",
|
||||
"type": "function_call",
|
||||
}
|
||||
items: list[TResponseInputItem] = [
|
||||
resp_msg.model_dump(), # type:ignore
|
||||
func_item,
|
||||
]
|
||||
messages = Converter.items_to_messages(items)
|
||||
# Should return a single assistant message
|
||||
assert len(messages) == 1
|
||||
assistant = messages[0]
|
||||
assert assistant["role"] == "assistant"
|
||||
# Content combines text portions of the output message
|
||||
assert "content" in assistant
|
||||
assert assistant["content"] == "Part 1"
|
||||
# Refusal in output message should be represented in assistant message
|
||||
assert "refusal" in assistant
|
||||
assert assistant["refusal"] == refusal.refusal
|
||||
# Tool calls list should contain one ChatCompletionMessageFunctionToolCall dict
|
||||
tool_calls = assistant.get("tool_calls")
|
||||
assert isinstance(tool_calls, list)
|
||||
assert len(tool_calls) == 1
|
||||
tool_call = tool_calls[0]
|
||||
assert tool_call["type"] == "function"
|
||||
assert tool_call["function"]["name"] == "math"
|
||||
assert tool_call["function"]["arguments"] == "{}"
|
||||
|
||||
|
||||
def test_convert_tool_choice_handles_standard_and_named_options() -> None:
|
||||
"""
|
||||
The `Converter.convert_tool_choice` method should return the omit sentinel
|
||||
if no choice is provided, pass through values like "auto", "required",
|
||||
or "none" unchanged, and translate any other string into a function
|
||||
selection dict.
|
||||
"""
|
||||
assert Converter.convert_tool_choice(None) is omit
|
||||
assert Converter.convert_tool_choice("auto") == "auto"
|
||||
assert Converter.convert_tool_choice("required") == "required"
|
||||
assert Converter.convert_tool_choice("none") == "none"
|
||||
tool_choice_dict = Converter.convert_tool_choice("mytool")
|
||||
assert isinstance(tool_choice_dict, dict)
|
||||
assert tool_choice_dict["type"] == "function"
|
||||
assert tool_choice_dict["function"]["name"] == "mytool"
|
||||
|
||||
|
||||
def test_convert_tool_choice_allows_tool_search_as_named_function_for_chat_models() -> None:
|
||||
tool_choice_dict = Converter.convert_tool_choice("tool_search")
|
||||
assert isinstance(tool_choice_dict, dict)
|
||||
assert tool_choice_dict["type"] == "function"
|
||||
assert tool_choice_dict["function"]["name"] == "tool_search"
|
||||
|
||||
|
||||
def test_convert_response_format_returns_not_given_for_plain_text_and_dict_for_schemas() -> None:
|
||||
"""
|
||||
The `Converter.convert_response_format` method should return the omit sentinel
|
||||
when no output schema is provided or if the output schema indicates
|
||||
plain text. For structured output schemas, it should return a dict
|
||||
with type `json_schema` and include the generated JSON schema and
|
||||
strict flag from the provided `AgentOutputSchema`.
|
||||
"""
|
||||
# when output is plain text (schema None or output_type str), do not include response_format
|
||||
assert Converter.convert_response_format(None) is omit
|
||||
assert Converter.convert_response_format(AgentOutputSchema(str)) is omit
|
||||
# For e.g. integer output, we expect a response_format dict
|
||||
schema = AgentOutputSchema(int)
|
||||
resp_format = Converter.convert_response_format(schema)
|
||||
assert isinstance(resp_format, dict)
|
||||
assert resp_format["type"] == "json_schema"
|
||||
assert resp_format["json_schema"]["name"] == "final_output"
|
||||
assert "strict" in resp_format["json_schema"]
|
||||
assert resp_format["json_schema"]["strict"] == schema.is_strict_json_schema()
|
||||
assert "schema" in resp_format["json_schema"]
|
||||
assert resp_format["json_schema"]["schema"] == schema.json_schema()
|
||||
|
||||
|
||||
def test_items_to_messages_with_function_output_item():
|
||||
"""
|
||||
A function call output item should be converted into a tool role message
|
||||
dict with the appropriate tool_call_id and content.
|
||||
"""
|
||||
func_output_item: FunctionCallOutput = {
|
||||
"type": "function_call_output",
|
||||
"call_id": "somecall",
|
||||
"output": '{"foo": "bar"}',
|
||||
}
|
||||
messages = Converter.items_to_messages([func_output_item])
|
||||
assert len(messages) == 1
|
||||
tool_msg = messages[0]
|
||||
assert tool_msg["role"] == "tool"
|
||||
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
|
||||
assert tool_msg["content"] == func_output_item["output"]
|
||||
|
||||
|
||||
def test_items_to_messages_with_non_text_only_function_output_uses_placeholder_by_default(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
):
|
||||
"""Default conversion should keep running without sending an empty tool message."""
|
||||
func_output_item: FunctionCallOutput = {
|
||||
"type": "function_call_output",
|
||||
"call_id": "somecall",
|
||||
"output": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/image.png",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="openai.agents"):
|
||||
messages = Converter.items_to_messages([func_output_item])
|
||||
|
||||
assert len(messages) == 1
|
||||
tool_msg = messages[0]
|
||||
assert tool_msg["role"] == "tool"
|
||||
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
|
||||
assert tool_msg["content"] == "[tool output omitted]"
|
||||
assert "Replacing the tool output with a placeholder" in caplog.text
|
||||
|
||||
|
||||
def test_items_to_messages_with_non_text_only_function_output_raises_in_strict_mode():
|
||||
"""Strict validation should fail explicitly instead of silently losing the output."""
|
||||
func_output_item: FunctionCallOutput = {
|
||||
"type": "function_call_output",
|
||||
"call_id": "somecall",
|
||||
"output": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/image.png",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
with pytest.raises(UserError, match="cannot be empty or contain only non-text content"):
|
||||
Converter.items_to_messages([func_output_item], strict_feature_validation=True)
|
||||
|
||||
|
||||
def test_items_to_messages_with_empty_function_output_uses_placeholder_by_default(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
):
|
||||
"""Default conversion should not send an empty tool message."""
|
||||
func_output_item: FunctionCallOutput = {
|
||||
"type": "function_call_output",
|
||||
"call_id": "somecall",
|
||||
"output": [],
|
||||
}
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="openai.agents"):
|
||||
messages = Converter.items_to_messages([func_output_item])
|
||||
|
||||
assert len(messages) == 1
|
||||
tool_msg = messages[0]
|
||||
assert tool_msg["role"] == "tool"
|
||||
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
|
||||
assert tool_msg["content"] == "[tool output omitted]"
|
||||
assert "Replacing the tool output with a placeholder" in caplog.text
|
||||
|
||||
|
||||
def test_items_to_messages_with_empty_function_output_raises_in_strict_mode():
|
||||
"""Strict validation should fail explicitly instead of sending empty output."""
|
||||
func_output_item: FunctionCallOutput = {
|
||||
"type": "function_call_output",
|
||||
"call_id": "somecall",
|
||||
"output": [],
|
||||
}
|
||||
|
||||
with pytest.raises(UserError, match="cannot be empty or contain only non-text content"):
|
||||
Converter.items_to_messages([func_output_item], strict_feature_validation=True)
|
||||
|
||||
|
||||
def test_items_to_messages_with_mixed_function_output_keeps_text_by_default(
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
):
|
||||
"""Default conversion should preserve text parts and omit unsupported non-text parts."""
|
||||
func_output_item: FunctionCallOutput = {
|
||||
"type": "function_call_output",
|
||||
"call_id": "somecall",
|
||||
"output": [
|
||||
{"type": "input_text", "text": "visible text"},
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/image.png",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="openai.agents"):
|
||||
messages = Converter.items_to_messages([func_output_item])
|
||||
|
||||
assert len(messages) == 1
|
||||
tool_msg = messages[0]
|
||||
assert tool_msg["role"] == "tool"
|
||||
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
|
||||
assert tool_msg["content"] == [{"type": "text", "text": "visible text"}]
|
||||
assert "tool output omitted" not in caplog.text
|
||||
|
||||
|
||||
def test_items_to_messages_can_preserve_non_text_function_output() -> None:
|
||||
"""Compatible providers can opt in to preserving non-text tool output."""
|
||||
func_output_item: FunctionCallOutput = {
|
||||
"type": "function_call_output",
|
||||
"call_id": "somecall",
|
||||
"output": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/image.png",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
[func_output_item],
|
||||
preserve_tool_output_all_content=True,
|
||||
)
|
||||
|
||||
assert len(messages) == 1
|
||||
tool_msg = messages[0]
|
||||
assert tool_msg["role"] == "tool"
|
||||
assert tool_msg["tool_call_id"] == func_output_item["call_id"]
|
||||
assert tool_msg["content"] == [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.com/image.png", "detail": "auto"},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_extract_all_and_text_content_for_strings_and_lists():
|
||||
"""
|
||||
The converter provides helpers for extracting user-supplied message content
|
||||
either as a simple string or as a list of `input_text` dictionaries.
|
||||
When passed a bare string, both `extract_all_content` and
|
||||
`extract_text_content` should return the string unchanged.
|
||||
When passed a list of input dictionaries, `extract_all_content` should
|
||||
produce a list of `ChatCompletionContentPart` dicts, and `extract_text_content`
|
||||
should filter to only the textual parts.
|
||||
"""
|
||||
prompt = "just text"
|
||||
assert Converter.extract_all_content(prompt) == prompt
|
||||
assert Converter.extract_text_content(prompt) == prompt
|
||||
text1: ResponseInputTextParam = {"type": "input_text", "text": "one"}
|
||||
text2: ResponseInputTextParam = {"type": "input_text", "text": "two"}
|
||||
all_parts = Converter.extract_all_content([text1, text2])
|
||||
assert isinstance(all_parts, list)
|
||||
assert len(all_parts) == 2
|
||||
assert all_parts[0]["type"] == "text" and all_parts[0]["text"] == "one"
|
||||
assert all_parts[1]["type"] == "text" and all_parts[1]["text"] == "two"
|
||||
text_parts = Converter.extract_text_content([text1, text2])
|
||||
assert isinstance(text_parts, list)
|
||||
assert all(p["type"] == "text" for p in text_parts)
|
||||
assert [p["text"] for p in text_parts] == ["one", "two"]
|
||||
|
||||
|
||||
def test_extract_all_content_handles_input_audio():
|
||||
"""
|
||||
input_audio entries should translate into ChatCompletion input_audio parts.
|
||||
"""
|
||||
audio: ResponseInputAudioParam = {
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": "AAA=", "format": "wav"},
|
||||
}
|
||||
parts = Converter.extract_all_content([audio])
|
||||
assert isinstance(parts, list)
|
||||
assert parts == [
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": "AAA=", "format": "wav"},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_extract_all_content_preserves_prompt_cache_breakpoints() -> None:
|
||||
breakpoint = {"mode": "explicit"}
|
||||
content: list[dict[str, Any]] = [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": "one",
|
||||
"prompt_cache_breakpoint": breakpoint,
|
||||
},
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/image.png",
|
||||
"prompt_cache_breakpoint": breakpoint,
|
||||
},
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": "AAA=", "format": "wav"},
|
||||
"prompt_cache_breakpoint": breakpoint,
|
||||
},
|
||||
{
|
||||
"type": "input_file",
|
||||
"file_data": "data:text/plain;base64,SGVsbG8=",
|
||||
"filename": "hello.txt",
|
||||
"prompt_cache_breakpoint": breakpoint,
|
||||
},
|
||||
]
|
||||
|
||||
parts = Converter.extract_all_content(content)
|
||||
|
||||
assert isinstance(parts, list)
|
||||
assert [part["prompt_cache_breakpoint"] for part in parts] == [breakpoint] * 4
|
||||
|
||||
|
||||
def test_raw_chat_content_aliases_preserve_prompt_cache_breakpoints() -> None:
|
||||
breakpoint = {"mode": "explicit"}
|
||||
|
||||
parts = Converter.extract_all_content(
|
||||
cast(
|
||||
list[dict[str, Any]],
|
||||
[
|
||||
{"type": "text", "text": "one", "prompt_cache_breakpoint": breakpoint},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": "https://example.com/image.png"},
|
||||
"prompt_cache_breakpoint": breakpoint,
|
||||
},
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
assert isinstance(parts, list)
|
||||
assert [part["prompt_cache_breakpoint"] for part in parts] == [breakpoint, breakpoint]
|
||||
|
||||
|
||||
def test_extract_all_content_rejects_invalid_input_audio():
|
||||
"""
|
||||
input_audio requires both data and format fields to be present.
|
||||
"""
|
||||
audio_missing_data = cast(
|
||||
ResponseInputAudioParam,
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {"format": "wav"},
|
||||
},
|
||||
)
|
||||
with pytest.raises(UserError):
|
||||
Converter.extract_all_content([audio_missing_data])
|
||||
|
||||
|
||||
def test_items_to_messages_handles_system_and_developer_roles():
|
||||
"""
|
||||
Roles other than `user` (e.g. `system` and `developer`) need to be
|
||||
converted appropriately whether provided as simple dicts or as full
|
||||
`message` typed dicts.
|
||||
"""
|
||||
sys_items: list[TResponseInputItem] = [{"role": "system", "content": "setup"}]
|
||||
sys_msgs = Converter.items_to_messages(sys_items)
|
||||
assert len(sys_msgs) == 1
|
||||
assert sys_msgs[0]["role"] == "system"
|
||||
assert sys_msgs[0]["content"] == "setup"
|
||||
dev_items: list[TResponseInputItem] = [{"role": "developer", "content": "debug"}]
|
||||
dev_msgs = Converter.items_to_messages(dev_items)
|
||||
assert len(dev_msgs) == 1
|
||||
assert dev_msgs[0]["role"] == "developer"
|
||||
assert dev_msgs[0]["content"] == "debug"
|
||||
|
||||
|
||||
def test_maybe_input_message_allows_message_typed_dict():
|
||||
"""
|
||||
The `Converter.maybe_input_message` should recognize a dict with
|
||||
"type": "message" and a supported role as an input message. Ensure
|
||||
that such dicts are passed through by `items_to_messages`.
|
||||
"""
|
||||
# Construct a dict with the proper required keys for a ResponseInputParam.Message
|
||||
message_dict: TResponseInputItem = {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": "hi",
|
||||
}
|
||||
assert Converter.maybe_input_message(message_dict) is not None
|
||||
# items_to_messages should process this correctly
|
||||
msgs = Converter.items_to_messages([message_dict])
|
||||
assert len(msgs) == 1
|
||||
assert msgs[0]["role"] == "user"
|
||||
assert msgs[0]["content"] == "hi"
|
||||
|
||||
|
||||
def test_tool_call_conversion():
|
||||
"""
|
||||
Test that tool calls are converted correctly.
|
||||
"""
|
||||
function_call = ResponseFunctionToolCallParam(
|
||||
id="tool1",
|
||||
call_id="abc",
|
||||
name="math",
|
||||
arguments="{}",
|
||||
type="function_call",
|
||||
)
|
||||
|
||||
messages = Converter.items_to_messages([function_call])
|
||||
assert len(messages) == 1
|
||||
tool_msg = messages[0]
|
||||
assert tool_msg["role"] == "assistant"
|
||||
assert tool_msg.get("content") is None
|
||||
|
||||
# Verify the content key exists in the message even when it is None.
|
||||
# This is for Chat Completions API compatibility.
|
||||
assert "content" in tool_msg, "content key should be present in assistant message"
|
||||
|
||||
tool_calls = list(tool_msg.get("tool_calls", []))
|
||||
assert len(tool_calls) == 1
|
||||
|
||||
tool_call = tool_calls[0]
|
||||
assert tool_call["id"] == function_call["call_id"]
|
||||
assert tool_call["function"]["name"] == function_call["name"] # type: ignore
|
||||
assert tool_call["function"]["arguments"] == function_call["arguments"] # type: ignore
|
||||
|
||||
|
||||
@pytest.mark.parametrize("role", ["user", "system", "developer"])
|
||||
def test_input_message_with_all_roles(role: str):
|
||||
"""
|
||||
The `Converter.maybe_input_message` should recognize a dict with
|
||||
"type": "message" and a supported role as an input message. Ensure
|
||||
that such dicts are passed through by `items_to_messages`.
|
||||
"""
|
||||
# Construct a dict with the proper required keys for a ResponseInputParam.Message
|
||||
casted_role = cast(Literal["user", "system", "developer"], role)
|
||||
message_dict: TResponseInputItem = {
|
||||
"type": "message",
|
||||
"role": casted_role,
|
||||
"content": "hi",
|
||||
}
|
||||
assert Converter.maybe_input_message(message_dict) is not None
|
||||
# items_to_messages should process this correctly
|
||||
msgs = Converter.items_to_messages([message_dict])
|
||||
assert len(msgs) == 1
|
||||
assert msgs[0]["role"] == casted_role
|
||||
assert msgs[0]["content"] == "hi"
|
||||
|
||||
|
||||
def test_item_reference_errors():
|
||||
"""
|
||||
Test that item references are converted correctly.
|
||||
"""
|
||||
with pytest.raises(UserError):
|
||||
Converter.items_to_messages(
|
||||
[
|
||||
{
|
||||
"type": "item_reference",
|
||||
"id": "item1",
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class TestObject:
|
||||
pass
|
||||
|
||||
|
||||
def test_unknown_object_errors():
|
||||
"""
|
||||
Test that unknown objects are converted correctly.
|
||||
"""
|
||||
with pytest.raises(UserError, match="Unhandled item type or structure"):
|
||||
# Purposely ignore the type error
|
||||
Converter.items_to_messages([TestObject()]) # type: ignore
|
||||
|
||||
|
||||
def test_assistant_messages_in_history():
|
||||
"""
|
||||
Test that assistant messages are added to the history.
|
||||
"""
|
||||
messages = Converter.items_to_messages(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": "Hello",
|
||||
},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "Hello?",
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": "What was my Name?",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
assert messages == [
|
||||
{"role": "user", "content": "Hello"},
|
||||
{"role": "assistant", "content": "Hello?"},
|
||||
{"role": "user", "content": "What was my Name?"},
|
||||
]
|
||||
assert len(messages) == 3
|
||||
assert messages[0]["role"] == "user"
|
||||
assert messages[0]["content"] == "Hello"
|
||||
assert messages[1]["role"] == "assistant"
|
||||
assert messages[1]["content"] == "Hello?"
|
||||
assert messages[2]["role"] == "user"
|
||||
assert messages[2]["content"] == "What was my Name?"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,43 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from agents.models.openai_client_utils import (
|
||||
is_official_openai_base_url,
|
||||
is_official_openai_client,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"https://api.openai.com",
|
||||
"https://api.openai.com/v1/",
|
||||
],
|
||||
)
|
||||
def test_official_openai_base_url_matches_exact_host(base_url: str) -> None:
|
||||
assert is_official_openai_base_url(base_url) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"base_url",
|
||||
[
|
||||
"https://api.openai.com.evil/v1/",
|
||||
"https://api.openai.com.proxy.local/v1/",
|
||||
"http://api.openai.com/v1/",
|
||||
"https://custom.example.test/v1/",
|
||||
],
|
||||
)
|
||||
def test_official_openai_base_url_rejects_non_openai_hosts(base_url: str) -> None:
|
||||
assert is_official_openai_base_url(base_url) is False
|
||||
|
||||
|
||||
def test_official_openai_websocket_base_url_matches_exact_host() -> None:
|
||||
assert is_official_openai_base_url("wss://api.openai.com/v1/", websocket=True) is True
|
||||
assert (
|
||||
is_official_openai_base_url("wss://api.openai.com.proxy.local/v1/", websocket=True) is False
|
||||
)
|
||||
|
||||
|
||||
def test_official_openai_client_rejects_client_without_base_url() -> None:
|
||||
assert is_official_openai_client(object()) is False # type: ignore[arg-type]
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,178 @@
|
||||
"""Unit tests for the low-level helpers in :mod:`agents.models._openai_retry`.
|
||||
|
||||
These exercise the header-parsing, status-extraction, and error-code helpers
|
||||
directly, plus a few public ``get_openai_retry_advice`` branches that the broader
|
||||
behavioral suite in ``test_model_retry.py`` does not reach.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from email.utils import format_datetime
|
||||
|
||||
import httpx
|
||||
|
||||
from agents.models._openai_retry import get_openai_retry_advice
|
||||
from agents.models._retry_runtime import (
|
||||
get_error_code as _get_error_code,
|
||||
get_error_header as _get_header_value,
|
||||
get_retry_after,
|
||||
get_status_code as _get_status_code,
|
||||
header_lookup as _header_lookup,
|
||||
parse_retry_after_ms as _parse_retry_after_ms,
|
||||
parse_retry_after_value as _parse_retry_after,
|
||||
)
|
||||
from agents.retry import ModelRetryAdviceRequest
|
||||
from agents.run_internal.model_retry import _normalize_retry_error
|
||||
|
||||
|
||||
class _HeaderError(Exception):
|
||||
"""Error that exposes headers through a plain attribute rather than a response."""
|
||||
|
||||
def __init__(self, message: str, *, headers: dict[str, str] | None = None) -> None:
|
||||
super().__init__(message)
|
||||
if headers is not None:
|
||||
self.headers = headers
|
||||
|
||||
|
||||
def _make_request(error: Exception, **kwargs: object) -> ModelRetryAdviceRequest:
|
||||
return ModelRetryAdviceRequest(error=error, attempt=1, stream=False, **kwargs) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_header_lookup_plain_mapping_matches_case_insensitively() -> None:
|
||||
headers = {"Retry-After": "5", "X-Other": "ignored"}
|
||||
assert _header_lookup(headers, "retry-after") == "5"
|
||||
assert _header_lookup(headers, "missing") is None
|
||||
|
||||
|
||||
def test_header_lookup_httpx_headers() -> None:
|
||||
headers = httpx.Headers({"retry-after": "7"})
|
||||
assert _header_lookup(headers, "retry-after") == "7"
|
||||
assert _header_lookup(None, "retry-after") is None
|
||||
|
||||
|
||||
def test_get_header_value_reads_response_headers_attr() -> None:
|
||||
class _Err(Exception):
|
||||
response_headers = {"retry-after": "3"}
|
||||
|
||||
assert _get_header_value(_Err("boom"), "retry-after") == "3"
|
||||
|
||||
|
||||
def test_parse_retry_after_ms_invalid_returns_none() -> None:
|
||||
assert _parse_retry_after_ms(None) is None
|
||||
assert _parse_retry_after_ms("not-a-number") is None
|
||||
assert _parse_retry_after_ms("-100") is None
|
||||
assert _parse_retry_after_ms("1500") == 1.5
|
||||
|
||||
|
||||
def test_parse_retry_after_numeric_and_http_date() -> None:
|
||||
assert _parse_retry_after(None) is None
|
||||
assert _parse_retry_after("2") == 2.0
|
||||
assert _parse_retry_after("-1") is None
|
||||
|
||||
future = datetime.now(timezone.utc) + timedelta(seconds=120)
|
||||
parsed = _parse_retry_after(format_datetime(future))
|
||||
assert parsed is not None and parsed > 0
|
||||
|
||||
assert _parse_retry_after("definitely not a date") is None
|
||||
|
||||
|
||||
def test_get_retry_after_preserves_outer_exception_precedence() -> None:
|
||||
outer = _HeaderError("wrapped", headers={"retry-after": "2"})
|
||||
outer.__cause__ = _HeaderError("provider", headers={"retry-after-ms": "1500"})
|
||||
|
||||
assert get_retry_after(outer) == 2.0
|
||||
|
||||
|
||||
def test_get_status_code_from_status_code_and_status_attrs() -> None:
|
||||
class _StatusCode(Exception):
|
||||
status_code = 503
|
||||
|
||||
class _Status(Exception):
|
||||
status = 504
|
||||
|
||||
assert _get_status_code(_StatusCode("a")) == 503
|
||||
assert _get_status_code(_Status("b")) == 504
|
||||
assert _get_status_code(Exception("none")) is None
|
||||
|
||||
|
||||
def test_get_error_code_from_body_mapping() -> None:
|
||||
class _NestedBody(Exception):
|
||||
body = {"error": {"code": "rate_limit_exceeded"}}
|
||||
|
||||
class _TopLevelBody(Exception):
|
||||
body = {"code": "server_error"}
|
||||
|
||||
assert _get_error_code(_NestedBody("a")) == "rate_limit_exceeded"
|
||||
assert _get_error_code(_TopLevelBody("b")) == "server_error"
|
||||
assert _get_error_code(Exception("none")) is None
|
||||
|
||||
|
||||
def test_provider_and_runner_retry_normalization_share_metadata() -> None:
|
||||
class _RetryableError(Exception):
|
||||
status_code = 429
|
||||
request_id = "req_test"
|
||||
body = {"error": {"code": "rate_limit_exceeded"}}
|
||||
headers = {"retry-after-ms": "1500"}
|
||||
|
||||
class _WrapperError(Exception):
|
||||
headers = {"x-other": "ignored"}
|
||||
|
||||
error = _WrapperError("wrapped")
|
||||
error.__cause__ = _RetryableError("slow down")
|
||||
advice = get_openai_retry_advice(_make_request(error))
|
||||
runner_normalized = _normalize_retry_error(error, None)
|
||||
|
||||
assert advice is not None
|
||||
assert advice.normalized is not None
|
||||
assert advice.normalized.status_code == runner_normalized.status_code
|
||||
assert advice.normalized.error_code == runner_normalized.error_code
|
||||
assert advice.normalized.request_id == runner_normalized.request_id
|
||||
assert advice.normalized.retry_after == runner_normalized.retry_after
|
||||
assert runner_normalized.retry_after == 1.5
|
||||
|
||||
|
||||
def test_advice_unsafe_to_replay() -> None:
|
||||
error = Exception("cannot replay")
|
||||
error.unsafe_to_replay = True # type: ignore[attr-defined]
|
||||
|
||||
advice = get_openai_retry_advice(_make_request(error))
|
||||
|
||||
assert advice is not None
|
||||
assert advice.suggested is False
|
||||
assert advice.replay_safety == "unsafe"
|
||||
|
||||
|
||||
def test_advice_websocket_request_is_unsafe() -> None:
|
||||
message = (
|
||||
"The request may have been accepted, so the SDK will not automatically "
|
||||
"retry this websocket request."
|
||||
)
|
||||
advice = get_openai_retry_advice(_make_request(Exception(message)))
|
||||
|
||||
assert advice is not None
|
||||
assert advice.suggested is False
|
||||
assert advice.replay_safety == "unsafe"
|
||||
|
||||
|
||||
def test_advice_respects_x_should_retry_false() -> None:
|
||||
error = _HeaderError("nope", headers={"x-should-retry": "false"})
|
||||
|
||||
advice = get_openai_retry_advice(_make_request(error))
|
||||
|
||||
assert advice is not None
|
||||
assert advice.suggested is False
|
||||
|
||||
|
||||
def test_advice_returns_retry_after_only_when_no_other_signal() -> None:
|
||||
# A 400 with no x-should-retry header and no network/timeout signal would not
|
||||
# normally retry, but a retry-after header still yields advice carrying the delay.
|
||||
error = _HeaderError("slow down", headers={"retry-after": "2"})
|
||||
|
||||
advice = get_openai_retry_advice(_make_request(error))
|
||||
|
||||
assert advice is not None
|
||||
assert advice.retry_after == 2.0
|
||||
# This branch only conveys the server-provided delay; it does not assert a
|
||||
# retry decision, so ``suggested`` keeps its unset default.
|
||||
assert advice.suggested is None
|
||||
@@ -0,0 +1,454 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
from openai.types.chat import ChatCompletion, ChatCompletionChunk, ChatCompletionMessage
|
||||
from openai.types.chat.chat_completion_chunk import Choice, ChoiceDelta
|
||||
from openai.types.completion_usage import (
|
||||
CompletionTokensDetails,
|
||||
CompletionUsage,
|
||||
PromptTokensDetails,
|
||||
)
|
||||
from openai.types.responses import (
|
||||
Response,
|
||||
ResponseOutputMessage,
|
||||
ResponseOutputText,
|
||||
ResponseReasoningItem,
|
||||
)
|
||||
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.interface import ModelTracing
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from agents.models.openai_provider import OpenAIProvider
|
||||
|
||||
|
||||
# Helper functions to create test objects consistently
|
||||
def create_content_delta(content: str) -> dict[str, Any]:
|
||||
"""Create a delta dictionary with regular content"""
|
||||
return {"content": content, "role": None, "function_call": None, "tool_calls": None}
|
||||
|
||||
|
||||
def create_reasoning_delta(content: str) -> dict[str, Any]:
|
||||
"""Create a delta dictionary with reasoning content. The Only difference is reasoning_content"""
|
||||
return {
|
||||
"content": None,
|
||||
"role": None,
|
||||
"function_call": None,
|
||||
"tool_calls": None,
|
||||
"reasoning_content": content,
|
||||
}
|
||||
|
||||
|
||||
def create_chunk(delta: dict[str, Any], include_usage: bool = False) -> ChatCompletionChunk:
|
||||
"""Create a ChatCompletionChunk with the given delta"""
|
||||
# Create a ChoiceDelta object from the dictionary
|
||||
delta_obj = ChoiceDelta(
|
||||
content=delta.get("content"),
|
||||
role=delta.get("role"),
|
||||
function_call=delta.get("function_call"),
|
||||
tool_calls=delta.get("tool_calls"),
|
||||
)
|
||||
|
||||
# Add reasoning_content attribute dynamically if present in the delta
|
||||
if "reasoning_content" in delta:
|
||||
# Use direct assignment for the reasoning_content attribute
|
||||
delta_obj_any = cast(Any, delta_obj)
|
||||
delta_obj_any.reasoning_content = delta["reasoning_content"]
|
||||
|
||||
# Create the chunk
|
||||
chunk = ChatCompletionChunk(
|
||||
id="chunk-id",
|
||||
created=1,
|
||||
model="deepseek is usually expected",
|
||||
object="chat.completion.chunk",
|
||||
choices=[Choice(index=0, delta=delta_obj)],
|
||||
)
|
||||
|
||||
if include_usage:
|
||||
chunk.usage = CompletionUsage(
|
||||
completion_tokens=4,
|
||||
prompt_tokens=2,
|
||||
total_tokens=6,
|
||||
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=2),
|
||||
prompt_tokens_details=PromptTokensDetails(cached_tokens=0),
|
||||
)
|
||||
|
||||
return chunk
|
||||
|
||||
|
||||
async def create_fake_stream(
|
||||
chunks: list[ChatCompletionChunk],
|
||||
) -> AsyncIterator[ChatCompletionChunk]:
|
||||
for chunk in chunks:
|
||||
yield chunk
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_yields_events_for_reasoning_content(monkeypatch) -> None:
|
||||
"""
|
||||
Validate that when a model streams reasoning content,
|
||||
`stream_response` emits the appropriate sequence of events including
|
||||
`response.reasoning_summary_text.delta` events for each chunk of the reasoning content and
|
||||
constructs a completed response with a `ResponseReasoningItem` part.
|
||||
"""
|
||||
# Create test chunks
|
||||
chunks = [
|
||||
# Reasoning content chunks
|
||||
create_chunk(create_reasoning_delta("Let me think")),
|
||||
create_chunk(create_reasoning_delta(" about this")),
|
||||
# Regular content chunks
|
||||
create_chunk(create_content_delta("The answer")),
|
||||
create_chunk(create_content_delta(" is 42"), include_usage=True),
|
||||
]
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, create_fake_stream(chunks)
|
||||
|
||||
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
|
||||
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
|
||||
# verify reasoning content events were emitted
|
||||
reasoning_delta_events = [
|
||||
e for e in output_events if e.type == "response.reasoning_summary_text.delta"
|
||||
]
|
||||
assert len(reasoning_delta_events) == 2
|
||||
assert reasoning_delta_events[0].delta == "Let me think"
|
||||
assert reasoning_delta_events[1].delta == " about this"
|
||||
|
||||
reasoning_done_index = next(
|
||||
index
|
||||
for index, event in enumerate(output_events)
|
||||
if event.type == "response.reasoning_summary_part.done"
|
||||
)
|
||||
first_text_delta_index = next(
|
||||
index
|
||||
for index, event in enumerate(output_events)
|
||||
if event.type == "response.output_text.delta"
|
||||
)
|
||||
assert reasoning_done_index < first_text_delta_index
|
||||
|
||||
# verify regular content events were emitted
|
||||
content_delta_events = [e for e in output_events if e.type == "response.output_text.delta"]
|
||||
assert len(content_delta_events) == 2
|
||||
assert content_delta_events[0].delta == "The answer"
|
||||
assert content_delta_events[1].delta == " is 42"
|
||||
|
||||
assistant_message_index_events = []
|
||||
for event in output_events:
|
||||
event_any = cast(Any, event)
|
||||
if event.type in {"response.output_item.added", "response.output_item.done"}:
|
||||
if event_any.item.type == "message":
|
||||
assistant_message_index_events.append(event_any)
|
||||
elif event.type in {
|
||||
"response.content_part.added",
|
||||
"response.output_text.delta",
|
||||
"response.content_part.done",
|
||||
}:
|
||||
assistant_message_index_events.append(event_any)
|
||||
|
||||
assert assistant_message_index_events
|
||||
for event in assistant_message_index_events:
|
||||
assert event.output_index == 1
|
||||
assert type(event.output_index) is int
|
||||
|
||||
# verify the final response contains both types of content
|
||||
response_event = output_events[-1]
|
||||
assert response_event.type == "response.completed"
|
||||
assert len(response_event.response.output) == 2
|
||||
|
||||
# first item should be reasoning
|
||||
assert isinstance(response_event.response.output[0], ResponseReasoningItem)
|
||||
assert response_event.response.output[0].summary[0].text == "Let me think about this"
|
||||
|
||||
# second item should be message with text
|
||||
assert isinstance(response_event.response.output[1], ResponseOutputMessage)
|
||||
assert isinstance(response_event.response.output[1].content[0], ResponseOutputText)
|
||||
assert response_event.response.output[1].content[0].text == "The answer is 42"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_keeps_reasoning_item_open_across_interleaved_text(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
chunks = [
|
||||
create_chunk(create_reasoning_delta("Let me think")),
|
||||
create_chunk(create_content_delta("The answer")),
|
||||
create_chunk(create_reasoning_delta(" more carefully")),
|
||||
create_chunk(create_content_delta(" is 42"), include_usage=True),
|
||||
]
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, create_fake_stream(chunks)
|
||||
|
||||
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
|
||||
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
|
||||
reasoning_part_added_events = [
|
||||
event for event in output_events if event.type == "response.reasoning_summary_part.added"
|
||||
]
|
||||
assert [event.summary_index for event in reasoning_part_added_events] == [0, 1]
|
||||
|
||||
reasoning_part_done_events = [
|
||||
event for event in output_events if event.type == "response.reasoning_summary_part.done"
|
||||
]
|
||||
assert [event.summary_index for event in reasoning_part_done_events] == [0, 1]
|
||||
|
||||
first_reasoning_done_index = output_events.index(reasoning_part_done_events[0])
|
||||
first_text_delta_index = next(
|
||||
index
|
||||
for index, event in enumerate(output_events)
|
||||
if event.type == "response.output_text.delta"
|
||||
)
|
||||
second_reasoning_delta_index = next(
|
||||
index
|
||||
for index, event in enumerate(output_events)
|
||||
if event.type == "response.reasoning_summary_text.delta" and event.summary_index == 1
|
||||
)
|
||||
reasoning_item_done_index = next(
|
||||
index
|
||||
for index, event in enumerate(output_events)
|
||||
if event.type == "response.output_item.done" and event.item.type == "reasoning"
|
||||
)
|
||||
|
||||
assert first_reasoning_done_index < first_text_delta_index
|
||||
assert second_reasoning_delta_index > first_text_delta_index
|
||||
assert reasoning_item_done_index > second_reasoning_delta_index
|
||||
|
||||
response_event = output_events[-1]
|
||||
assert response_event.type == "response.completed"
|
||||
assert isinstance(response_event.response.output[0], ResponseReasoningItem)
|
||||
assert [summary.text for summary in response_event.response.output[0].summary] == [
|
||||
"Let me think",
|
||||
" more carefully",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_response_with_reasoning_content(monkeypatch) -> None:
|
||||
"""
|
||||
Test that when a model returns reasoning content in addition to regular content,
|
||||
`get_response` properly includes both in the response output.
|
||||
"""
|
||||
# create a message with reasoning content
|
||||
msg = ChatCompletionMessage(
|
||||
role="assistant",
|
||||
content="The answer is 42",
|
||||
)
|
||||
# Use dynamic attribute for reasoning_content
|
||||
# We need to cast to Any to avoid mypy errors since reasoning_content is not a defined attribute
|
||||
msg_with_reasoning = cast(Any, msg)
|
||||
msg_with_reasoning.reasoning_content = "Let me think about this question carefully"
|
||||
|
||||
# create a choice with the message
|
||||
mock_choice = {
|
||||
"index": 0,
|
||||
"finish_reason": "stop",
|
||||
"message": msg_with_reasoning,
|
||||
"delta": None,
|
||||
}
|
||||
|
||||
chat = ChatCompletion(
|
||||
id="resp-id",
|
||||
created=0,
|
||||
model="deepseek is expected",
|
||||
object="chat.completion",
|
||||
choices=[mock_choice], # type: ignore[list-item]
|
||||
usage=CompletionUsage(
|
||||
completion_tokens=10,
|
||||
prompt_tokens=5,
|
||||
total_tokens=15,
|
||||
completion_tokens_details=CompletionTokensDetails(reasoning_tokens=6),
|
||||
prompt_tokens_details=PromptTokensDetails(cached_tokens=0),
|
||||
),
|
||||
)
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
return chat
|
||||
|
||||
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
|
||||
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
|
||||
resp = await model.get_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
)
|
||||
|
||||
# should have produced a reasoning item and a message with text content
|
||||
assert len(resp.output) == 2
|
||||
|
||||
# first output should be the reasoning item
|
||||
assert isinstance(resp.output[0], ResponseReasoningItem)
|
||||
assert resp.output[0].summary[0].text == "Let me think about this question carefully"
|
||||
|
||||
# second output should be the message with text content
|
||||
assert isinstance(resp.output[1], ResponseOutputMessage)
|
||||
assert isinstance(resp.output[1].content[0], ResponseOutputText)
|
||||
assert resp.output[1].content[0].text == "The answer is 42"
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_preserves_usage_from_earlier_chunk(monkeypatch) -> None:
|
||||
"""
|
||||
Test that when an earlier chunk has usage data and later chunks don't,
|
||||
the usage from the earlier chunk is preserved in the final response.
|
||||
This handles cases where some providers (e.g., LiteLLM) may not include
|
||||
usage in every chunk.
|
||||
"""
|
||||
# Create test chunks where first chunk has usage, last chunk doesn't
|
||||
chunks = [
|
||||
create_chunk(create_content_delta("Hello"), include_usage=True), # Has usage
|
||||
create_chunk(create_content_delta("")), # No usage (usage=None)
|
||||
]
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, create_fake_stream(chunks)
|
||||
|
||||
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
|
||||
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
|
||||
# Verify the final response preserves usage from the first chunk
|
||||
response_event = output_events[-1]
|
||||
assert response_event.type == "response.completed"
|
||||
assert response_event.response.usage is not None
|
||||
assert response_event.response.usage.input_tokens == 2
|
||||
assert response_event.response.usage.output_tokens == 4
|
||||
assert response_event.response.usage.total_tokens == 6
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_response_with_empty_reasoning_content(monkeypatch) -> None:
|
||||
"""
|
||||
Test that when a model streams empty reasoning content,
|
||||
the response still processes correctly without errors.
|
||||
"""
|
||||
# create test chunks with empty reasoning content
|
||||
chunks = [
|
||||
create_chunk(create_reasoning_delta("")),
|
||||
create_chunk(create_content_delta("The answer is 42"), include_usage=True),
|
||||
]
|
||||
|
||||
async def patched_fetch_response(self, *args, **kwargs):
|
||||
resp = Response(
|
||||
id="resp-id",
|
||||
created_at=0,
|
||||
model="fake-model",
|
||||
object="response",
|
||||
output=[],
|
||||
tool_choice="none",
|
||||
tools=[],
|
||||
parallel_tool_calls=False,
|
||||
)
|
||||
return resp, create_fake_stream(chunks)
|
||||
|
||||
monkeypatch.setattr(OpenAIChatCompletionsModel, "_fetch_response", patched_fetch_response)
|
||||
model = OpenAIProvider(use_responses=False).get_model("gpt-4")
|
||||
output_events = []
|
||||
async for event in model.stream_response(
|
||||
system_instructions=None,
|
||||
input="",
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
conversation_id=None,
|
||||
prompt=None,
|
||||
):
|
||||
output_events.append(event)
|
||||
|
||||
# verify the final response contains the content
|
||||
response_event = output_events[-1]
|
||||
assert response_event.type == "response.completed"
|
||||
|
||||
# should only have the message, not an empty reasoning item
|
||||
assert len(response_event.response.output) == 1
|
||||
assert isinstance(response_event.response.output[0], ResponseOutputMessage)
|
||||
assert isinstance(response_event.response.output[0].content[0], ResponseOutputText)
|
||||
assert response_event.response.output[0].content[0].text == "The answer is 42"
|
||||
@@ -0,0 +1,403 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
import httpx
|
||||
import litellm
|
||||
import pytest
|
||||
from litellm.types.utils import Choices, Message, ModelResponse, Usage
|
||||
from openai.types.chat.chat_completion import ChatCompletion, Choice
|
||||
from openai.types.chat.chat_completion_message import ChatCompletionMessage
|
||||
from openai.types.completion_usage import CompletionUsage
|
||||
|
||||
from agents.extensions.models.litellm_model import LitellmModel
|
||||
from agents.items import TResponseInputItem
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models.chatcmpl_converter import Converter
|
||||
from agents.models.interface import ModelTracing
|
||||
from agents.models.openai_chatcompletions import OpenAIChatCompletionsModel
|
||||
from agents.models.reasoning_content_replay import ReasoningContentReplayContext
|
||||
|
||||
REASONING_CONTENT_MODEL_A = "reasoning-content-model-a"
|
||||
REASONING_CONTENT_MODEL_B = "reasoning-content-model-b"
|
||||
# The converter currently keys Anthropic thinking-block reconstruction off the model name,
|
||||
# so this test model keeps the "anthropic" substring while staying otherwise generic.
|
||||
REASONING_CONTENT_MODEL_C = "reasoning-content-model-c-anthropic"
|
||||
|
||||
|
||||
def _second_turn_input_items(model_name: str) -> list[TResponseInputItem]:
|
||||
return cast(
|
||||
list[TResponseInputItem],
|
||||
[
|
||||
{"role": "user", "content": "What's the weather in Tokyo?"},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [
|
||||
{"text": "I should call the weather tool first.", "type": "summary_text"}
|
||||
],
|
||||
"type": "reasoning",
|
||||
"content": None,
|
||||
"encrypted_content": None,
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"arguments": '{"city": "Tokyo"}',
|
||||
"call_id": "call_weather_123",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"id": "__fake_id__",
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_123",
|
||||
"output": "The weather in Tokyo is sunny and 22°C.",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _second_turn_input_items_with_message(model_name: str) -> list[TResponseInputItem]:
|
||||
return cast(
|
||||
list[TResponseInputItem],
|
||||
[
|
||||
{"role": "user", "content": "What's the weather in Tokyo?"},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [
|
||||
{"text": "I should call the weather tool first.", "type": "summary_text"}
|
||||
],
|
||||
"type": "reasoning",
|
||||
"content": None,
|
||||
"encrypted_content": None,
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "I'll call the weather tool now.",
|
||||
"annotations": [],
|
||||
"logprobs": [],
|
||||
}
|
||||
],
|
||||
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"arguments": '{"city": "Tokyo"}',
|
||||
"call_id": "call_weather_123",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"id": "__fake_id__",
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_123",
|
||||
"output": "The weather in Tokyo is sunny and 22°C.",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _second_turn_input_items_with_file_search(model_name: str) -> list[TResponseInputItem]:
|
||||
return cast(
|
||||
list[TResponseInputItem],
|
||||
[
|
||||
{"role": "user", "content": "Find notes about Tokyo weather."},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [
|
||||
{"text": "I should search the knowledge base first.", "type": "summary_text"}
|
||||
],
|
||||
"type": "reasoning",
|
||||
"content": None,
|
||||
"encrypted_content": None,
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"id": "__fake_file_search_id__",
|
||||
"queries": ["Tokyo weather"],
|
||||
"status": "completed",
|
||||
"type": "file_search_call",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _second_turn_input_items_with_message_then_reasoning(
|
||||
model_name: str,
|
||||
) -> list[TResponseInputItem]:
|
||||
return cast(
|
||||
list[TResponseInputItem],
|
||||
[
|
||||
{"role": "user", "content": "What's the weather in Tokyo?"},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"status": "completed",
|
||||
"content": [
|
||||
{
|
||||
"type": "output_text",
|
||||
"text": "I'll call the weather tool now.",
|
||||
"annotations": [],
|
||||
"logprobs": [],
|
||||
}
|
||||
],
|
||||
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [
|
||||
{"text": "I should call the weather tool first.", "type": "summary_text"}
|
||||
],
|
||||
"type": "reasoning",
|
||||
"content": None,
|
||||
"encrypted_content": None,
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"arguments": '{"city": "Tokyo"}',
|
||||
"call_id": "call_weather_123",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"id": "__fake_id__",
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_123",
|
||||
"output": "The weather in Tokyo is sunny and 22°C.",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _second_turn_input_items_with_thinking_blocks(model_name: str) -> list[TResponseInputItem]:
|
||||
return cast(
|
||||
list[TResponseInputItem],
|
||||
[
|
||||
{"role": "user", "content": "What's the weather in Tokyo?"},
|
||||
{
|
||||
"id": "__fake_id__",
|
||||
"summary": [
|
||||
{"text": "I should call the weather tool first.", "type": "summary_text"}
|
||||
],
|
||||
"type": "reasoning",
|
||||
"content": [
|
||||
{
|
||||
"type": "reasoning_text",
|
||||
"text": "First, I need to inspect the request.",
|
||||
}
|
||||
],
|
||||
"encrypted_content": "test-signature",
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name, "response_id": "chatcmpl-test"},
|
||||
},
|
||||
{
|
||||
"arguments": '{"city": "Tokyo"}',
|
||||
"call_id": "call_weather_123",
|
||||
"name": "get_weather",
|
||||
"type": "function_call",
|
||||
"id": "__fake_id__",
|
||||
"status": None,
|
||||
"provider_data": {"model": model_name},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_weather_123",
|
||||
"output": "The weather in Tokyo is sunny and 22°C.",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _assistant_with_tool_calls(messages: list[Any]) -> dict[str, Any]:
|
||||
for msg in messages:
|
||||
if isinstance(msg, dict) and msg.get("role") == "assistant" and msg.get("tool_calls"):
|
||||
return msg
|
||||
raise AssertionError("Expected an assistant message with tool_calls.")
|
||||
|
||||
|
||||
def test_converter_keeps_default_reasoning_replay_behavior_for_non_default_model() -> None:
|
||||
messages = Converter.items_to_messages(
|
||||
_second_turn_input_items(REASONING_CONTENT_MODEL_A),
|
||||
model=REASONING_CONTENT_MODEL_A,
|
||||
)
|
||||
|
||||
assistant = _assistant_with_tool_calls(messages)
|
||||
assert "reasoning_content" not in assistant
|
||||
|
||||
|
||||
def test_converter_preserves_reasoning_content_across_output_message_with_hook() -> None:
|
||||
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
|
||||
return True
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
_second_turn_input_items_with_message(REASONING_CONTENT_MODEL_A),
|
||||
model=REASONING_CONTENT_MODEL_A,
|
||||
should_replay_reasoning_content=should_replay_reasoning_content,
|
||||
)
|
||||
|
||||
assistant = _assistant_with_tool_calls(messages)
|
||||
assert assistant["content"] == "I'll call the weather tool now."
|
||||
assert assistant["reasoning_content"] == "I should call the weather tool first."
|
||||
|
||||
|
||||
def test_converter_replays_reasoning_content_when_reasoning_follows_message_with_hook() -> None:
|
||||
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
|
||||
return True
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
_second_turn_input_items_with_message_then_reasoning(REASONING_CONTENT_MODEL_A),
|
||||
model=REASONING_CONTENT_MODEL_A,
|
||||
should_replay_reasoning_content=should_replay_reasoning_content,
|
||||
)
|
||||
|
||||
assistant = _assistant_with_tool_calls(messages)
|
||||
assert assistant["content"] == "I'll call the weather tool now."
|
||||
assert assistant["reasoning_content"] == "I should call the weather tool first."
|
||||
|
||||
|
||||
def test_converter_replays_reasoning_content_for_file_search_call_with_hook() -> None:
|
||||
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
|
||||
return True
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
_second_turn_input_items_with_file_search(REASONING_CONTENT_MODEL_A),
|
||||
model=REASONING_CONTENT_MODEL_A,
|
||||
should_replay_reasoning_content=should_replay_reasoning_content,
|
||||
)
|
||||
|
||||
assistant = _assistant_with_tool_calls(messages)
|
||||
assert assistant["reasoning_content"] == "I should search the knowledge base first."
|
||||
assert assistant["tool_calls"][0]["function"]["name"] == "file_search_call"
|
||||
|
||||
|
||||
def test_converter_replays_reasoning_content_with_thinking_blocks_and_hook() -> None:
|
||||
def should_replay_reasoning_content(_context: ReasoningContentReplayContext) -> bool:
|
||||
return True
|
||||
|
||||
messages = Converter.items_to_messages(
|
||||
_second_turn_input_items_with_thinking_blocks(REASONING_CONTENT_MODEL_C),
|
||||
model=REASONING_CONTENT_MODEL_C,
|
||||
preserve_thinking_blocks=True,
|
||||
should_replay_reasoning_content=should_replay_reasoning_content,
|
||||
)
|
||||
|
||||
assistant = _assistant_with_tool_calls(messages)
|
||||
assert assistant["reasoning_content"] == "I should call the weather tool first."
|
||||
assert assistant["content"][0]["type"] == "thinking"
|
||||
assert assistant["content"][0]["thinking"] == "First, I need to inspect the request."
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_openai_chatcompletions_hook_can_enable_reasoning_content_replay() -> None:
|
||||
captured: dict[str, Any] = {}
|
||||
contexts: list[ReasoningContentReplayContext] = []
|
||||
|
||||
def should_replay_reasoning_content(context: ReasoningContentReplayContext) -> bool:
|
||||
contexts.append(context)
|
||||
return context.model == REASONING_CONTENT_MODEL_B
|
||||
|
||||
class MockChatCompletions:
|
||||
async def create(self, **kwargs):
|
||||
captured.update(kwargs)
|
||||
msg = ChatCompletionMessage(role="assistant", content="done")
|
||||
choice = Choice(index=0, message=msg, finish_reason="stop")
|
||||
return ChatCompletion(
|
||||
id="test-id",
|
||||
created=0,
|
||||
model=REASONING_CONTENT_MODEL_B,
|
||||
object="chat.completion",
|
||||
choices=[choice],
|
||||
usage=CompletionUsage(completion_tokens=5, prompt_tokens=10, total_tokens=15),
|
||||
)
|
||||
|
||||
class MockChat:
|
||||
def __init__(self):
|
||||
self.completions = MockChatCompletions()
|
||||
|
||||
class MockClient:
|
||||
def __init__(self):
|
||||
self.chat = MockChat()
|
||||
self.base_url = httpx.URL("https://example.com/v1/")
|
||||
|
||||
model = OpenAIChatCompletionsModel(
|
||||
model=REASONING_CONTENT_MODEL_B,
|
||||
openai_client=cast(Any, MockClient()),
|
||||
should_replay_reasoning_content=should_replay_reasoning_content,
|
||||
)
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=_second_turn_input_items(REASONING_CONTENT_MODEL_B),
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assistant = _assistant_with_tool_calls(cast(list[dict[str, Any]], captured["messages"]))
|
||||
assert assistant["reasoning_content"] == "I should call the weather tool first."
|
||||
assert len(contexts) == 1
|
||||
assert contexts[0].model == REASONING_CONTENT_MODEL_B
|
||||
assert contexts[0].base_url == "https://example.com/v1"
|
||||
assert contexts[0].reasoning.origin_model == REASONING_CONTENT_MODEL_B
|
||||
|
||||
|
||||
@pytest.mark.allow_call_model_methods
|
||||
@pytest.mark.asyncio
|
||||
async def test_litellm_hook_can_enable_reasoning_content_replay(monkeypatch) -> None:
|
||||
captured: dict[str, Any] = {}
|
||||
contexts: list[ReasoningContentReplayContext] = []
|
||||
|
||||
def should_replay_reasoning_content(context: ReasoningContentReplayContext) -> bool:
|
||||
contexts.append(context)
|
||||
return context.model == REASONING_CONTENT_MODEL_B
|
||||
|
||||
async def fake_acompletion(model, messages=None, **kwargs):
|
||||
captured["messages"] = messages
|
||||
msg = Message(role="assistant", content="done")
|
||||
choice = Choices(index=0, message=msg)
|
||||
return ModelResponse(choices=[choice], usage=Usage(0, 0, 0))
|
||||
|
||||
monkeypatch.setattr(litellm, "acompletion", fake_acompletion)
|
||||
|
||||
model = LitellmModel(
|
||||
model=REASONING_CONTENT_MODEL_B,
|
||||
should_replay_reasoning_content=should_replay_reasoning_content,
|
||||
)
|
||||
|
||||
await model.get_response(
|
||||
system_instructions=None,
|
||||
input=_second_turn_input_items(REASONING_CONTENT_MODEL_B),
|
||||
model_settings=ModelSettings(),
|
||||
tools=[],
|
||||
output_schema=None,
|
||||
handoffs=[],
|
||||
tracing=ModelTracing.DISABLED,
|
||||
previous_response_id=None,
|
||||
)
|
||||
|
||||
assistant = _assistant_with_tool_calls(cast(list[dict[str, Any]], captured["messages"]))
|
||||
assert assistant["reasoning_content"] == "I should call the weather tool first."
|
||||
assert len(contexts) == 1
|
||||
assert contexts[0].model == REASONING_CONTENT_MODEL_B
|
||||
assert contexts[0].base_url is None
|
||||
assert contexts[0].reasoning.origin_model == REASONING_CONTENT_MODEL_B
|
||||
@@ -0,0 +1,162 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from agents.models.fake_id import FAKE_RESPONSES_ID
|
||||
from agents.models.openai_responses import OpenAIResponsesModel
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def model() -> OpenAIResponsesModel:
|
||||
"""Create a model instance for testing."""
|
||||
mock_client = MagicMock()
|
||||
return OpenAIResponsesModel(model="gpt-5", openai_client=mock_client)
|
||||
|
||||
|
||||
class TestRemoveOpenAIResponsesAPIIncompatibleFields:
|
||||
"""Tests for _remove_openai_responses_api_incompatible_fields method."""
|
||||
|
||||
def test_returns_unchanged_when_no_provider_data(self, model: OpenAIResponsesModel):
|
||||
"""When no items have provider_data, the input should be returned unchanged."""
|
||||
list_input = [
|
||||
{"type": "message", "content": "hello"},
|
||||
{"type": "function_call", "call_id": "call_123", "name": "test"},
|
||||
]
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
assert result is list_input # Same object reference.
|
||||
|
||||
def test_removes_reasoning_items_with_provider_data(self, model: OpenAIResponsesModel):
|
||||
"""Reasoning items with provider_data should be completely removed."""
|
||||
list_input = [
|
||||
{"type": "message", "content": "hello"},
|
||||
{"type": "reasoning", "provider_data": {"model": "gemini/gemini-3"}},
|
||||
{"type": "function_call", "call_id": "call_123"},
|
||||
]
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0] == {"type": "message", "content": "hello"}
|
||||
assert result[1] == {"type": "function_call", "call_id": "call_123"}
|
||||
|
||||
def test_keeps_reasoning_items_without_provider_data(self, model: OpenAIResponsesModel):
|
||||
"""Reasoning items without provider_data should be kept."""
|
||||
list_input = [
|
||||
{"type": "reasoning", "summary": []},
|
||||
{"type": "message", "content": "hello", "provider_data": {"foo": "bar"}},
|
||||
]
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
assert len(result) == 2
|
||||
assert result[0] == {"type": "reasoning", "summary": []}
|
||||
assert result[1] == {"type": "message", "content": "hello"}
|
||||
|
||||
def test_removes_provider_data_from_all_items(self, model: OpenAIResponsesModel):
|
||||
"""provider_data field should be removed from all dict items."""
|
||||
list_input = [
|
||||
{"type": "message", "content": "hello", "provider_data": {"model": "gemini/gemini-3"}},
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_123",
|
||||
"provider_data": {"model": "gemini/gemini-3"},
|
||||
},
|
||||
]
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
assert len(result) == 2
|
||||
assert "provider_data" not in result[0]
|
||||
assert "provider_data" not in result[1]
|
||||
|
||||
def test_removes_fake_responses_id(self, model: OpenAIResponsesModel):
|
||||
"""Items with id equal to FAKE_RESPONSES_ID should have their id removed."""
|
||||
list_input = [
|
||||
{
|
||||
"type": "message",
|
||||
"id": FAKE_RESPONSES_ID,
|
||||
"content": "hello",
|
||||
"provider_data": {"model": "gemini/gemini-3"},
|
||||
},
|
||||
]
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
assert len(result) == 1
|
||||
assert "id" not in result[0]
|
||||
assert result[0]["content"] == "hello"
|
||||
|
||||
def test_preserves_real_ids(self, model: OpenAIResponsesModel):
|
||||
"""Real IDs (not FAKE_RESPONSES_ID) should be preserved."""
|
||||
list_input = [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_real123",
|
||||
"content": "hello",
|
||||
"provider_data": {},
|
||||
},
|
||||
]
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
assert result[0]["id"] == "msg_real123"
|
||||
|
||||
def test_handles_empty_list(self, model: OpenAIResponsesModel):
|
||||
"""Empty list should be returned unchanged."""
|
||||
list_input: list[dict[str, Any]] = []
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
assert result == []
|
||||
|
||||
def test_combined_scenario(self, model: OpenAIResponsesModel):
|
||||
"""Test a realistic scenario with multiple items needing different processing."""
|
||||
list_input = [
|
||||
{"type": "message", "content": "user input"},
|
||||
{"type": "reasoning", "summary": [], "provider_data": {"model": "gemini/gemini-3"}},
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_abc_123",
|
||||
"name": "get_weather",
|
||||
"provider_data": {"model": "gemini/gemini-3"},
|
||||
},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "call_abc_123",
|
||||
"output": '{"temp": 72}',
|
||||
},
|
||||
{
|
||||
"type": "message",
|
||||
"id": FAKE_RESPONSES_ID,
|
||||
"content": "The weather is 72F",
|
||||
"provider_data": {"model": "gemini/gemini-3"},
|
||||
},
|
||||
]
|
||||
|
||||
result = model._remove_openai_responses_api_incompatible_fields(list_input)
|
||||
|
||||
# Should have 4 items (reasoning with provider_data removed).
|
||||
assert len(result) == 4
|
||||
|
||||
# First item unchanged (no provider_data).
|
||||
assert result[0] == {"type": "message", "content": "user input"}
|
||||
|
||||
# Function call: __thought__ suffix removed, provider_data removed.
|
||||
assert result[1]["type"] == "function_call"
|
||||
assert result[1]["call_id"] == "call_abc_123"
|
||||
assert "provider_data" not in result[1]
|
||||
|
||||
# Function call output: __thought__ suffix removed, provider_data removed.
|
||||
assert result[2]["type"] == "function_call_output"
|
||||
assert result[2]["call_id"] == "call_abc_123"
|
||||
|
||||
# Last message: fake id removed, provider_data removed.
|
||||
assert result[3]["type"] == "message"
|
||||
assert result[3]["content"] == "The weather is 72F"
|
||||
assert "id" not in result[3]
|
||||
assert "provider_data" not in result[3]
|
||||
@@ -0,0 +1,149 @@
|
||||
import importlib
|
||||
|
||||
import pytest
|
||||
|
||||
from agents import Agent, responses_websocket_session
|
||||
from agents.models.multi_provider import MultiProvider
|
||||
from agents.models.openai_provider import OpenAIProvider
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_builds_shared_run_config():
|
||||
async with responses_websocket_session() as ws:
|
||||
assert isinstance(ws.provider, OpenAIProvider)
|
||||
assert ws.provider._use_responses is True
|
||||
assert ws.provider._use_responses_websocket is True
|
||||
assert isinstance(ws.run_config.model_provider, MultiProvider)
|
||||
assert ws.run_config.model_provider.openai_provider is ws.provider
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_preserves_openai_prefix_routing(monkeypatch):
|
||||
captured: dict[str, object] = {}
|
||||
sentinel = object()
|
||||
|
||||
def fake_get_model(model_name):
|
||||
captured["model_name"] = model_name
|
||||
return sentinel
|
||||
|
||||
async with responses_websocket_session() as ws:
|
||||
monkeypatch.setattr(ws.provider, "get_model", fake_get_model)
|
||||
|
||||
result = ws.run_config.model_provider.get_model("openai/gpt-4.1")
|
||||
|
||||
assert result is sentinel
|
||||
assert captured["model_name"] == "gpt-4.1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_can_preserve_openai_prefix_model_ids(monkeypatch):
|
||||
captured: dict[str, object] = {}
|
||||
sentinel = object()
|
||||
|
||||
def fake_get_model(model_name):
|
||||
captured["model_name"] = model_name
|
||||
return sentinel
|
||||
|
||||
async with responses_websocket_session(openai_prefix_mode="model_id") as ws:
|
||||
monkeypatch.setattr(ws.provider, "get_model", fake_get_model)
|
||||
|
||||
result = ws.run_config.model_provider.get_model("openai/gpt-4.1")
|
||||
|
||||
assert result is sentinel
|
||||
assert captured["model_name"] == "openai/gpt-4.1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_can_preserve_unknown_prefix_model_ids(monkeypatch):
|
||||
captured: dict[str, object] = {}
|
||||
sentinel = object()
|
||||
|
||||
def fake_get_model(model_name):
|
||||
captured["model_name"] = model_name
|
||||
return sentinel
|
||||
|
||||
async with responses_websocket_session(unknown_prefix_mode="model_id") as ws:
|
||||
monkeypatch.setattr(ws.provider, "get_model", fake_get_model)
|
||||
|
||||
result = ws.run_config.model_provider.get_model("openrouter/openai/gpt-4.1")
|
||||
|
||||
assert result is sentinel
|
||||
assert captured["model_name"] == "openrouter/openai/gpt-4.1"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_run_streamed_injects_run_config(monkeypatch):
|
||||
agent = Agent(name="test", instructions="Be concise.", model="gpt-4")
|
||||
captured = {}
|
||||
sentinel = object()
|
||||
|
||||
def fake_run_streamed(starting_agent, input, **kwargs):
|
||||
captured["starting_agent"] = starting_agent
|
||||
captured["input"] = input
|
||||
captured["kwargs"] = kwargs
|
||||
return sentinel
|
||||
|
||||
ws_module = importlib.import_module("agents.responses_websocket_session")
|
||||
monkeypatch.setattr(ws_module.Runner, "run_streamed", fake_run_streamed)
|
||||
|
||||
async with responses_websocket_session() as ws:
|
||||
result = ws.run_streamed(agent, "hello")
|
||||
|
||||
assert result is sentinel
|
||||
assert captured["starting_agent"] is agent
|
||||
assert captured["input"] == "hello"
|
||||
assert captured["kwargs"]["run_config"] is ws.run_config
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_run_injects_run_config(monkeypatch):
|
||||
agent = Agent(name="test", instructions="Be concise.", model="gpt-4")
|
||||
captured = {}
|
||||
sentinel = object()
|
||||
|
||||
async def fake_run(starting_agent, input, **kwargs):
|
||||
captured["starting_agent"] = starting_agent
|
||||
captured["input"] = input
|
||||
captured["kwargs"] = kwargs
|
||||
return sentinel
|
||||
|
||||
ws_module = importlib.import_module("agents.responses_websocket_session")
|
||||
monkeypatch.setattr(ws_module.Runner, "run", fake_run)
|
||||
|
||||
async with responses_websocket_session() as ws:
|
||||
result = await ws.run(agent, "hello")
|
||||
|
||||
assert result is sentinel
|
||||
assert captured["starting_agent"] is agent
|
||||
assert captured["input"] == "hello"
|
||||
assert captured["kwargs"]["run_config"] is ws.run_config
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_rejects_run_config_override():
|
||||
agent = Agent(name="test", instructions="Be concise.", model="gpt-4")
|
||||
|
||||
async with responses_websocket_session() as ws:
|
||||
with pytest.raises(ValueError, match="run_config"):
|
||||
ws.run_streamed(agent, "hello", run_config=object())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_context_manager_closes_provider(monkeypatch):
|
||||
close_calls: list[OpenAIProvider] = []
|
||||
|
||||
async def fake_aclose(self):
|
||||
close_calls.append(self)
|
||||
|
||||
monkeypatch.setattr(OpenAIProvider, "aclose", fake_aclose)
|
||||
|
||||
async with responses_websocket_session() as ws:
|
||||
provider = ws.provider
|
||||
|
||||
assert close_calls == [provider]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_websocket_session_does_not_expose_run_sync():
|
||||
async with responses_websocket_session() as ws:
|
||||
assert not hasattr(ws, "run_sync")
|
||||
@@ -0,0 +1,32 @@
|
||||
from agents.model_settings import ModelSettings
|
||||
from agents.models._trace import model_config_for_trace, sanitize_url_for_trace
|
||||
|
||||
|
||||
def test_sanitize_url_for_trace_strips_auth_query_and_fragment() -> None:
|
||||
assert (
|
||||
sanitize_url_for_trace("https://user:pass@example.com/v1?api-key=secret#fragment")
|
||||
== "https://example.com/v1"
|
||||
)
|
||||
assert sanitize_url_for_trace("https://example.com/v1?token=secret") == "https://example.com/v1"
|
||||
|
||||
|
||||
def test_model_config_for_trace_sanitizes_base_url_and_omits_request_extras() -> None:
|
||||
config = model_config_for_trace(
|
||||
ModelSettings(
|
||||
temperature=0.5,
|
||||
extra_headers={"Authorization": "Bearer provider-token"},
|
||||
extra_query={"api-key": "query-token"},
|
||||
extra_body={"secret": "body-token"},
|
||||
extra_args={"api_key": "arg-token"},
|
||||
),
|
||||
base_url="https://user:pass@example.com/v1?api-key=secret#fragment",
|
||||
extra_config={"model_impl": "test-model"},
|
||||
)
|
||||
|
||||
assert config["temperature"] == 0.5
|
||||
assert config["base_url"] == "https://example.com/v1"
|
||||
assert config["model_impl"] == "test-model"
|
||||
assert "extra_headers" not in config
|
||||
assert "extra_query" not in config
|
||||
assert "extra_body" not in config
|
||||
assert "extra_args" not in config
|
||||
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from agents import RunContextWrapper
|
||||
from agents.realtime.agent import RealtimeAgent
|
||||
|
||||
|
||||
def test_can_initialize_realtime_agent():
|
||||
agent = RealtimeAgent(name="test", instructions="Hello")
|
||||
assert agent.name == "test"
|
||||
assert agent.instructions == "Hello"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dynamic_instructions():
|
||||
agent = RealtimeAgent(name="test")
|
||||
assert agent.instructions is None
|
||||
|
||||
def _instructions(ctx, agt) -> str:
|
||||
assert ctx.context is None
|
||||
assert agt == agent
|
||||
return "Dynamic"
|
||||
|
||||
agent = RealtimeAgent(name="test", instructions=_instructions)
|
||||
instructions = await agent.get_system_prompt(RunContextWrapper(context=None))
|
||||
assert instructions == "Dynamic"
|
||||
|
||||
|
||||
def test_post_init_rejects_invalid_field_types() -> None:
|
||||
with pytest.raises(TypeError, match="RealtimeAgent name must be a string"):
|
||||
RealtimeAgent(name=1) # type: ignore[arg-type]
|
||||
with pytest.raises(TypeError, match="RealtimeAgent tools must be a list"):
|
||||
RealtimeAgent(name="x", tools="nope") # type: ignore[arg-type]
|
||||
with pytest.raises(TypeError, match="RealtimeAgent handoffs must be a list"):
|
||||
RealtimeAgent(name="x", handoffs="nope") # type: ignore[arg-type]
|
||||
with pytest.raises(TypeError, match="RealtimeAgent instructions must be"):
|
||||
RealtimeAgent(name="x", instructions=123) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def test_clone_does_not_mutate_original_lists() -> None:
|
||||
"""Cloning with a new list must not affect the original agent's lists."""
|
||||
original = RealtimeAgent(name="orig", tools=[], handoffs=[])
|
||||
new_tools: list[Any] = ["t1"]
|
||||
cloned = original.clone(tools=new_tools)
|
||||
assert original.tools == []
|
||||
assert len(cloned.tools) == 1
|
||||
assert cloned.tools is not original.tools
|
||||
# Shared reference when not overridden (documented shallow-copy behavior).
|
||||
assert cloned.handoffs is original.handoffs
|
||||
@@ -0,0 +1,44 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
|
||||
import pytest
|
||||
|
||||
from agents.realtime.events import RealtimeEventInfo, RealtimeRawModelEvent
|
||||
from agents.realtime.items import InputText, UserMessageItem
|
||||
from agents.realtime.model_events import RealtimeModelItemUpdatedEvent
|
||||
from agents.run_context import RunContextWrapper
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def app_server(monkeypatch: pytest.MonkeyPatch) -> ModuleType:
|
||||
app_dir = Path(__file__).parents[2] / "examples" / "realtime" / "app"
|
||||
monkeypatch.chdir(app_dir)
|
||||
monkeypatch.setenv("LOG_LEVEL", "DEBUG")
|
||||
module = importlib.import_module("examples.realtime.app.server")
|
||||
return importlib.reload(module)
|
||||
|
||||
|
||||
def test_item_updated_debug_summary_uses_concrete_event_type(
|
||||
app_server: ModuleType,
|
||||
caplog: pytest.LogCaptureFixture,
|
||||
) -> None:
|
||||
item = UserMessageItem(
|
||||
item_id="item-1",
|
||||
content=[InputText(text="sensitive transcript")],
|
||||
)
|
||||
event = RealtimeRawModelEvent(
|
||||
data=RealtimeModelItemUpdatedEvent(item=item),
|
||||
info=RealtimeEventInfo(context=RunContextWrapper(None)),
|
||||
)
|
||||
|
||||
with caplog.at_level(logging.DEBUG, logger=app_server.__name__):
|
||||
app_server.manager._log_debug_event("session-1", event)
|
||||
|
||||
assert "item_updated" in caplog.text
|
||||
assert "item-1" in caplog.text
|
||||
assert "input_text" in caplog.text
|
||||
assert "sensitive transcript" not in caplog.text
|
||||
@@ -0,0 +1,53 @@
|
||||
from openai.types.realtime.realtime_audio_formats import AudioPCM, AudioPCMA, AudioPCMU
|
||||
|
||||
from agents.realtime.audio_formats import to_realtime_audio_format
|
||||
|
||||
|
||||
def test_to_realtime_audio_format_from_strings():
|
||||
assert to_realtime_audio_format("pcm").type == "audio/pcm" # type: ignore[union-attr]
|
||||
assert to_realtime_audio_format("pcm16").type == "audio/pcm" # type: ignore[union-attr]
|
||||
assert to_realtime_audio_format("audio/pcm").type == "audio/pcm" # type: ignore[union-attr]
|
||||
assert to_realtime_audio_format("pcmu").type == "audio/pcmu" # type: ignore[union-attr]
|
||||
assert to_realtime_audio_format("audio/pcmu").type == "audio/pcmu" # type: ignore[union-attr]
|
||||
assert to_realtime_audio_format("g711_ulaw").type == "audio/pcmu" # type: ignore[union-attr]
|
||||
assert to_realtime_audio_format("pcma").type == "audio/pcma" # type: ignore[union-attr]
|
||||
assert to_realtime_audio_format("audio/pcma").type == "audio/pcma" # type: ignore[union-attr]
|
||||
assert to_realtime_audio_format("g711_alaw").type == "audio/pcma" # type: ignore[union-attr]
|
||||
|
||||
|
||||
def test_to_realtime_audio_format_passthrough_and_unknown_logs():
|
||||
fmt = AudioPCM(type="audio/pcm", rate=24000)
|
||||
# Passing a RealtimeAudioFormats should return the same instance
|
||||
assert to_realtime_audio_format(fmt) is fmt
|
||||
|
||||
# Unknown string returns None (and logs at debug level internally)
|
||||
assert to_realtime_audio_format("something_else") is None
|
||||
|
||||
|
||||
def test_to_realtime_audio_format_none():
|
||||
assert to_realtime_audio_format(None) is None
|
||||
|
||||
|
||||
def test_to_realtime_audio_format_from_mapping():
|
||||
pcm_exact_rate = to_realtime_audio_format({"type": "audio/pcm", "rate": 24000})
|
||||
assert isinstance(pcm_exact_rate, AudioPCM)
|
||||
assert pcm_exact_rate.rate == 24000
|
||||
|
||||
pcm = to_realtime_audio_format({"type": "audio/pcm", "rate": 16000})
|
||||
assert isinstance(pcm, AudioPCM)
|
||||
assert pcm.type == "audio/pcm"
|
||||
assert pcm.rate == 24000
|
||||
|
||||
pcm_default_rate = to_realtime_audio_format({"type": "audio/pcm"})
|
||||
assert isinstance(pcm_default_rate, AudioPCM)
|
||||
assert pcm_default_rate.rate == 24000
|
||||
|
||||
ulaw = to_realtime_audio_format({"type": "audio/pcmu"})
|
||||
assert isinstance(ulaw, AudioPCMU)
|
||||
assert ulaw.type == "audio/pcmu"
|
||||
|
||||
alaw = to_realtime_audio_format({"type": "audio/pcma"})
|
||||
assert isinstance(alaw, AudioPCMA)
|
||||
assert alaw.type == "audio/pcma"
|
||||
|
||||
assert to_realtime_audio_format({"type": "audio/unknown", "rate": 8000}) is None
|
||||
@@ -0,0 +1,380 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from openai.types.realtime.conversation_item_create_event import ConversationItemCreateEvent
|
||||
from openai.types.realtime.conversation_item_truncate_event import ConversationItemTruncateEvent
|
||||
from openai.types.realtime.input_audio_buffer_append_event import InputAudioBufferAppendEvent
|
||||
from openai.types.realtime.realtime_conversation_item_function_call_output import (
|
||||
RealtimeConversationItemFunctionCallOutput,
|
||||
)
|
||||
from pydantic import ValidationError
|
||||
|
||||
from agents.realtime.config import RealtimeModelTracingConfig
|
||||
from agents.realtime.model_inputs import (
|
||||
RealtimeModelSendAudio,
|
||||
RealtimeModelSendRawMessage,
|
||||
RealtimeModelSendToolOutput,
|
||||
RealtimeModelSendUserInput,
|
||||
RealtimeModelUserInputMessage,
|
||||
)
|
||||
from agents.realtime.openai_realtime import _ConversionHelper
|
||||
|
||||
|
||||
class TestConversionHelperTryConvertRawMessage:
|
||||
"""Test suite for _ConversionHelper.try_convert_raw_message method."""
|
||||
|
||||
def test_try_convert_raw_message_valid_session_update(self):
|
||||
"""Test converting a valid session.update raw message."""
|
||||
raw_message = RealtimeModelSendRawMessage(
|
||||
message={
|
||||
"type": "session.update",
|
||||
"other_data": {
|
||||
"session": {
|
||||
"model": "gpt-realtime-2.1",
|
||||
"type": "realtime",
|
||||
"modalities": ["text", "audio"],
|
||||
"voice": "ash",
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
result = _ConversionHelper.try_convert_raw_message(raw_message)
|
||||
|
||||
assert result is not None
|
||||
assert result.type == "session.update"
|
||||
|
||||
def test_try_convert_raw_message_valid_response_create(self):
|
||||
"""Test converting a valid response.create raw message."""
|
||||
raw_message = RealtimeModelSendRawMessage(
|
||||
message={
|
||||
"type": "response.create",
|
||||
"other_data": {},
|
||||
}
|
||||
)
|
||||
|
||||
result = _ConversionHelper.try_convert_raw_message(raw_message)
|
||||
|
||||
assert result is not None
|
||||
assert result.type == "response.create"
|
||||
|
||||
def test_try_convert_raw_message_invalid_type(self):
|
||||
"""Test converting an invalid message type returns None."""
|
||||
raw_message = RealtimeModelSendRawMessage(
|
||||
message={
|
||||
"type": "invalid.message.type",
|
||||
"other_data": {},
|
||||
}
|
||||
)
|
||||
|
||||
result = _ConversionHelper.try_convert_raw_message(raw_message)
|
||||
|
||||
assert result is None
|
||||
|
||||
def test_try_convert_raw_message_malformed_data(self):
|
||||
"""Test converting malformed message data returns None."""
|
||||
raw_message = RealtimeModelSendRawMessage(
|
||||
message={
|
||||
"type": "session.update",
|
||||
"other_data": {
|
||||
"session": "invalid_session_data" # Should be dict
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
result = _ConversionHelper.try_convert_raw_message(raw_message)
|
||||
|
||||
assert result is None
|
||||
|
||||
def test_try_convert_raw_message_missing_type(self):
|
||||
"""Test converting message without type returns None."""
|
||||
raw_message = RealtimeModelSendRawMessage(
|
||||
message={
|
||||
"type": "missing.type.test",
|
||||
"other_data": {"some": "data"},
|
||||
}
|
||||
)
|
||||
|
||||
result = _ConversionHelper.try_convert_raw_message(raw_message)
|
||||
|
||||
assert result is None
|
||||
|
||||
|
||||
class TestConversionHelperTracingConfig:
|
||||
"""Test suite for _ConversionHelper.convert_tracing_config method."""
|
||||
|
||||
def test_convert_tracing_config_none(self):
|
||||
"""Test converting None tracing config."""
|
||||
result = _ConversionHelper.convert_tracing_config(None)
|
||||
assert result is None
|
||||
|
||||
def test_convert_tracing_config_auto(self):
|
||||
"""Test converting 'auto' tracing config."""
|
||||
result = _ConversionHelper.convert_tracing_config("auto")
|
||||
assert result == "auto"
|
||||
|
||||
def test_convert_tracing_config_dict_full(self):
|
||||
"""Test converting full tracing config dict."""
|
||||
tracing_config: RealtimeModelTracingConfig = {
|
||||
"group_id": "test-group",
|
||||
"metadata": {"env": "test"},
|
||||
"workflow_name": "test-workflow",
|
||||
}
|
||||
|
||||
result = _ConversionHelper.convert_tracing_config(tracing_config)
|
||||
|
||||
assert result is not None
|
||||
assert result != "auto"
|
||||
assert result.group_id == "test-group"
|
||||
assert result.metadata == {"env": "test"}
|
||||
assert result.workflow_name == "test-workflow"
|
||||
|
||||
def test_convert_tracing_config_dict_partial(self):
|
||||
"""Test converting partial tracing config dict."""
|
||||
tracing_config: RealtimeModelTracingConfig = {
|
||||
"group_id": "test-group",
|
||||
}
|
||||
|
||||
result = _ConversionHelper.convert_tracing_config(tracing_config)
|
||||
|
||||
assert result is not None
|
||||
assert result != "auto"
|
||||
assert result.group_id == "test-group"
|
||||
assert result.metadata is None
|
||||
assert result.workflow_name is None
|
||||
|
||||
def test_convert_tracing_config_empty_dict(self):
|
||||
"""Test converting empty tracing config dict."""
|
||||
tracing_config: RealtimeModelTracingConfig = {}
|
||||
|
||||
result = _ConversionHelper.convert_tracing_config(tracing_config)
|
||||
|
||||
assert result is not None
|
||||
assert result != "auto"
|
||||
assert result.group_id is None
|
||||
assert result.metadata is None
|
||||
assert result.workflow_name is None
|
||||
|
||||
|
||||
class TestConversionHelperUserInput:
|
||||
"""Test suite for _ConversionHelper user input conversion methods."""
|
||||
|
||||
def test_convert_user_input_to_conversation_item_string(self):
|
||||
"""Test converting string user input to conversation item."""
|
||||
event = RealtimeModelSendUserInput(user_input="Hello, world!")
|
||||
|
||||
result = _ConversionHelper.convert_user_input_to_conversation_item(event)
|
||||
|
||||
assert result.type == "message"
|
||||
assert result.role == "user"
|
||||
assert result.content is not None
|
||||
assert len(result.content) == 1
|
||||
assert result.content[0].type == "input_text"
|
||||
assert result.content[0].text == "Hello, world!"
|
||||
|
||||
def test_convert_user_input_to_conversation_item_dict(self):
|
||||
"""Test converting dict user input to conversation item."""
|
||||
user_input_dict: RealtimeModelUserInputMessage = {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "Hello"},
|
||||
{"type": "input_text", "text": "World"},
|
||||
],
|
||||
}
|
||||
event = RealtimeModelSendUserInput(user_input=user_input_dict)
|
||||
|
||||
result = _ConversionHelper.convert_user_input_to_conversation_item(event)
|
||||
|
||||
assert result.type == "message"
|
||||
assert result.role == "user"
|
||||
assert result.content is not None
|
||||
assert len(result.content) == 2
|
||||
assert result.content[0].type == "input_text"
|
||||
assert result.content[0].text == "Hello"
|
||||
assert result.content[1].type == "input_text"
|
||||
assert result.content[1].text == "World"
|
||||
|
||||
def test_convert_user_input_to_conversation_item_dict_empty_content(self):
|
||||
"""Test converting dict user input with empty content."""
|
||||
user_input_dict: RealtimeModelUserInputMessage = {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [],
|
||||
}
|
||||
event = RealtimeModelSendUserInput(user_input=user_input_dict)
|
||||
|
||||
result = _ConversionHelper.convert_user_input_to_conversation_item(event)
|
||||
|
||||
assert result.type == "message"
|
||||
assert result.role == "user"
|
||||
assert result.content is not None
|
||||
assert len(result.content) == 0
|
||||
|
||||
def test_convert_user_input_to_item_create(self):
|
||||
"""Test converting user input to item create event."""
|
||||
event = RealtimeModelSendUserInput(user_input="Test message")
|
||||
|
||||
result = _ConversionHelper.convert_user_input_to_item_create(event)
|
||||
|
||||
assert isinstance(result, ConversationItemCreateEvent)
|
||||
assert result.type == "conversation.item.create"
|
||||
assert result.item.type == "message"
|
||||
assert result.item.role == "user"
|
||||
|
||||
|
||||
class TestConversionHelperAudio:
|
||||
"""Test suite for _ConversionHelper.convert_audio_to_input_audio_buffer_append."""
|
||||
|
||||
def test_convert_audio_to_input_audio_buffer_append(self):
|
||||
"""Test converting audio data to input audio buffer append event."""
|
||||
audio_data = b"test audio data"
|
||||
event = RealtimeModelSendAudio(audio=audio_data, commit=False)
|
||||
|
||||
result = _ConversionHelper.convert_audio_to_input_audio_buffer_append(event)
|
||||
|
||||
assert isinstance(result, InputAudioBufferAppendEvent)
|
||||
assert result.type == "input_audio_buffer.append"
|
||||
|
||||
# Verify base64 encoding
|
||||
expected_b64 = base64.b64encode(audio_data).decode("utf-8")
|
||||
assert result.audio == expected_b64
|
||||
|
||||
def test_convert_audio_to_input_audio_buffer_append_empty(self):
|
||||
"""Test converting empty audio data."""
|
||||
audio_data = b""
|
||||
event = RealtimeModelSendAudio(audio=audio_data, commit=True)
|
||||
|
||||
result = _ConversionHelper.convert_audio_to_input_audio_buffer_append(event)
|
||||
|
||||
assert isinstance(result, InputAudioBufferAppendEvent)
|
||||
assert result.type == "input_audio_buffer.append"
|
||||
assert result.audio == ""
|
||||
|
||||
def test_convert_audio_to_input_audio_buffer_append_large_data(self):
|
||||
"""Test converting large audio data."""
|
||||
audio_data = b"x" * 10000 # Large audio buffer
|
||||
event = RealtimeModelSendAudio(audio=audio_data, commit=False)
|
||||
|
||||
result = _ConversionHelper.convert_audio_to_input_audio_buffer_append(event)
|
||||
|
||||
assert isinstance(result, InputAudioBufferAppendEvent)
|
||||
assert result.type == "input_audio_buffer.append"
|
||||
|
||||
# Verify it can be decoded back
|
||||
decoded = base64.b64decode(result.audio)
|
||||
assert decoded == audio_data
|
||||
|
||||
|
||||
class TestConversionHelperToolOutput:
|
||||
"""Test suite for _ConversionHelper.convert_tool_output method."""
|
||||
|
||||
def test_convert_tool_output(self):
|
||||
"""Test converting tool output to conversation item create event."""
|
||||
mock_tool_call = Mock()
|
||||
mock_tool_call.call_id = "call_123"
|
||||
|
||||
event = RealtimeModelSendToolOutput(
|
||||
tool_call=mock_tool_call,
|
||||
output="Function executed successfully",
|
||||
start_response=False,
|
||||
)
|
||||
|
||||
result = _ConversionHelper.convert_tool_output(event)
|
||||
|
||||
assert isinstance(result, ConversationItemCreateEvent)
|
||||
assert result.type == "conversation.item.create"
|
||||
assert result.item.type == "function_call_output"
|
||||
assert isinstance(result.item, RealtimeConversationItemFunctionCallOutput)
|
||||
tool_output_item = result.item
|
||||
assert tool_output_item.output == "Function executed successfully"
|
||||
assert tool_output_item.call_id == "call_123"
|
||||
|
||||
def test_convert_tool_output_no_call_id(self):
|
||||
"""Test converting tool output with None call_id."""
|
||||
mock_tool_call = Mock()
|
||||
mock_tool_call.call_id = None
|
||||
|
||||
event = RealtimeModelSendToolOutput(
|
||||
tool_call=mock_tool_call,
|
||||
output="Output without call ID",
|
||||
start_response=False,
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
ValidationError,
|
||||
match="1 validation error for RealtimeConversationItemFunctionCallOutput",
|
||||
):
|
||||
_ConversionHelper.convert_tool_output(event)
|
||||
|
||||
def test_convert_tool_output_empty_output(self):
|
||||
"""Test converting tool output with empty output."""
|
||||
mock_tool_call = Mock()
|
||||
mock_tool_call.call_id = "call_456"
|
||||
|
||||
event = RealtimeModelSendToolOutput(
|
||||
tool_call=mock_tool_call,
|
||||
output="",
|
||||
start_response=True,
|
||||
)
|
||||
|
||||
result = _ConversionHelper.convert_tool_output(event)
|
||||
|
||||
assert isinstance(result, ConversationItemCreateEvent)
|
||||
assert result.type == "conversation.item.create"
|
||||
assert isinstance(result.item, RealtimeConversationItemFunctionCallOutput)
|
||||
assert result.item.output == ""
|
||||
assert result.item.call_id == "call_456"
|
||||
|
||||
|
||||
class TestConversionHelperInterrupt:
|
||||
"""Test suite for _ConversionHelper.convert_interrupt method."""
|
||||
|
||||
def test_convert_interrupt(self):
|
||||
"""Test converting interrupt parameters to conversation item truncate event."""
|
||||
current_item_id = "item_789"
|
||||
current_audio_content_index = 2
|
||||
elapsed_time_ms = 1500
|
||||
|
||||
result = _ConversionHelper.convert_interrupt(
|
||||
current_item_id, current_audio_content_index, elapsed_time_ms
|
||||
)
|
||||
|
||||
assert isinstance(result, ConversationItemTruncateEvent)
|
||||
assert result.type == "conversation.item.truncate"
|
||||
assert result.item_id == "item_789"
|
||||
assert result.content_index == 2
|
||||
assert result.audio_end_ms == 1500
|
||||
|
||||
def test_convert_interrupt_zero_time(self):
|
||||
"""Test converting interrupt with zero elapsed time."""
|
||||
result = _ConversionHelper.convert_interrupt("item_1", 0, 0)
|
||||
|
||||
assert isinstance(result, ConversationItemTruncateEvent)
|
||||
assert result.type == "conversation.item.truncate"
|
||||
assert result.item_id == "item_1"
|
||||
assert result.content_index == 0
|
||||
assert result.audio_end_ms == 0
|
||||
|
||||
def test_convert_interrupt_large_values(self):
|
||||
"""Test converting interrupt with large values."""
|
||||
result = _ConversionHelper.convert_interrupt("item_xyz", 99, 999999)
|
||||
|
||||
assert isinstance(result, ConversationItemTruncateEvent)
|
||||
assert result.type == "conversation.item.truncate"
|
||||
assert result.item_id == "item_xyz"
|
||||
assert result.content_index == 99
|
||||
assert result.audio_end_ms == 999999
|
||||
|
||||
def test_convert_interrupt_empty_item_id(self):
|
||||
"""Test converting interrupt with empty item ID."""
|
||||
result = _ConversionHelper.convert_interrupt("", 1, 100)
|
||||
|
||||
assert isinstance(result, ConversationItemTruncateEvent)
|
||||
assert result.type == "conversation.item.truncate"
|
||||
assert result.item_id == ""
|
||||
assert result.content_index == 1
|
||||
assert result.audio_end_ms == 100
|
||||
@@ -0,0 +1,35 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
from websockets.asyncio.client import ClientConnection
|
||||
|
||||
from agents.realtime.openai_realtime import OpenAIRealtimeWebSocketModel
|
||||
|
||||
|
||||
class _DummyWS:
|
||||
def __init__(self) -> None:
|
||||
self.sent: list[str] = []
|
||||
|
||||
async def send(self, data: str) -> None:
|
||||
self.sent.append(data)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_auto_interrupt_on_vad_speech_started(monkeypatch: Any) -> None:
|
||||
model = OpenAIRealtimeWebSocketModel()
|
||||
|
||||
called = {"interrupt": False}
|
||||
|
||||
async def _fake_interrupt(event: Any) -> None:
|
||||
called["interrupt"] = True
|
||||
|
||||
# Prevent network use; _websocket only needed for other paths
|
||||
model._websocket = cast(ClientConnection, _DummyWS())
|
||||
monkeypatch.setattr(model, "_send_interrupt", _fake_interrupt)
|
||||
|
||||
# This event previously triggered an interrupt; now it should be ignored
|
||||
await model._handle_ws_event({"type": "input_audio_buffer.speech_started"})
|
||||
|
||||
assert called["interrupt"] is False
|
||||
@@ -0,0 +1,80 @@
|
||||
from openai.types.realtime.realtime_conversation_item_assistant_message import (
|
||||
Content as AssistantMessageContent,
|
||||
RealtimeConversationItemAssistantMessage,
|
||||
)
|
||||
from openai.types.realtime.realtime_conversation_item_system_message import (
|
||||
Content as SystemMessageContent,
|
||||
RealtimeConversationItemSystemMessage,
|
||||
)
|
||||
from openai.types.realtime.realtime_conversation_item_user_message import (
|
||||
Content as UserMessageContent,
|
||||
RealtimeConversationItemUserMessage,
|
||||
)
|
||||
|
||||
from agents.realtime.items import (
|
||||
AssistantMessageItem,
|
||||
RealtimeMessageItem,
|
||||
SystemMessageItem,
|
||||
UserMessageItem,
|
||||
)
|
||||
from agents.realtime.openai_realtime import _ConversionHelper
|
||||
|
||||
|
||||
def test_user_message_conversion() -> None:
|
||||
item = RealtimeConversationItemUserMessage(
|
||||
id="123",
|
||||
type="message",
|
||||
role="user",
|
||||
content=[
|
||||
UserMessageContent(type="input_text", text=None),
|
||||
],
|
||||
)
|
||||
|
||||
converted: RealtimeMessageItem = _ConversionHelper.conversation_item_to_realtime_message_item(
|
||||
item, None
|
||||
)
|
||||
|
||||
assert isinstance(converted, UserMessageItem)
|
||||
|
||||
item = RealtimeConversationItemUserMessage(
|
||||
id="123",
|
||||
type="message",
|
||||
role="user",
|
||||
content=[
|
||||
UserMessageContent(type="input_audio", audio=None),
|
||||
],
|
||||
)
|
||||
|
||||
converted = _ConversionHelper.conversation_item_to_realtime_message_item(item, None)
|
||||
|
||||
assert isinstance(converted, UserMessageItem)
|
||||
|
||||
|
||||
def test_assistant_message_conversion() -> None:
|
||||
item = RealtimeConversationItemAssistantMessage(
|
||||
id="123",
|
||||
type="message",
|
||||
role="assistant",
|
||||
content=[AssistantMessageContent(type="output_text", text=None)],
|
||||
)
|
||||
|
||||
converted: RealtimeMessageItem = _ConversionHelper.conversation_item_to_realtime_message_item(
|
||||
item, None
|
||||
)
|
||||
|
||||
assert isinstance(converted, AssistantMessageItem)
|
||||
|
||||
|
||||
def test_system_message_conversion() -> None:
|
||||
item = RealtimeConversationItemSystemMessage(
|
||||
id="123",
|
||||
type="message",
|
||||
role="system",
|
||||
content=[SystemMessageContent(type="input_text", text=None)],
|
||||
)
|
||||
|
||||
converted: RealtimeMessageItem = _ConversionHelper.conversation_item_to_realtime_message_item(
|
||||
item, None
|
||||
)
|
||||
|
||||
assert isinstance(converted, SystemMessageItem)
|
||||
@@ -0,0 +1,51 @@
|
||||
from typing import get_args
|
||||
|
||||
import agents.realtime as realtime
|
||||
from agents.realtime.model_events import RealtimeModelEvent
|
||||
from agents.usage import Usage
|
||||
|
||||
|
||||
def test_all_events_have_type() -> None:
|
||||
"""Test that all events have a type."""
|
||||
events = get_args(RealtimeModelEvent)
|
||||
assert len(events) > 0
|
||||
for event in events:
|
||||
assert event.type is not None
|
||||
assert isinstance(event.type, str)
|
||||
|
||||
|
||||
def test_usage_event_types_are_publicly_exported() -> None:
|
||||
expected_exports = {
|
||||
"RealtimeModelCachedTokensDetails",
|
||||
"RealtimeModelInputTokensDetails",
|
||||
"RealtimeModelOutputTokensDetails",
|
||||
"RealtimeModelUsageEvent",
|
||||
}
|
||||
|
||||
assert expected_exports <= set(realtime.__all__)
|
||||
for name in expected_exports:
|
||||
assert getattr(realtime, name) is not None
|
||||
|
||||
|
||||
def test_custom_model_can_construct_typed_usage_without_openai_types() -> None:
|
||||
event = realtime.RealtimeModelUsageEvent(
|
||||
usage=Usage(requests=1, input_tokens=8, output_tokens=5, total_tokens=13),
|
||||
input_tokens_details=realtime.RealtimeModelInputTokensDetails(
|
||||
text_tokens=2,
|
||||
audio_tokens=6,
|
||||
cached_tokens=3,
|
||||
cached_tokens_details=realtime.RealtimeModelCachedTokensDetails(
|
||||
text_tokens=1,
|
||||
audio_tokens=2,
|
||||
),
|
||||
),
|
||||
output_tokens_details=realtime.RealtimeModelOutputTokensDetails(
|
||||
text_tokens=1,
|
||||
audio_tokens=4,
|
||||
),
|
||||
)
|
||||
|
||||
assert event.input_tokens_details is not None
|
||||
assert event.input_tokens_details.audio_tokens == 6
|
||||
assert event.output_tokens_details is not None
|
||||
assert event.output_tokens_details.audio_tokens == 4
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,295 @@
|
||||
from typing import cast
|
||||
|
||||
import pytest
|
||||
from openai.types.realtime.realtime_conversation_item_user_message import (
|
||||
RealtimeConversationItemUserMessage,
|
||||
)
|
||||
from openai.types.realtime.realtime_response_usage import RealtimeResponseUsage
|
||||
from openai.types.realtime.realtime_tracing_config import (
|
||||
TracingConfiguration,
|
||||
)
|
||||
|
||||
from agents import Agent, function_tool, tool_namespace
|
||||
from agents.exceptions import UserError
|
||||
from agents.handoffs import handoff
|
||||
from agents.realtime.config import RealtimeModelTracingConfig
|
||||
from agents.realtime.model_inputs import (
|
||||
RealtimeModelSendRawMessage,
|
||||
RealtimeModelSendUserInput,
|
||||
RealtimeModelUserInputMessage,
|
||||
)
|
||||
from agents.realtime.openai_realtime import (
|
||||
OpenAIRealtimeWebSocketModel,
|
||||
_ConversionHelper,
|
||||
get_api_key,
|
||||
)
|
||||
from agents.tool import Tool
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_api_key_from_env(monkeypatch):
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "env-key")
|
||||
assert await get_api_key(None) == "env-key"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_api_key_from_callable_async():
|
||||
async def f():
|
||||
return "k"
|
||||
|
||||
assert await get_api_key(f) == "k"
|
||||
|
||||
|
||||
def test_try_convert_raw_message_invalid_returns_none():
|
||||
msg = RealtimeModelSendRawMessage(message={"type": "invalid.event", "other_data": {}})
|
||||
assert _ConversionHelper.try_convert_raw_message(msg) is None
|
||||
|
||||
|
||||
def test_convert_response_usage_preserves_modality_details():
|
||||
event = _ConversionHelper.convert_response_usage(
|
||||
RealtimeResponseUsage.model_validate(
|
||||
{
|
||||
"total_tokens": 253,
|
||||
"input_tokens": 132,
|
||||
"output_tokens": 121,
|
||||
"input_token_details": {
|
||||
"text_tokens": 119,
|
||||
"audio_tokens": 13,
|
||||
"image_tokens": 0,
|
||||
"cached_tokens": 64,
|
||||
"cached_tokens_details": {
|
||||
"text_tokens": 60,
|
||||
"audio_tokens": 4,
|
||||
"image_tokens": 0,
|
||||
},
|
||||
},
|
||||
"output_token_details": {"text_tokens": 30, "audio_tokens": 91},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert event.usage.requests == 1
|
||||
assert event.usage.input_tokens == 132
|
||||
assert event.usage.output_tokens == 121
|
||||
assert event.usage.total_tokens == 253
|
||||
assert event.usage.input_tokens_details.cached_tokens == 64
|
||||
assert event.input_tokens_details is not None
|
||||
assert event.input_tokens_details.text_tokens == 119
|
||||
assert event.input_tokens_details.audio_tokens == 13
|
||||
assert event.input_tokens_details.image_tokens == 0
|
||||
assert event.input_tokens_details.cached_tokens == 64
|
||||
assert event.input_tokens_details.cached_tokens_details is not None
|
||||
assert event.input_tokens_details.cached_tokens_details.text_tokens == 60
|
||||
assert event.input_tokens_details.cached_tokens_details.audio_tokens == 4
|
||||
assert event.input_tokens_details.cached_tokens_details.image_tokens == 0
|
||||
assert event.output_tokens_details is not None
|
||||
assert event.output_tokens_details.text_tokens == 30
|
||||
assert event.output_tokens_details.audio_tokens == 91
|
||||
|
||||
|
||||
def test_convert_response_usage_preserves_missing_details_and_derives_total():
|
||||
event = _ConversionHelper.convert_response_usage(
|
||||
RealtimeResponseUsage.model_validate(
|
||||
{
|
||||
"input_tokens": 12,
|
||||
"output_tokens": 3,
|
||||
"input_token_details": {"audio_tokens": 0},
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
assert event.usage.total_tokens == 15
|
||||
assert event.input_tokens_details is not None
|
||||
assert event.input_tokens_details.audio_tokens == 0
|
||||
assert event.input_tokens_details.text_tokens is None
|
||||
assert event.input_tokens_details.cached_tokens is None
|
||||
assert event.input_tokens_details.cached_tokens_details is None
|
||||
assert event.output_tokens_details is None
|
||||
|
||||
|
||||
def test_convert_user_input_to_conversation_item_dict_and_str():
|
||||
# Dict with mixed, including unknown parts (silently skipped)
|
||||
dict_input_any = {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "hello"},
|
||||
{"type": "input_image", "image_url": "http://x/y.png", "detail": "auto"},
|
||||
{"type": "bogus", "x": 1},
|
||||
],
|
||||
}
|
||||
event = RealtimeModelSendUserInput(
|
||||
user_input=cast(RealtimeModelUserInputMessage, dict_input_any)
|
||||
)
|
||||
item_any = _ConversionHelper.convert_user_input_to_conversation_item(event)
|
||||
item = cast(RealtimeConversationItemUserMessage, item_any)
|
||||
assert item.role == "user"
|
||||
|
||||
# String input becomes input_text
|
||||
event2 = RealtimeModelSendUserInput(user_input="hi")
|
||||
item2_any = _ConversionHelper.convert_user_input_to_conversation_item(event2)
|
||||
item2 = cast(RealtimeConversationItemUserMessage, item2_any)
|
||||
assert item2.content[0].type == "input_text"
|
||||
|
||||
|
||||
def test_convert_user_input_dict_skips_invalid_input_text_parts():
|
||||
"""input_text parts with missing/non-string text must be skipped, not
|
||||
forwarded as Content(text=None) which the realtime API rejects."""
|
||||
dict_input_any = {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text"}, # missing text
|
||||
{"type": "input_text", "text": 123}, # non-string text
|
||||
{"type": "input_text", "text": "ok"}, # valid
|
||||
],
|
||||
}
|
||||
event = RealtimeModelSendUserInput(
|
||||
user_input=cast(RealtimeModelUserInputMessage, dict_input_any)
|
||||
)
|
||||
item = cast(
|
||||
RealtimeConversationItemUserMessage,
|
||||
_ConversionHelper.convert_user_input_to_conversation_item(event),
|
||||
)
|
||||
assert item.content is not None
|
||||
assert len(item.content) == 1
|
||||
assert item.content[0].type == "input_text"
|
||||
assert item.content[0].text == "ok"
|
||||
|
||||
|
||||
def test_convert_tracing_config_variants():
|
||||
from agents.realtime.openai_realtime import _ConversionHelper as CH
|
||||
|
||||
assert CH.convert_tracing_config(None) is None
|
||||
assert CH.convert_tracing_config("auto") == "auto"
|
||||
cfg: RealtimeModelTracingConfig = {
|
||||
"group_id": "g",
|
||||
"metadata": {"k": "v"},
|
||||
"workflow_name": "wf",
|
||||
}
|
||||
oc_any = CH.convert_tracing_config(cfg)
|
||||
oc = cast(TracingConfiguration, oc_any)
|
||||
assert oc.group_id == "g"
|
||||
assert oc.workflow_name == "wf"
|
||||
|
||||
|
||||
def test_tools_to_session_tools_raises_on_non_function_tool():
|
||||
class NotFunctionTool:
|
||||
def __init__(self):
|
||||
self.name = "x"
|
||||
|
||||
m = OpenAIRealtimeWebSocketModel()
|
||||
with pytest.raises(UserError):
|
||||
m._tools_to_session_tools(cast(list[Tool], [NotFunctionTool()]), [])
|
||||
|
||||
|
||||
def test_tools_to_session_tools_includes_handoffs():
|
||||
a = Agent(name="a")
|
||||
h = handoff(a)
|
||||
m = OpenAIRealtimeWebSocketModel()
|
||||
out = m._tools_to_session_tools([], [h])
|
||||
assert out[0].name is not None and out[0].name.startswith("transfer_to_")
|
||||
|
||||
|
||||
def test_tools_to_session_tools_rejects_duplicate_function_tool_names():
|
||||
tool_one = function_tool(lambda: "one", name_override="lookup_account")
|
||||
tool_two = function_tool(lambda: "two", name_override="lookup_account")
|
||||
m = OpenAIRealtimeWebSocketModel()
|
||||
|
||||
with pytest.raises(
|
||||
UserError,
|
||||
match=("Duplicate Realtime tool name found: 'lookup_account' \\(2 function tools\\)"),
|
||||
):
|
||||
m._tools_to_session_tools([tool_one, tool_two], [])
|
||||
|
||||
|
||||
def test_tools_to_session_tools_rejects_function_handoff_name_conflict():
|
||||
tool = function_tool(lambda: "ok", name_override="transfer_to_billing")
|
||||
h = handoff(Agent(name="billing"), tool_name_override="transfer_to_billing")
|
||||
m = OpenAIRealtimeWebSocketModel()
|
||||
|
||||
with pytest.raises(
|
||||
UserError,
|
||||
match=(
|
||||
"Duplicate Realtime tool name found: "
|
||||
"'transfer_to_billing' \\(function tool and handoff\\)"
|
||||
),
|
||||
):
|
||||
m._tools_to_session_tools([tool], [h])
|
||||
|
||||
|
||||
def test_tools_to_session_tools_ignores_disabled_function_tool_name_conflict():
|
||||
tool = function_tool(
|
||||
lambda: "ok",
|
||||
name_override="transfer_to_billing",
|
||||
is_enabled=False,
|
||||
)
|
||||
h = handoff(Agent(name="billing"), tool_name_override="transfer_to_billing")
|
||||
m = OpenAIRealtimeWebSocketModel()
|
||||
|
||||
out = m._tools_to_session_tools([tool], [h])
|
||||
|
||||
assert [tool.name for tool in out] == ["transfer_to_billing"]
|
||||
|
||||
|
||||
def test_tools_to_session_tools_omits_disabled_function_tool():
|
||||
tool = function_tool(
|
||||
lambda: "ok",
|
||||
name_override="hidden_tool",
|
||||
is_enabled=False,
|
||||
)
|
||||
m = OpenAIRealtimeWebSocketModel()
|
||||
|
||||
out = m._tools_to_session_tools([tool], [])
|
||||
|
||||
assert out == []
|
||||
|
||||
|
||||
def test_tools_to_session_tools_ignores_disabled_handoff_name_conflict():
|
||||
tool = function_tool(lambda: "ok", name_override="transfer_to_billing")
|
||||
h = handoff(
|
||||
Agent(name="billing"),
|
||||
tool_name_override="transfer_to_billing",
|
||||
is_enabled=False,
|
||||
)
|
||||
m = OpenAIRealtimeWebSocketModel()
|
||||
|
||||
out = m._tools_to_session_tools([tool], [h])
|
||||
|
||||
assert [tool.name for tool in out] == ["transfer_to_billing"]
|
||||
|
||||
|
||||
def test_tools_to_session_tools_rejects_duplicate_handoff_names():
|
||||
handoff_one = handoff(Agent(name="billing"), tool_name_override="transfer_to_support")
|
||||
handoff_two = handoff(Agent(name="technical"), tool_name_override="transfer_to_support")
|
||||
m = OpenAIRealtimeWebSocketModel()
|
||||
|
||||
with pytest.raises(
|
||||
UserError,
|
||||
match=("Duplicate Realtime tool name found: 'transfer_to_support' \\(2 handoffs\\)"),
|
||||
):
|
||||
m._tools_to_session_tools([], [handoff_one, handoff_two])
|
||||
|
||||
|
||||
def test_tools_to_session_tools_rejects_namespaced_function_tools():
|
||||
tool = tool_namespace(
|
||||
name="crm",
|
||||
description="CRM tools",
|
||||
tools=[function_tool(lambda customer_id: customer_id, name_override="lookup_account")],
|
||||
)[0]
|
||||
m = OpenAIRealtimeWebSocketModel()
|
||||
|
||||
with pytest.raises(UserError, match="tool_namespace\\(\\)"):
|
||||
m._tools_to_session_tools([tool], [])
|
||||
|
||||
|
||||
def test_tools_to_session_tools_rejects_deferred_function_tools():
|
||||
tool = function_tool(
|
||||
lambda customer_id: customer_id,
|
||||
name_override="lookup_account",
|
||||
defer_loading=True,
|
||||
)
|
||||
m = OpenAIRealtimeWebSocketModel()
|
||||
|
||||
with pytest.raises(UserError, match="defer_loading=True"):
|
||||
m._tools_to_session_tools([tool], [])
|
||||
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from agents.exceptions import UserError
|
||||
from agents.realtime.openai_realtime import OpenAIRealtimeSIPModel
|
||||
|
||||
|
||||
class _DummyWebSocket:
|
||||
def __init__(self) -> None:
|
||||
self.sent_messages: list[str] = []
|
||||
self.closed = False
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self): # pragma: no cover - simple termination
|
||||
raise StopAsyncIteration
|
||||
|
||||
async def send(self, data: str) -> None:
|
||||
self.sent_messages.append(data)
|
||||
|
||||
async def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sip_model_uses_call_id_in_url(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
dummy_ws = _DummyWebSocket()
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
async def fake_connect(url: str, **kwargs):
|
||||
captured["url"] = url
|
||||
captured["kwargs"] = kwargs
|
||||
return dummy_ws
|
||||
|
||||
monkeypatch.setattr("agents.realtime.openai_realtime.websockets.connect", fake_connect)
|
||||
|
||||
model = OpenAIRealtimeSIPModel()
|
||||
await model.connect({"api_key": "sk-test", "call_id": "call_789", "initial_model_settings": {}})
|
||||
|
||||
assert captured["url"] == "wss://api.openai.com/v1/realtime?call_id=call_789"
|
||||
|
||||
await asyncio.sleep(0) # allow listener task to start and finish
|
||||
await model.close()
|
||||
assert dummy_ws.closed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sip_model_requires_call_id() -> None:
|
||||
model = OpenAIRealtimeSIPModel()
|
||||
|
||||
with pytest.raises(UserError):
|
||||
await model.connect({"api_key": "sk-test", "initial_model_settings": {}})
|
||||
@@ -0,0 +1,193 @@
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from agents.realtime._default_tracker import ModelAudioTracker
|
||||
from agents.realtime.model import RealtimePlaybackTracker
|
||||
from agents.realtime.model_inputs import RealtimeModelSendInterrupt
|
||||
from agents.realtime.openai_realtime import OpenAIRealtimeWebSocketModel
|
||||
|
||||
|
||||
class TestPlaybackTracker:
|
||||
"""Test playback tracker functionality for interrupt timing."""
|
||||
|
||||
@pytest.fixture
|
||||
def model(self):
|
||||
"""Create a fresh model instance for each test."""
|
||||
return OpenAIRealtimeWebSocketModel()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interrupt_timing_with_custom_playback_tracker(self, model):
|
||||
"""Test interrupt uses custom playback tracker elapsed time instead of default timing."""
|
||||
|
||||
# Create custom tracker and set elapsed time
|
||||
custom_tracker = RealtimePlaybackTracker()
|
||||
custom_tracker.set_audio_format("pcm16")
|
||||
custom_tracker.on_play_ms("item_1", 1, 500.0) # content_index 1, 500ms played
|
||||
|
||||
# Set up model with custom tracker directly
|
||||
model._playback_tracker = custom_tracker
|
||||
|
||||
# Mock send_raw_message to capture interrupt
|
||||
model._send_raw_message = AsyncMock()
|
||||
|
||||
# Send interrupt
|
||||
|
||||
await model._send_interrupt(RealtimeModelSendInterrupt())
|
||||
|
||||
# Should use custom tracker's 500ms elapsed time
|
||||
truncate_events = [
|
||||
call.args[0]
|
||||
for call in model._send_raw_message.await_args_list
|
||||
if getattr(call.args[0], "type", None) == "conversation.item.truncate"
|
||||
]
|
||||
assert truncate_events
|
||||
assert truncate_events[0].audio_end_ms == 500
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interrupt_skipped_when_no_audio_playing(self, model):
|
||||
"""Test interrupt returns early when no audio is currently playing."""
|
||||
model._send_raw_message = AsyncMock()
|
||||
|
||||
# No audio playing (default state)
|
||||
|
||||
await model._send_interrupt(RealtimeModelSendInterrupt())
|
||||
|
||||
# Should not send any interrupt message
|
||||
model._send_raw_message.assert_not_called()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interrupt_skips_when_elapsed_exceeds_audio_length(self, model):
|
||||
"""Test interrupt skips truncation when playback appears complete."""
|
||||
model._send_raw_message = AsyncMock()
|
||||
model._audio_state_tracker.set_audio_format("pcm16")
|
||||
|
||||
# 48_000 bytes of PCM16 at 24kHz equals ~1000ms of audio.
|
||||
model._audio_state_tracker.on_audio_delta("item_1", 0, b"a" * 48_000)
|
||||
model._playback_tracker = RealtimePlaybackTracker()
|
||||
model._playback_tracker.on_play_ms("item_1", 0, 2000.0)
|
||||
|
||||
await model._send_interrupt(RealtimeModelSendInterrupt())
|
||||
|
||||
truncate_events = [
|
||||
call.args[0]
|
||||
for call in model._send_raw_message.await_args_list
|
||||
if getattr(call.args[0], "type", None) == "conversation.item.truncate"
|
||||
]
|
||||
assert truncate_events == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interrupt_sends_truncate_when_ongoing_response(self, model):
|
||||
"""Test interrupt still truncates while response is ongoing."""
|
||||
model._ongoing_response = True
|
||||
model._send_raw_message = AsyncMock()
|
||||
model._audio_state_tracker.set_audio_format("pcm16")
|
||||
|
||||
# 48_000 bytes of PCM16 at 24kHz equals ~1000ms of audio.
|
||||
model._audio_state_tracker.on_audio_delta("item_1", 0, b"a" * 48_000)
|
||||
model._playback_tracker = RealtimePlaybackTracker()
|
||||
model._playback_tracker.on_play_ms("item_1", 0, 2000.0)
|
||||
|
||||
await model._send_interrupt(RealtimeModelSendInterrupt())
|
||||
|
||||
truncate_events = [
|
||||
call.args[0]
|
||||
for call in model._send_raw_message.await_args_list
|
||||
if getattr(call.args[0], "type", None) == "conversation.item.truncate"
|
||||
]
|
||||
assert truncate_events
|
||||
assert truncate_events[0].audio_end_ms == 2000
|
||||
|
||||
def test_audio_delta_before_set_audio_format_does_not_raise(self):
|
||||
"""ModelAudioTracker must tolerate audio deltas before a format is negotiated.
|
||||
|
||||
For transcription-only sessions or session payloads that omit an audio
|
||||
format, ``set_audio_format`` is never called. Previously, the first
|
||||
``on_audio_delta`` call raised ``AttributeError`` because ``self._format``
|
||||
was unset. The length calculator already accepts ``None`` as the
|
||||
unknown-format fallback, so the tracker should pass that through.
|
||||
"""
|
||||
|
||||
tracker = ModelAudioTracker()
|
||||
# Intentionally do NOT call set_audio_format here.
|
||||
tracker.on_audio_delta("item_1", 0, b"test")
|
||||
|
||||
state = tracker.get_state("item_1", 0)
|
||||
assert state is not None
|
||||
# With no format, calculate_audio_length_ms falls back to PCM math.
|
||||
expected_length = (4 / (24_000 * 2)) * 1000
|
||||
assert state.audio_length_ms == pytest.approx(expected_length, rel=0, abs=1e-6)
|
||||
assert tracker.get_last_audio_item() == ("item_1", 0)
|
||||
|
||||
def test_audio_state_accumulation_across_deltas(self):
|
||||
"""Test ModelAudioTracker accumulates audio length across multiple deltas."""
|
||||
|
||||
tracker = ModelAudioTracker()
|
||||
tracker.set_audio_format("pcm16")
|
||||
|
||||
# Send multiple deltas for same item
|
||||
tracker.on_audio_delta("item_1", 0, b"test") # 4 bytes
|
||||
tracker.on_audio_delta("item_1", 0, b"more") # 4 bytes
|
||||
|
||||
state = tracker.get_state("item_1", 0)
|
||||
assert state is not None
|
||||
# Should accumulate: 8 bytes -> 4 samples -> (4 / 24000) * 1000 ≈ 0.167ms
|
||||
expected_length = (8 / (24_000 * 2)) * 1000
|
||||
assert state.audio_length_ms == pytest.approx(expected_length, rel=0, abs=1e-6)
|
||||
|
||||
def test_default_playback_timing_uses_monotonic_clock(self, model):
|
||||
model._audio_state_tracker.set_audio_format("pcm16")
|
||||
|
||||
with patch("agents.realtime._default_tracker.time.monotonic", return_value=42.0):
|
||||
model._audio_state_tracker.on_audio_delta("item_1", 0, b"test")
|
||||
|
||||
with patch("agents.realtime.openai_realtime.time.monotonic", return_value=42.25):
|
||||
state = model._get_playback_state()
|
||||
|
||||
assert state["current_item_id"] == "item_1"
|
||||
assert state["current_item_content_index"] == 0
|
||||
assert state["elapsed_ms"] == pytest.approx(250.0)
|
||||
|
||||
def test_state_cleanup_on_interruption(self):
|
||||
"""Test both trackers properly reset state on interruption."""
|
||||
|
||||
# Test ModelAudioTracker cleanup
|
||||
model_tracker = ModelAudioTracker()
|
||||
model_tracker.set_audio_format("pcm16")
|
||||
model_tracker.on_audio_delta("item_1", 0, b"test")
|
||||
assert model_tracker.get_last_audio_item() == ("item_1", 0)
|
||||
|
||||
model_tracker.on_interrupted()
|
||||
assert model_tracker.get_last_audio_item() is None
|
||||
|
||||
# Test RealtimePlaybackTracker cleanup
|
||||
playback_tracker = RealtimePlaybackTracker()
|
||||
playback_tracker.on_play_ms("item_1", 0, 100.0)
|
||||
|
||||
state = playback_tracker.get_state()
|
||||
assert state["current_item_id"] == "item_1"
|
||||
assert state["elapsed_ms"] == 100.0
|
||||
|
||||
playback_tracker.on_interrupted()
|
||||
state = playback_tracker.get_state()
|
||||
assert state["current_item_id"] is None
|
||||
assert state["elapsed_ms"] is None
|
||||
|
||||
def test_audio_length_calculation_with_different_formats(self):
|
||||
"""Test calculate_audio_length_ms handles g711 and PCM formats correctly."""
|
||||
from agents.realtime._util import calculate_audio_length_ms
|
||||
|
||||
# Test g711 format (8kHz)
|
||||
g711_bytes = b"12345678" # 8 bytes
|
||||
g711_length = calculate_audio_length_ms("g711_ulaw", g711_bytes)
|
||||
assert g711_length == 1 # (8 / 8000) * 1000
|
||||
|
||||
# Test PCM format (24kHz, default)
|
||||
pcm_bytes = b"test" # 4 bytes
|
||||
pcm_length = calculate_audio_length_ms("pcm16", pcm_bytes)
|
||||
expected_pcm = (len(pcm_bytes) / (24_000 * 2)) * 1000
|
||||
assert pcm_length == pytest.approx(expected_pcm, rel=0, abs=1e-6)
|
||||
|
||||
# Test None format (defaults to PCM)
|
||||
none_length = calculate_audio_length_ms(None, pcm_bytes)
|
||||
assert none_length == pytest.approx(expected_pcm, rel=0, abs=1e-6)
|
||||
@@ -0,0 +1,23 @@
|
||||
from agents.realtime.model import RealtimePlaybackTracker
|
||||
|
||||
|
||||
def test_playback_tracker_on_play_bytes_and_state():
|
||||
tr = RealtimePlaybackTracker()
|
||||
tr.set_audio_format("pcm16") # PCM path
|
||||
|
||||
# 48k bytes -> (48000 / (24000 * 2)) * 1000 = 1_000ms
|
||||
tr.on_play_bytes("item1", 0, b"x" * 48000)
|
||||
st = tr.get_state()
|
||||
assert st["current_item_id"] == "item1"
|
||||
assert st["elapsed_ms"] and abs(st["elapsed_ms"] - 1_000.0) < 1e-6
|
||||
|
||||
# Subsequent play on same item accumulates
|
||||
tr.on_play_ms("item1", 0, 500.0)
|
||||
st2 = tr.get_state()
|
||||
assert st2["elapsed_ms"] and abs(st2["elapsed_ms"] - 1_500.0) < 1e-6
|
||||
|
||||
# Interruption clears state
|
||||
tr.on_interrupted()
|
||||
st3 = tr.get_state()
|
||||
assert st3["current_item_id"] is None
|
||||
assert st3["elapsed_ms"] is None
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user