chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,372 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from openai.types.responses import ResponseOutputMessage
|
||||
from pydantic import BaseModel
|
||||
|
||||
from agents import (
|
||||
Agent,
|
||||
AgentHookContext,
|
||||
GuardrailFunctionOutput,
|
||||
ItemHelpers,
|
||||
MessageOutputItem,
|
||||
ModelBehaviorError,
|
||||
OutputGuardrail,
|
||||
RunContextWrapper,
|
||||
RunErrorHandlerInput,
|
||||
RunErrorHandlerResult,
|
||||
RunErrorHandlers,
|
||||
RunHooks,
|
||||
Runner,
|
||||
UserError,
|
||||
function_tool,
|
||||
)
|
||||
from agents.items import TResponseInputItem, TResponseOutputItem
|
||||
from agents.stream_events import RunItemStreamEvent
|
||||
|
||||
from .fake_model import FakeModel
|
||||
from .test_responses import get_function_tool_call, get_text_message
|
||||
from .utils.simple_session import SimpleListSession
|
||||
|
||||
|
||||
class FinalOutput(BaseModel):
|
||||
summary: str
|
||||
|
||||
|
||||
class RecordingRunHooks(RunHooks[None]):
|
||||
def __init__(self) -> None:
|
||||
self.final_outputs: list[Any] = []
|
||||
|
||||
async def on_agent_end(
|
||||
self,
|
||||
context: AgentHookContext[None],
|
||||
agent: Agent[None],
|
||||
output: Any,
|
||||
) -> None:
|
||||
self.final_outputs.append(output)
|
||||
|
||||
|
||||
def _message_texts(items: list[TResponseInputItem]) -> list[str]:
|
||||
texts: list[str] = []
|
||||
for item in items:
|
||||
if not isinstance(item, dict) or item.get("type") != "message":
|
||||
continue
|
||||
message = ResponseOutputMessage.model_validate(item)
|
||||
texts.append(ItemHelpers.extract_text(message) or "")
|
||||
return texts
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_final_output_raises_without_handler() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("not valid json")])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match="Invalid JSON"):
|
||||
await Runner.run(agent, input="user_message")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_final_output_handler_returns_validated_fallback() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("not valid json")])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
def handler(data: RunErrorHandlerInput[None]) -> FinalOutput:
|
||||
assert isinstance(data.error, ModelBehaviorError)
|
||||
assert data.run_data.raw_responses
|
||||
assert ItemHelpers.text_message_outputs(data.run_data.new_items) == "not valid json"
|
||||
return FinalOutput(summary="safe fallback")
|
||||
|
||||
result = await Runner.run(
|
||||
agent,
|
||||
input="user_message",
|
||||
error_handlers={"invalid_final_output": handler},
|
||||
)
|
||||
|
||||
assert result.final_output == FinalOutput(summary="safe fallback")
|
||||
assert [
|
||||
ItemHelpers.text_message_output(item)
|
||||
for item in result.new_items
|
||||
if isinstance(item, MessageOutputItem)
|
||||
] == ["not valid json", '{"summary":"safe fallback"}']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_final_output_handler_can_skip_fallback_history() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("not valid json")])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
result = await Runner.run(
|
||||
agent,
|
||||
input="user_message",
|
||||
error_handlers={
|
||||
"invalid_final_output": lambda _data: RunErrorHandlerResult(
|
||||
final_output=FinalOutput(summary="safe fallback"),
|
||||
include_in_history=False,
|
||||
)
|
||||
},
|
||||
)
|
||||
|
||||
assert result.final_output == FinalOutput(summary="safe fallback")
|
||||
assert ItemHelpers.text_message_outputs(result.new_items) == "not valid json"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_final_output_handler_rejects_invalid_fallback() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("not valid json")])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
with pytest.warns(UserWarning, match="Pydantic serializer warnings"):
|
||||
with pytest.raises(UserError, match="Invalid run error handler final_output"):
|
||||
await Runner.run(
|
||||
agent,
|
||||
input="user_message",
|
||||
error_handlers={"invalid_final_output": lambda _data: {"unexpected": "value"}},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_final_output_handler_can_decline_recovery() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("not valid json")])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match="Invalid JSON"):
|
||||
await Runner.run(
|
||||
agent,
|
||||
input="user_message",
|
||||
error_handlers={"invalid_final_output": lambda _data: None},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_final_output_handler_does_not_catch_other_model_behavior_errors() -> None:
|
||||
model = FakeModel(initial_output=[get_function_tool_call("missing_tool")])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
handler_called = False
|
||||
|
||||
def handler(_data: RunErrorHandlerInput[None]) -> FinalOutput:
|
||||
nonlocal handler_called
|
||||
handler_called = True
|
||||
return FinalOutput(summary="safe fallback")
|
||||
|
||||
with pytest.raises(ModelBehaviorError, match="not found"):
|
||||
await Runner.run(
|
||||
agent,
|
||||
input="user_message",
|
||||
error_handlers={"invalid_final_output": handler},
|
||||
)
|
||||
|
||||
assert not handler_called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("invalid_output", [[], [get_text_message("")]])
|
||||
async def test_empty_structured_output_handler_avoids_another_model_turn(
|
||||
invalid_output: list[TResponseOutputItem],
|
||||
) -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs([invalid_output, [get_text_message('{"summary":"unused"}')]])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
def handler(data: RunErrorHandlerInput[None]) -> FinalOutput:
|
||||
assert isinstance(data.error, ModelBehaviorError)
|
||||
assert data.error.message == (
|
||||
"Model returned no final output for the structured output type."
|
||||
)
|
||||
return FinalOutput(summary="safe fallback")
|
||||
|
||||
result = await Runner.run(
|
||||
agent,
|
||||
input="user_message",
|
||||
error_handlers={"invalid_final_output": handler},
|
||||
)
|
||||
|
||||
assert result.final_output == FinalOutput(summary="safe fallback")
|
||||
assert len(model.turn_outputs) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"error_handlers",
|
||||
[None, {"invalid_final_output": lambda _data: None}],
|
||||
)
|
||||
async def test_empty_structured_output_without_fallback_keeps_existing_next_turn_behavior(
|
||||
error_handlers: RunErrorHandlers[None] | None,
|
||||
) -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs([[], [get_text_message('{"summary":"second turn"}')]])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
result = await Runner.run(agent, input="user_message", error_handlers=error_handlers)
|
||||
|
||||
assert result.final_output == FinalOutput(summary="second turn")
|
||||
assert not model.turn_outputs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_invalid_final_output_emits_exact_fallback_item() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("not valid json")])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
session = SimpleListSession()
|
||||
|
||||
result = Runner.run_streamed(
|
||||
agent,
|
||||
input="user_message",
|
||||
session=session,
|
||||
error_handlers={"invalid_final_output": lambda _data: FinalOutput(summary="safe fallback")},
|
||||
)
|
||||
events = [event async for event in result.stream_events()]
|
||||
|
||||
assert result.final_output == FinalOutput(summary="safe fallback")
|
||||
fallback_events = [
|
||||
event
|
||||
for event in events
|
||||
if isinstance(event, RunItemStreamEvent)
|
||||
and event.name == "message_output_created"
|
||||
and isinstance(event.item, MessageOutputItem)
|
||||
and ItemHelpers.text_message_output(event.item) == '{"summary":"safe fallback"}'
|
||||
]
|
||||
assert len(fallback_events) == 1
|
||||
assert [
|
||||
ItemHelpers.text_message_output(item)
|
||||
for item in result.new_items
|
||||
if isinstance(item, MessageOutputItem)
|
||||
] == ["not valid json", '{"summary":"safe fallback"}']
|
||||
assert _message_texts(await session.get_items()) == [
|
||||
"not valid json",
|
||||
'{"summary":"safe fallback"}',
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streamed_empty_structured_output_handler_avoids_another_model_turn() -> None:
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs([[], [get_text_message('{"summary":"unused"}')]])
|
||||
agent = Agent(name="test", model=model, output_type=FinalOutput)
|
||||
|
||||
result = Runner.run_streamed(
|
||||
agent,
|
||||
input="user_message",
|
||||
error_handlers={"invalid_final_output": lambda _data: FinalOutput(summary="safe fallback")},
|
||||
)
|
||||
events = [event async for event in result.stream_events()]
|
||||
|
||||
assert result.final_output == FinalOutput(summary="safe fallback")
|
||||
assert len(model.turn_outputs) == 1
|
||||
assert any(
|
||||
isinstance(event, RunItemStreamEvent)
|
||||
and event.name == "message_output_created"
|
||||
and isinstance(event.item, MessageOutputItem)
|
||||
and ItemHelpers.text_message_output(event.item) == '{"summary":"safe fallback"}'
|
||||
for event in events
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_final_output_fallback_runs_hooks_and_output_guardrails() -> None:
|
||||
model = FakeModel(initial_output=[get_text_message("not valid json")])
|
||||
hooks = RecordingRunHooks()
|
||||
guarded_outputs: list[Any] = []
|
||||
|
||||
def record_output(
|
||||
context: RunContextWrapper[None],
|
||||
agent: Agent[Any],
|
||||
output: Any,
|
||||
) -> GuardrailFunctionOutput:
|
||||
guarded_outputs.append(output)
|
||||
return GuardrailFunctionOutput(output_info=None, tripwire_triggered=False)
|
||||
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
output_type=FinalOutput,
|
||||
output_guardrails=[OutputGuardrail(guardrail_function=record_output)],
|
||||
)
|
||||
|
||||
result = await Runner.run(
|
||||
agent,
|
||||
input="user_message",
|
||||
hooks=hooks,
|
||||
error_handlers={"invalid_final_output": lambda _data: FinalOutput(summary="safe fallback")},
|
||||
)
|
||||
|
||||
expected = FinalOutput(summary="safe fallback")
|
||||
assert result.final_output == expected
|
||||
assert hooks.final_outputs == [expected]
|
||||
assert guarded_outputs == [expected]
|
||||
assert len(result.output_guardrail_results) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("streamed", [False, True])
|
||||
async def test_invalid_final_output_fallback_does_not_retry_or_replay_tools(
|
||||
streamed: bool,
|
||||
) -> None:
|
||||
side_effects: list[str] = []
|
||||
|
||||
@function_tool
|
||||
async def record_side_effect(value: str) -> str:
|
||||
side_effects.append(value)
|
||||
return f"recorded:{value}"
|
||||
|
||||
model = FakeModel()
|
||||
model.add_multiple_turn_outputs(
|
||||
[
|
||||
[
|
||||
get_function_tool_call(
|
||||
"record_side_effect",
|
||||
json.dumps({"value": "once"}),
|
||||
call_id="first_call",
|
||||
)
|
||||
],
|
||||
[get_text_message("not valid json")],
|
||||
[
|
||||
get_function_tool_call(
|
||||
"record_side_effect",
|
||||
json.dumps({"value": "replayed"}),
|
||||
call_id="replayed_call",
|
||||
)
|
||||
],
|
||||
[get_text_message('{"summary":"unexpected retry"}')],
|
||||
]
|
||||
)
|
||||
agent = Agent(
|
||||
name="test",
|
||||
model=model,
|
||||
tools=[record_side_effect],
|
||||
output_type=FinalOutput,
|
||||
)
|
||||
error_handlers: RunErrorHandlers[None] = {
|
||||
"invalid_final_output": lambda _data: FinalOutput(summary="safe fallback")
|
||||
}
|
||||
|
||||
if streamed:
|
||||
streamed_result = Runner.run_streamed(
|
||||
agent,
|
||||
input="user_message",
|
||||
error_handlers=error_handlers,
|
||||
)
|
||||
events = [event async for event in streamed_result.stream_events()]
|
||||
final_output = streamed_result.final_output
|
||||
fallback_events = [
|
||||
event
|
||||
for event in events
|
||||
if isinstance(event, RunItemStreamEvent)
|
||||
and event.name == "message_output_created"
|
||||
and isinstance(event.item, MessageOutputItem)
|
||||
and ItemHelpers.text_message_output(event.item) == '{"summary":"safe fallback"}'
|
||||
]
|
||||
assert len(fallback_events) == 1
|
||||
else:
|
||||
result = await Runner.run(
|
||||
agent,
|
||||
input="user_message",
|
||||
error_handlers=error_handlers,
|
||||
)
|
||||
final_output = result.final_output
|
||||
|
||||
assert final_output == FinalOutput(summary="safe fallback")
|
||||
assert side_effects == ["once"]
|
||||
assert len(model.turn_outputs) == 2
|
||||
Reference in New Issue
Block a user