Files
2026-07-13 12:39:17 +08:00

2448 lines
82 KiB
Python

"""Regression tests for HITL edge cases."""
from __future__ import annotations
from collections.abc import Callable
from typing import Any, Optional, cast
import pytest
from openai.types.responses import ResponseComputerToolCall, ResponseFunctionToolCall
from openai.types.responses.response_computer_tool_call import ActionScreenshot
from openai.types.responses.response_input_param import (
ComputerCallOutput,
LocalShellCallOutput,
)
from openai.types.responses.response_output_item import LocalShellCall, McpApprovalRequest
from agents import (
Agent,
ApplyPatchTool,
ComputerTool,
LocalShellTool,
Runner,
RunResult,
RunState,
ShellTool,
ToolApprovalItem,
function_tool,
tool_namespace,
)
from agents._public_agent import set_public_agent
from agents.computer import Computer, Environment
from agents.exceptions import ModelBehaviorError, UserError
from agents.items import (
MCPApprovalResponseItem,
MessageOutputItem,
ModelResponse,
RunItem,
ToolCallOutputItem,
TResponseOutputItem,
)
from agents.lifecycle import RunHooks
from agents.run import RunConfig
from agents.run_internal import run_loop
from agents.run_internal.agent_bindings import bind_execution_agent, bind_public_agent
from agents.run_internal.run_loop import (
NextStepInterruption,
NextStepRunAgain,
ProcessedResponse,
ToolRunApplyPatchCall,
ToolRunComputerAction,
ToolRunFunction,
ToolRunMCPApprovalRequest,
ToolRunShellCall,
extract_tool_call_id,
)
from agents.run_internal.tool_planning import (
_collect_runs_by_approval,
_select_function_tool_runs_for_resume,
)
from agents.run_state import RunState as RunStateClass
from agents.tool import HostedMCPTool
from agents.usage import Usage
from .fake_model import FakeModel
from .mcp.helpers import FakeMCPServer
from .test_responses import get_text_message
from .utils.hitl import (
HITL_REJECTION_MSG,
ApprovalScenario,
PendingScenario,
RecordingEditor,
approve_first_interruption,
assert_pending_resume,
assert_roundtrip_tool_name,
assert_tool_output_roundtrip,
collect_tool_outputs,
consume_stream,
make_agent,
make_apply_patch_dict,
make_context_wrapper,
make_function_tool_call,
make_mcp_approval_item,
make_model_and_agent,
make_shell_call,
make_state_with_interruptions,
queue_function_call_and_text,
require_approval,
resume_after_first_approval,
run_and_resume_after_approval,
)
def _bind_agent(agent: Agent[Any]):
public_agent = getattr(agent, "_agents_public_agent", None)
if isinstance(public_agent, Agent):
return bind_execution_agent(public_agent=public_agent, execution_agent=agent)
return bind_public_agent(agent)
async def _resolve_interrupted_turn(*, agent: Agent[Any], **kwargs: Any):
return await run_loop.resolve_interrupted_turn(
bindings=_bind_agent(agent),
**kwargs,
)
class TrackingComputer(Computer):
"""Minimal computer implementation that records method calls."""
def __init__(self) -> None:
self.calls: list[str] = []
@property
def environment(self) -> Environment:
return "mac"
@property
def dimensions(self) -> tuple[int, int]:
return (1, 1)
def screenshot(self) -> str:
self.calls.append("screenshot")
return "img"
def click(self, _x: int, _y: int, _button: str) -> None:
self.calls.append("click")
def double_click(self, _x: int, _y: int) -> None:
self.calls.append("double_click")
def scroll(self, _x: int, _y: int, _scroll_x: int, _scroll_y: int) -> None:
self.calls.append("scroll")
def type(self, _text: str) -> None:
self.calls.append("type")
def wait(self) -> None:
self.calls.append("wait")
def move(self, _x: int, _y: int) -> None:
self.calls.append("move")
def keypress(self, _keys: list[str]) -> None:
self.calls.append("keypress")
def drag(self, _path: list[tuple[int, int]]) -> None:
self.calls.append("drag")
def _shell_approval_setup() -> ApprovalScenario:
tool = ShellTool(executor=lambda request: "shell_output", needs_approval=require_approval)
shell_call = make_shell_call("call_shell_1", id_value="shell_1", commands=["echo test"])
def _assert(result: RunResult) -> None:
shell_outputs = collect_tool_outputs(result.new_items, output_type="shell_call_output")
assert shell_outputs, "Shell tool should have been executed after approval"
assert any("shell_output" in str(item.output) for item in shell_outputs)
return ApprovalScenario(
tool=tool,
raw_call=shell_call,
final_output=get_text_message("done"),
assert_result=_assert,
)
def _apply_patch_approval_setup() -> ApprovalScenario:
editor = RecordingEditor()
tool = ApplyPatchTool(editor=editor, needs_approval=require_approval)
apply_patch_call = make_apply_patch_dict("call_apply_1")
def _assert(result: RunResult) -> None:
apply_patch_outputs = collect_tool_outputs(
result.new_items, output_type="apply_patch_call_output"
)
assert apply_patch_outputs, "ApplyPatch tool should have been executed after approval"
assert editor.operations, "Editor should have been called"
return ApprovalScenario(
tool=tool,
raw_call=apply_patch_call,
final_output=get_text_message("done"),
assert_result=_assert,
)
def _shell_pending_setup() -> PendingScenario:
tool = ShellTool(executor=lambda _req: "shell_output", needs_approval=True)
raw_call = make_shell_call(
"call_shell_pending", id_value="shell_pending", commands=["echo pending"]
)
return PendingScenario(tool=tool, raw_call=raw_call)
def _apply_patch_pending_setup() -> PendingScenario:
editor = RecordingEditor()
apply_patch_tool = ApplyPatchTool(editor=editor, needs_approval=True)
def _assert_editor(_resumed: RunResult) -> None:
assert editor.operations == [], "editor should not run before approval"
return PendingScenario(
tool=apply_patch_tool,
raw_call=make_apply_patch_dict("call_apply_pending"),
assert_result=_assert_editor,
)
@pytest.mark.parametrize(
"setup_fn, user_input",
[
(_shell_approval_setup, "run shell command"),
(_apply_patch_approval_setup, "update file"),
],
ids=["shell_approved", "apply_patch_approved"],
)
@pytest.mark.asyncio
async def test_resumed_hitl_executes_approved_tools(
setup_fn: Callable[[], ApprovalScenario],
user_input: str,
) -> None:
"""Approved tools should run once the interrupted turn resumes."""
scenario = setup_fn()
model, agent = make_model_and_agent(tools=[scenario.tool])
result = await run_and_resume_after_approval(
agent,
model,
scenario.raw_call,
scenario.final_output,
user_input=user_input,
)
scenario.assert_result(result)
@pytest.mark.parametrize(
"tool_kind", ["shell", "apply_patch"], ids=["shell_auto", "apply_patch_auto"]
)
@pytest.mark.asyncio
async def test_resuming_skips_approvals_for_non_hitl_tools(tool_kind: str) -> None:
"""Auto-approved tools should not trigger new approvals when resuming a turn."""
shell_runs: list[str] = []
editor: RecordingEditor | None = None
auto_tool: ShellTool | ApplyPatchTool
if tool_kind == "shell":
def _executor(_req: Any) -> str:
shell_runs.append("run")
return "shell_output"
auto_tool = ShellTool(executor=_executor)
raw_call = make_shell_call("call_shell_auto", id_value="shell_auto", commands=["echo auto"])
output_type = "shell_call_output"
else:
editor = RecordingEditor()
auto_tool = ApplyPatchTool(editor=editor)
raw_call = make_apply_patch_dict("call_apply_auto")
output_type = "apply_patch_call_output"
async def needs_hitl() -> str:
return "approved"
approval_tool = function_tool(needs_hitl, needs_approval=require_approval)
model, agent = make_model_and_agent(tools=[auto_tool, approval_tool])
function_call = make_function_tool_call(approval_tool.name, call_id="call-func-auto")
queue_function_call_and_text(
model,
function_call,
first_turn_extra=[raw_call],
followup=[get_text_message("done")],
)
first = await Runner.run(agent, "resume approvals")
assert first.interruptions, "function tool should require approval"
resumed = await resume_after_first_approval(agent, first, always_approve=True)
assert not resumed.interruptions, "non-HITL tools should not request approval on resume"
outputs = collect_tool_outputs(resumed.new_items, output_type=output_type)
assert len(outputs) == 1, f"{tool_kind} should run exactly once without extra approvals"
if tool_kind == "shell":
assert len(shell_runs) == 1, "shell should execute automatically when resuming"
else:
assert editor is not None
assert len(editor.operations) == 1, "apply_patch should execute once when resuming"
@pytest.mark.asyncio
async def test_nested_agent_tool_resumes_after_rejection() -> None:
"""A nested agent tool should resume after a rejection to continue its own flow."""
@function_tool(needs_approval=True)
async def inner_hitl_tool() -> str:
return "ok"
inner_model = FakeModel()
inner_agent = Agent(name="Inner", model=inner_model, tools=[inner_hitl_tool])
inner_call_first = make_function_tool_call(inner_hitl_tool.name, call_id="inner-1")
inner_call_retry = make_function_tool_call(inner_hitl_tool.name, call_id="inner-2")
inner_final = get_text_message("done")
inner_model.add_multiple_turn_outputs(
[
[inner_call_first],
[inner_call_retry],
[inner_final],
]
)
agent_tool = inner_agent.as_tool(
tool_name="inner_agent_tool",
tool_description="Inner agent tool with HITL",
needs_approval=True,
)
outer_model = FakeModel()
outer_agent = Agent(name="Outer", model=outer_model, tools=[agent_tool])
outer_call = make_function_tool_call(
agent_tool.name, call_id="outer-1", arguments='{"input":"hi"}'
)
outer_model.add_multiple_turn_outputs([[outer_call]])
first = await Runner.run(outer_agent, "start")
assert first.interruptions, "agent tool should request approval first"
assert first.interruptions[0].tool_name == agent_tool.name
state_after_outer_approval = first.to_state()
state_after_outer_approval.approve(first.interruptions[0], always_approve=True)
second = await Runner.run(outer_agent, state_after_outer_approval)
assert second.interruptions, "inner tool should request approval on first run"
assert second.interruptions[0].tool_name == inner_hitl_tool.name
state_after_inner_reject = second.to_state()
state_after_inner_reject.reject(second.interruptions[0])
third = await Runner.run(outer_agent, state_after_inner_reject)
assert third.interruptions, "nested agent should resume and request new approval"
assert third.interruptions[0].tool_name == inner_hitl_tool.name
assert extract_tool_call_id(third.interruptions[0].raw_item) == "inner-2"
rejection_outputs = [
item
for item in third.new_items
if isinstance(item, ToolCallOutputItem)
and item.output == HITL_REJECTION_MSG
and extract_tool_call_id(item.raw_item) == "outer-1"
]
assert not rejection_outputs, "Nested rejection should not short-circuit the agent tool"
@pytest.mark.asyncio
async def test_nested_agent_tool_interruptions_dont_collide_on_duplicate_call_ids() -> None:
"""Nested agent tool interruptions should survive duplicate outer call IDs."""
@function_tool(needs_approval=True)
async def inner_hitl_tool() -> str:
return "ok"
inner_model = FakeModel()
inner_agent = Agent(name="Inner", model=inner_model, tools=[inner_hitl_tool])
inner_model.add_multiple_turn_outputs(
[
[make_function_tool_call(inner_hitl_tool.name, call_id="inner-1")],
[make_function_tool_call(inner_hitl_tool.name, call_id="inner-2")],
]
)
agent_tool = inner_agent.as_tool(
tool_name="inner_agent_tool",
tool_description="Inner agent tool",
needs_approval=False,
)
outer_model = FakeModel()
outer_agent = Agent(name="Outer", model=outer_model, tools=[agent_tool])
outer_model.add_multiple_turn_outputs(
[
[
make_function_tool_call(
agent_tool.name, call_id="outer-dup", arguments='{"input":"a"}'
),
make_function_tool_call(
agent_tool.name, call_id="outer-dup", arguments='{"input":"b"}'
),
]
]
)
result = await Runner.run(outer_agent, "start")
assert result.interruptions, "nested agent tool should request approvals"
nested_interruptions = [
item for item in result.interruptions if item.tool_name == inner_hitl_tool.name
]
assert len(nested_interruptions) == 2
@pytest.mark.asyncio
async def test_nested_agent_tool_does_not_inherit_parent_approvals() -> None:
"""Nested agent tools should request approval even if parent approved the same call ID."""
@function_tool(needs_approval=True, name_override="shared_tool")
async def outer_shared_tool() -> str:
return "outer"
@function_tool(needs_approval=True, name_override="shared_tool")
async def inner_shared_tool() -> str:
return "inner"
inner_model = FakeModel()
inner_agent = Agent(name="Inner", model=inner_model, tools=[inner_shared_tool])
inner_model.add_multiple_turn_outputs(
[[make_function_tool_call(inner_shared_tool.name, call_id="dup")]]
)
agent_tool = inner_agent.as_tool(
tool_name="inner_agent_tool",
tool_description="Inner agent tool",
needs_approval=False,
)
outer_model = FakeModel()
outer_agent = Agent(name="Outer", model=outer_model, tools=[outer_shared_tool, agent_tool])
outer_model.add_multiple_turn_outputs(
[
[make_function_tool_call(outer_shared_tool.name, call_id="dup")],
[
make_function_tool_call(
agent_tool.name, call_id="outer-agent", arguments='{"input":"hi"}'
)
],
]
)
first = await Runner.run(outer_agent, "start")
assert first.interruptions, "parent tool should request approval first"
approved_state = first.to_state()
approved_state.approve(first.interruptions[0])
second = await Runner.run(outer_agent, approved_state)
assert second.interruptions, "nested tool should still require approval"
assert any(item.tool_name == inner_shared_tool.name for item in second.interruptions), (
"inner tool approvals should not inherit parent approvals"
)
@pytest.mark.parametrize(
"setup_fn, output_type",
[
(_shell_pending_setup, "shell_call_output"),
(_apply_patch_pending_setup, "apply_patch_call_output"),
],
ids=["shell_pending", "apply_patch_pending"],
)
@pytest.mark.asyncio
async def test_pending_approvals_stay_pending_on_resume(
setup_fn: Callable[[], PendingScenario],
output_type: str,
) -> None:
"""Unapproved tool calls should remain pending after resuming a run."""
scenario = setup_fn()
model, _ = make_model_and_agent()
resumed = await assert_pending_resume(
scenario.tool,
model,
scenario.raw_call,
user_input="resume pending approval",
output_type=output_type,
)
if scenario.assert_result:
scenario.assert_result(resumed)
@pytest.mark.asyncio
async def test_resume_does_not_duplicate_pending_shell_approvals() -> None:
"""Resuming should not duplicate pending shell approvals."""
tool = ShellTool(executor=lambda _request: "shell_output", needs_approval=True)
model, agent = make_model_and_agent(tools=[tool])
raw_call = make_shell_call(
"call_shell_pending_dup",
id_value="shell_pending_dup",
commands=["echo pending"],
)
call_id = extract_tool_call_id(raw_call)
assert call_id, "shell call must have a call_id"
model.set_next_output([raw_call])
first = await Runner.run(agent, "run shell")
assert first.interruptions, "shell tool should require approval"
resumed = await Runner.run(agent, first.to_state())
pending_items = [
item
for item in resumed.new_items
if isinstance(item, ToolApprovalItem) and extract_tool_call_id(item.raw_item) == call_id
]
assert len(pending_items) == 1
@pytest.mark.asyncio
async def test_resuming_pending_mcp_approvals_raises_typeerror():
"""ToolApprovalItem must be hashable so pending MCP approvals can be tracked in a set."""
_, agent = make_model_and_agent(tools=[])
mcp_approval_item = make_mcp_approval_item(
agent, call_id="mcp-approval-1", include_provider_data=False
)
pending_hosted_mcp_approvals: set[ToolApprovalItem] = set()
pending_hosted_mcp_approvals.add(mcp_approval_item)
assert mcp_approval_item in pending_hosted_mcp_approvals
@pytest.mark.asyncio
async def test_route_local_shell_calls_to_remote_shell_tool():
"""Test that local shell calls are routed to the local shell tool.
When processing model output with LocalShellCall items, they should be handled by
LocalShellTool (not ShellTool), even when both tools are registered. This ensures
local shell operations use the correct executor and approval hooks.
"""
remote_shell_executed = []
local_shell_executed = []
def remote_executor(request: Any) -> str:
remote_shell_executed.append(request)
return "remote_output"
def local_executor(request: Any) -> str:
local_shell_executed.append(request)
return "local_output"
shell_tool = ShellTool(executor=remote_executor)
local_shell_tool = LocalShellTool(executor=local_executor)
model, agent = make_model_and_agent(tools=[shell_tool, local_shell_tool])
# Model emits a local_shell_call
local_shell_call = LocalShellCall(
id="local_1",
call_id="call_local_1",
type="local_shell_call",
action={"type": "exec", "command": ["echo", "test"], "env": {}}, # type: ignore[arg-type]
status="in_progress",
)
model.set_next_output([local_shell_call])
await Runner.run(agent, "run local shell")
# Local shell call should be handled by LocalShellTool, not ShellTool
# This test will fail because LocalShellCall is routed to shell_tool first
assert len(local_shell_executed) > 0, "LocalShellTool should have been executed"
assert len(remote_shell_executed) == 0, (
"ShellTool should not have been executed for local shell call"
)
@pytest.mark.asyncio
async def test_preserve_max_turns_when_resuming_from_runresult_state():
"""Test that max_turns is preserved when resuming from RunResult state.
A run configured with max_turns=20 should keep that limit after resuming from
result.to_state() without re-passing max_turns.
"""
async def test_tool() -> str:
return "tool_result"
# Create the tool with needs_approval directly
# The tool name will be "test_tool" based on the function name
tool = function_tool(test_tool, needs_approval=require_approval)
model, agent = make_model_and_agent(tools=[tool])
model.add_multiple_turn_outputs([[make_function_tool_call("test_tool", call_id="call-1")]])
result1 = await Runner.run(agent, "call test_tool", max_turns=20)
assert result1.interruptions, "should have an interruption"
state = approve_first_interruption(result1, always_approve=True)
# Provide 10 more turns (turns 2-11) to ensure we exceed the default 10 but not 20.
model.add_multiple_turn_outputs(
[
[
get_text_message(f"turn {i + 2}"), # Text message first (doesn't finish)
make_function_tool_call("test_tool", call_id=f"call-{i + 2}"),
]
for i in range(10)
]
)
result2 = await Runner.run(agent, state)
assert result2 is not None, "Run should complete successfully with max_turns=20 from state"
@pytest.mark.asyncio
async def test_current_turn_not_preserved_in_to_state():
"""Test that current turn counter is preserved when converting RunResult to RunState."""
async def test_tool() -> str:
return "tool_result"
tool = function_tool(test_tool, needs_approval=require_approval)
model, agent = make_model_and_agent(tools=[tool])
# Model emits a tool call requiring approval
model.set_next_output([make_function_tool_call("test_tool", call_id="call-1")])
# First turn with interruption
result1 = await Runner.run(agent, "call test_tool")
assert result1.interruptions, "should have interruption on turn 1"
# Convert to state - this should preserve current_turn=1
state1 = result1.to_state()
# Regression guard: to_state should keep the turn counter instead of resetting it.
assert state1._current_turn == 1, (
f"Expected current_turn=1 after 1 turn, got {state1._current_turn}. "
"to_state() should preserve the current turn counter."
)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"tool_factory, raw_call_factory, expected_tool_name, user_input",
[
(
lambda: ShellTool(executor=lambda request: "output", needs_approval=require_approval),
lambda: make_shell_call("call_shell_1", id_value="shell_1", commands=["echo test"]),
"shell",
"run shell",
),
(
lambda: ApplyPatchTool(editor=RecordingEditor(), needs_approval=require_approval),
lambda: cast(Any, make_apply_patch_dict("call_apply_1")),
"apply_patch",
"update file",
),
],
ids=["shell", "apply_patch"],
)
@pytest.mark.asyncio
async def test_deserialize_interruptions_preserve_tool_calls(
tool_factory: Callable[[], Any],
raw_call_factory: Callable[[], TResponseOutputItem],
expected_tool_name: str,
user_input: str,
) -> None:
"""Ensure deserialized interruptions preserve tool types instead of forcing function calls."""
model, agent = make_model_and_agent(tools=[tool_factory()])
await assert_roundtrip_tool_name(
agent, model, raw_call_factory(), expected_tool_name, user_input=user_input
)
@pytest.mark.parametrize("include_provider_data", [True, False])
@pytest.mark.asyncio
async def test_deserialize_interruptions_preserve_mcp_tools(
include_provider_data: bool,
) -> None:
"""Ensure MCP/hosted tool approvals survive serialization."""
model, agent = make_model_and_agent(tools=[])
mcp_approval_item = make_mcp_approval_item(
agent, call_id="mcp-approval-1", include_provider_data=include_provider_data
)
state = make_state_with_interruptions(agent, [mcp_approval_item])
state_json = state.to_json()
deserialized_state = await RunStateClass.from_json(agent, state_json)
interruptions = deserialized_state.get_interruptions()
assert len(interruptions) > 0, "Interruptions should be preserved after deserialization"
assert interruptions[0].tool_name == "test_mcp_tool", (
"MCP tool approval should be preserved, not converted to function"
)
@pytest.mark.asyncio
async def test_hosted_mcp_approval_matches_unknown_tool_key() -> None:
"""Approved hosted MCP interruptions should resume even when the tool name is missing."""
agent = make_agent()
context_wrapper = make_context_wrapper()
approval_item = make_mcp_approval_item(
agent,
call_id="mcp-123",
provider_data={"type": "mcp_approval_request"},
tool_name=None,
include_name=False,
use_call_id=False,
)
context_wrapper.approve_tool(approval_item)
class DummyMcpTool:
on_approval_request: Any = None
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[
ToolRunMCPApprovalRequest(
request_item=McpApprovalRequest(
id="mcp-123",
type="mcp_approval_request",
server_label="test_server",
arguments="{}",
name="hosted_mcp",
),
mcp_tool=cast(Any, DummyMcpTool()),
)
],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=agent,
original_input="test",
original_pre_step_items=[approval_item],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=None,
)
assert any(
isinstance(item, MCPApprovalResponseItem) and item.raw_item.get("approve") is True
for item in result.new_step_items
), "Approved hosted MCP call should emit an approval response"
@pytest.mark.asyncio
async def test_shell_call_without_call_id_raises() -> None:
"""Shell calls missing call_id should raise ModelBehaviorError instead of being skipped."""
agent = make_agent()
context_wrapper = make_context_wrapper()
shell_tool = ShellTool(executor=lambda _request: "")
shell_call = {"type": "shell_call", "action": {"commands": ["echo", "hi"]}}
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[ToolRunShellCall(tool_call=shell_call, shell_tool=shell_tool)],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
with pytest.raises(ModelBehaviorError):
await _resolve_interrupted_turn(
agent=agent,
original_input="test",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=None,
)
@pytest.mark.asyncio
async def test_preserve_persisted_item_counter_when_resuming_streamed_runs():
"""Preserve the persisted-item counter on streamed resume to avoid losing history."""
model, agent = make_model_and_agent()
# Simulate a turn interrupted mid-persistence: 5 items generated, 3 actually saved.
context_wrapper = make_context_wrapper()
state = RunState(
context=context_wrapper,
original_input="test input",
starting_agent=agent,
max_turns=10,
)
# Create 5 generated items (simulating multiple outputs before interruption)
from openai.types.responses import ResponseOutputMessage, ResponseOutputText
for i in range(5):
message_item = MessageOutputItem(
agent=agent,
raw_item=ResponseOutputMessage(
id=f"msg_{i}",
type="message",
role="assistant",
status="completed",
content=[
ResponseOutputText(
type="output_text", text=f"Message {i}", annotations=[], logprobs=[]
)
],
),
)
state._generated_items.append(message_item)
# Persisted count reflects what was already written before interruption.
state._current_turn_persisted_item_count = 3
# Add a model response so the state is valid for resumption
state._model_responses = [
ModelResponse(
output=[get_text_message("test")],
usage=Usage(),
response_id="resp_1",
)
]
# Set up model to return final output immediately (so the run completes)
model.set_next_output([get_text_message("done")])
result = Runner.run_streamed(agent, state)
assert result._current_turn_persisted_item_count == 3, (
f"Expected _current_turn_persisted_item_count=3 (the actual persisted count), "
f"but got {result._current_turn_persisted_item_count}. "
f"The counter should reflect persisted items, not len(_generated_items)="
f"{len(state._generated_items)}."
)
await consume_stream(result)
@pytest.mark.asyncio
async def test_preserve_tool_output_types_during_serialization():
"""Keep tool output types intact during RunState serialization/deserialization."""
model, agent = make_model_and_agent(tools=[])
computer_output: ComputerCallOutput = {
"type": "computer_call_output",
"call_id": "call_computer_1",
"output": {"type": "computer_screenshot", "image_url": "base64_screenshot_data"},
}
await assert_tool_output_roundtrip(
agent, computer_output, "computer_call_output", output="screenshot_data"
)
# TypedDict requires "id", but runtime objects use "call_id"; cast to align with runtime shape.
shell_output = cast(
LocalShellCallOutput,
{
"type": "local_shell_call_output",
"id": "shell_1",
"call_id": "call_shell_1",
"output": "command output",
},
)
await assert_tool_output_roundtrip(agent, shell_output, "local_shell_call_output")
@pytest.mark.asyncio
async def test_function_needs_approval_invalid_type_raises() -> None:
"""needs_approval must be bool or callable; invalid types should raise UserError."""
@function_tool(name_override="bad_tool", needs_approval=cast(Any, "always"))
def bad_tool() -> str:
return "ok"
model, agent = make_model_and_agent(tools=[bad_tool])
model.set_next_output([make_function_tool_call("bad_tool")])
with pytest.raises(UserError, match="needs_approval"):
await Runner.run(agent, "run invalid")
@pytest.mark.asyncio
async def test_resume_invalid_needs_approval_raises() -> None:
"""Resume path should surface invalid needs_approval configuration errors."""
@function_tool(name_override="bad_tool", needs_approval=cast(Any, "always"))
def bad_tool() -> str:
return "ok"
agent = make_agent(tools=[bad_tool])
context_wrapper = make_context_wrapper()
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[
ToolRunFunction(
function_tool=bad_tool,
tool_call=make_function_tool_call("bad_tool", call_id="call-1"),
)
],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
with pytest.raises(UserError, match="needs_approval"):
await _resolve_interrupted_turn(
agent=agent,
original_input="resume invalid",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=None,
)
@pytest.mark.asyncio
async def test_agent_as_tool_with_nested_approvals_propagates() -> None:
"""Agent-as-tool with needs_approval should still surface nested tool approvals."""
nested_model, spanish_agent = make_model_and_agent(name="spanish_agent")
tool_calls: list[str] = []
@function_tool(needs_approval=True)
async def get_current_timestamp() -> str:
tool_calls.append("called")
return "timestamp"
spanish_agent.tools = [get_current_timestamp]
# Spanish agent will first request timestamp, then return text.
nested_model.add_multiple_turn_outputs(
[
[make_function_tool_call("get_current_timestamp")],
[get_text_message("hola")],
]
)
# Orchestrator model will call the spanish agent tool.
orchestrator_model = FakeModel()
orchestrator = Agent(
name="orchestrator",
tools=[
spanish_agent.as_tool(
tool_name="respond_spanish",
tool_description="Respond in Spanish",
needs_approval=True,
)
],
model=orchestrator_model,
)
orchestrator_model.add_multiple_turn_outputs(
[
[
make_function_tool_call(
"respond_spanish",
call_id="spanish-call",
arguments='{"input": "hola"}',
)
],
[get_text_message("done")],
]
)
# First run should surface approval for respond_spanish.
first = await Runner.run(orchestrator, "hola")
assert first.interruptions, "Outer agent tool should require approval"
# Resuming should now surface nested approval from the Spanish agent.
state = approve_first_interruption(first, always_approve=True)
resumed = await Runner.run(orchestrator, state)
assert resumed.interruptions, "Nested agent tool approval should bubble up"
assert resumed.interruptions[0].tool_name == "get_current_timestamp"
assert isinstance(resumed.to_input_list(), list)
assert not tool_calls, "Nested tool should not execute before approval"
final_state = approve_first_interruption(resumed, always_approve=True)
final = await Runner.run(orchestrator, final_state)
assert final.final_output == "done"
assert tool_calls == ["called"]
@pytest.mark.asyncio
async def test_resume_rebuilds_function_runs_from_pending_approvals() -> None:
"""Resuming with only pending approvals should reconstruct and run function calls."""
@function_tool(needs_approval=True)
def approve_me(reason: Optional[str] = None) -> str: # noqa: UP007
return f"approved:{reason}" if reason else "approved"
model, agent = make_model_and_agent(tools=[approve_me])
approval_raw = {
"type": "function_call",
"name": approve_me.name,
"call_id": "call-rebuild-1",
"arguments": '{"reason": "ok"}',
"status": "completed",
}
approval_item = ToolApprovalItem(agent=agent, raw_item=approval_raw)
context_wrapper = make_context_wrapper()
context_wrapper.approve_tool(approval_item)
run_state = make_state_with_interruptions(agent, [approval_item])
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume approvals",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=run_state,
)
assert not isinstance(result.next_step, NextStepInterruption), (
"Approved function should run instead of requesting approval again"
)
executed_call_ids = {
extract_tool_call_id(item.raw_item)
for item in result.new_step_items
if isinstance(item, ToolCallOutputItem)
}
assert "call-rebuild-1" in executed_call_ids, "Function should be rebuilt and executed"
@pytest.mark.asyncio
async def test_resume_rebuilds_deferred_function_runs_from_lookup_key_without_raw_namespace() -> (
None
):
"""Resumed approvals should use persisted lookup identity when raw namespace is missing."""
@function_tool(needs_approval=True, name_override="lookup_account")
async def visible_lookup_account(customer_id: str) -> str:
return f"visible:{customer_id}"
@function_tool(
needs_approval=True,
name_override="lookup_account",
defer_loading=True,
)
async def deferred_lookup_account(customer_id: str) -> str:
return f"deferred:{customer_id}"
_model, agent = make_model_and_agent(tools=[visible_lookup_account, deferred_lookup_account])
approval_item = ToolApprovalItem(
agent=agent,
raw_item={
"type": "function_call",
"name": "lookup_account",
"call_id": "call-deferred-rebuild",
"arguments": '{"customer_id":"customer_1"}',
"status": "completed",
},
tool_name="lookup_account",
tool_namespace="lookup_account",
tool_lookup_key=("deferred_top_level", "lookup_account"),
)
context_wrapper = make_context_wrapper()
context_wrapper.approve_tool(approval_item)
run_state = make_state_with_interruptions(agent, [approval_item])
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume approvals",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=run_state,
)
assert not isinstance(result.next_step, NextStepInterruption)
deferred_outputs = [
item.output
for item in result.new_step_items
if isinstance(item, ToolCallOutputItem) and item.output == "deferred:customer_1"
]
assert deferred_outputs == ["deferred:customer_1"]
@pytest.mark.asyncio
async def test_resume_does_not_rebuild_approved_calls_for_same_named_sibling_agent() -> None:
"""Approved interruptions should match the current public agent, not any same-named sibling."""
first_calls: list[str] = []
second_calls: list[str] = []
@function_tool(needs_approval=True, name_override="approval_tool")
async def first_approval_tool() -> str:
first_calls.append("first")
return "first"
@function_tool(needs_approval=True, name_override="approval_tool")
async def second_approval_tool() -> str:
second_calls.append("second")
return "second"
first = Agent(name="sandbox", tools=[first_approval_tool])
second = Agent(name="sandbox", tools=[second_approval_tool])
first.handoffs = [second]
second.handoffs = [first]
approval_item = ToolApprovalItem(
agent=second,
raw_item=make_function_tool_call(
name="approval_tool",
call_id="call-sibling-approval",
arguments="{}",
),
tool_name="approval_tool",
)
context_wrapper = make_context_wrapper()
context_wrapper.approve_tool(approval_item)
run_state = make_state_with_interruptions(first, [approval_item])
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
execution_agent = set_public_agent(first.clone(), first)
result = await _resolve_interrupted_turn(
agent=execution_agent,
original_input="resume approvals",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=run_state,
)
assert first_calls == []
assert second_calls == []
assert not any(isinstance(item, ToolCallOutputItem) for item in result.new_step_items)
@pytest.mark.asyncio
async def test_resume_honors_permanent_namespaced_function_approval_with_new_call_id() -> None:
@function_tool(needs_approval=True, name_override="lookup_account")
async def lookup_account(customer_id: str) -> str:
return customer_id
namespaced_tool = tool_namespace(
name="billing",
description="Billing tools",
tools=[lookup_account],
)[0]
context_wrapper = make_context_wrapper()
approved_item = ToolApprovalItem(
agent=Agent(name="billing-agent"),
raw_item=make_function_tool_call(
"lookup_account",
call_id="approved-call",
arguments='{"customer_id":"customer_1"}',
namespace="billing",
),
)
context_wrapper.approve_tool(approved_item, always_approve=True)
resumed_run = ToolRunFunction(
tool_call=make_function_tool_call(
"lookup_account",
call_id="resumed-call",
arguments='{"customer_id":"customer_2"}',
namespace="billing",
),
function_tool=namespaced_tool,
)
pending: list[ToolApprovalItem] = []
rejections: list[str | None] = []
async def _needs_approval_checker(_run: ToolRunFunction) -> bool:
return True
async def _record_rejection(
call_id: str | None,
_tool_call: ResponseFunctionToolCall,
_tool: Any,
) -> None:
rejections.append(call_id)
selected = await _select_function_tool_runs_for_resume(
[resumed_run],
approval_items_by_call_id={},
context_wrapper=context_wrapper,
needs_approval_checker=_needs_approval_checker,
output_exists_checker=lambda _run: False,
record_rejection=_record_rejection,
pending_interruption_adder=pending.append,
pending_item_builder=lambda run: ToolApprovalItem(
agent=Agent(name="billing-agent"),
raw_item=run.tool_call,
tool_name=run.function_tool.name,
tool_namespace="billing",
),
)
assert selected == [resumed_run]
assert pending == []
assert rejections == []
@pytest.mark.asyncio
async def test_resume_skips_needs_approval_checker_when_status_resolved() -> None:
"""Resolved approve/reject decisions must short-circuit needs_approval_checker.
A user-supplied checker may have side effects (telemetry, network, exceptions).
When the approval status is already True or False, we must not invoke it.
"""
@function_tool(needs_approval=True)
async def approve_me(value: str) -> str:
return value
approved_call = make_function_tool_call(
approve_me.name, call_id="approved-call", arguments='{"value":"a"}'
)
rejected_call = make_function_tool_call(
approve_me.name, call_id="rejected-call", arguments='{"value":"b"}'
)
agent = Agent(name="agent")
context_wrapper = make_context_wrapper()
context_wrapper.approve_tool(ToolApprovalItem(agent=agent, raw_item=approved_call))
context_wrapper.reject_tool(ToolApprovalItem(agent=agent, raw_item=rejected_call))
runs = [
ToolRunFunction(tool_call=approved_call, function_tool=approve_me),
ToolRunFunction(tool_call=rejected_call, function_tool=approve_me),
]
checker_calls: list[str] = []
async def _needs_approval_checker(run: ToolRunFunction) -> bool:
checker_calls.append(run.tool_call.call_id)
raise AssertionError("checker must not run for resolved approvals")
rejections: list[str | None] = []
async def _record_rejection(
call_id: str | None,
_tool_call: ResponseFunctionToolCall,
_tool: Any,
) -> None:
rejections.append(call_id)
selected = await _select_function_tool_runs_for_resume(
runs,
approval_items_by_call_id={},
context_wrapper=context_wrapper,
needs_approval_checker=_needs_approval_checker,
output_exists_checker=lambda _run: False,
record_rejection=_record_rejection,
pending_interruption_adder=lambda _item: None,
pending_item_builder=lambda run: ToolApprovalItem(agent=agent, raw_item=run.tool_call),
)
assert checker_calls == []
assert [run.tool_call.call_id for run in selected] == ["approved-call"]
assert rejections == ["rejected-call"]
@pytest.mark.asyncio
async def test_collect_runs_by_approval_skips_checker_when_status_resolved() -> None:
"""Approved/rejected shell calls must not invoke needs_approval_checker.
Mirrors #3229 for non-function tools: when the approval status is already
True or False, a user-supplied checker (which may have side effects, hit
the network, or raise) must be short-circuited.
"""
shell_tool = ShellTool(executor=lambda _req: "ok", needs_approval=True)
approved_call = make_shell_call("approved-shell")
rejected_call = make_shell_call("rejected-shell")
agent = Agent(name="agent")
context_wrapper = make_context_wrapper()
context_wrapper.approve_tool(
ToolApprovalItem(
agent=agent,
raw_item=cast(dict[str, Any], approved_call),
tool_name=shell_tool.name,
)
)
context_wrapper.reject_tool(
ToolApprovalItem(
agent=agent,
raw_item=cast(dict[str, Any], rejected_call),
tool_name=shell_tool.name,
)
)
runs = [
ToolRunShellCall(tool_call=approved_call, shell_tool=shell_tool),
ToolRunShellCall(tool_call=rejected_call, shell_tool=shell_tool),
]
checker_calls: list[str] = []
async def _needs_approval(run: ToolRunShellCall) -> bool:
checker_calls.append(run.tool_call["call_id"])
raise AssertionError("checker must not run for resolved approvals")
async def _build_rejection(run: ToolRunShellCall, call_id: str) -> RunItem:
return ToolCallOutputItem(
output="rejected",
raw_item={"type": "function_call_output", "call_id": call_id, "output": "rejected"},
agent=agent,
)
approved, rejections = await _collect_runs_by_approval(
runs,
call_id_extractor=lambda run: run.tool_call["call_id"],
tool_name_resolver=lambda run: run.shell_tool.name,
rejection_builder=_build_rejection,
context_wrapper=context_wrapper,
approval_items_by_call_id={},
agent=agent,
pending_interruption_adder=lambda _item: None,
needs_approval_checker=_needs_approval,
output_exists_checker=lambda _call_id: False,
)
assert checker_calls == []
assert approved == [runs[0]]
assert len(rejections) == 1
@pytest.mark.asyncio
async def test_resume_rebuilds_function_runs_from_object_approvals() -> None:
"""Rebuild should handle ResponseFunctionToolCall approval items."""
@function_tool(needs_approval=True)
def approve_me(reason: Optional[str] = None) -> str: # noqa: UP007
return f"approved:{reason}" if reason else "approved"
model, agent = make_model_and_agent(tools=[approve_me])
tool_call = make_function_tool_call(
approve_me.name,
call_id="call-rebuild-obj",
arguments='{"reason": "ok"}',
)
assert isinstance(tool_call, ResponseFunctionToolCall)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call)
context_wrapper = make_context_wrapper()
context_wrapper.approve_tool(approval_item)
run_state = make_state_with_interruptions(agent, [approval_item])
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume approvals",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=run_state,
)
assert not isinstance(result.next_step, NextStepInterruption)
executed_call_ids = {
extract_tool_call_id(item.raw_item)
for item in result.new_step_items
if isinstance(item, ToolCallOutputItem)
}
assert "call-rebuild-obj" in executed_call_ids, (
"Function should be rebuilt from ResponseFunctionToolCall approval"
)
@pytest.mark.asyncio
async def test_resume_rebuilds_local_mcp_function_runs_from_approvals() -> None:
"""Rebuild should resolve approved MCP-backed function tools from agent.mcp_servers."""
server = FakeMCPServer(require_approval="always")
server.add_tool("add", {"type": "object", "properties": {}})
agent = Agent(name="TestAgent", mcp_servers=[server])
tool_call = make_function_tool_call(
"add",
call_id="call-mcp-rebuild",
arguments='{"value": 1}',
)
approval_item = ToolApprovalItem(agent=agent, raw_item=tool_call, tool_name="add")
context_wrapper = make_context_wrapper()
context_wrapper.approve_tool(approval_item)
run_state = make_state_with_interruptions(agent, [approval_item])
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume approvals",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=run_state,
)
assert not isinstance(result.next_step, NextStepInterruption)
assert server.tool_calls == ["add"]
executed_call_ids = {
extract_tool_call_id(item.raw_item)
for item in result.new_step_items
if isinstance(item, ToolCallOutputItem)
}
assert "call-mcp-rebuild" in executed_call_ids, (
"Approved local MCP tool should be rebuilt and executed from pending approvals"
)
@pytest.mark.asyncio
async def test_resume_rebuild_rejections_use_deferred_tool_display_name() -> None:
"""Resume-time rejection formatting should collapse synthetic deferred namespaces."""
async def get_weather() -> str:
return "sunny"
_model, agent = make_model_and_agent(
tools=[function_tool(get_weather, name_override="get_weather", defer_loading=True)]
)
context_wrapper = make_context_wrapper()
rejected_call = make_function_tool_call(
"get_weather",
call_id="call-deferred-reject",
namespace="get_weather",
)
assert isinstance(rejected_call, ResponseFunctionToolCall)
rejected_item = ToolApprovalItem(
agent=agent,
raw_item=rejected_call,
tool_name="get_weather",
tool_namespace="get_weather",
)
context_wrapper.reject_tool(rejected_item)
run_state = make_state_with_interruptions(agent, [rejected_item])
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume approvals",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(
tool_error_formatter=lambda args: (
f"resume-level {args.tool_name} denied ({args.call_id})"
)
),
run_state=run_state,
)
rejection_outputs = [
item.output for item in result.new_step_items if isinstance(item, ToolCallOutputItem)
]
assert rejection_outputs == ["resume-level get_weather denied (call-deferred-reject)"]
@pytest.mark.asyncio
async def test_rebuild_function_runs_handles_object_pending_and_rejections() -> None:
"""Rebuild should surface pending approvals and emit rejections for object approvals."""
@function_tool(needs_approval=True)
def reject_me(text: str = "nope") -> str:
return text
@function_tool(needs_approval=True)
def pending_me(text: str = "wait") -> str:
return text
_model, agent = make_model_and_agent(tools=[reject_me, pending_me])
context_wrapper = make_context_wrapper()
rejected_call = make_function_tool_call(reject_me.name, call_id="obj-reject")
pending_call = make_function_tool_call(pending_me.name, call_id="obj-pending")
assert isinstance(rejected_call, ResponseFunctionToolCall)
assert isinstance(pending_call, ResponseFunctionToolCall)
rejected_item = ToolApprovalItem(agent=agent, raw_item=rejected_call)
pending_item = ToolApprovalItem(agent=agent, raw_item=pending_call)
context_wrapper.reject_tool(rejected_item)
run_state = make_state_with_interruptions(agent, [rejected_item, pending_item])
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume approvals",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=run_state,
)
assert isinstance(result.next_step, NextStepInterruption)
assert pending_item in result.next_step.interruptions
rejection_outputs = [
item
for item in result.new_step_items
if isinstance(item, ToolCallOutputItem) and item.output == HITL_REJECTION_MSG
]
assert rejection_outputs, "Rejected function call should emit rejection output"
@pytest.mark.asyncio
async def test_resume_function_rejection_outputs_use_public_agent() -> None:
@function_tool(needs_approval=True)
def reject_me(text: str = "nope") -> str:
return text
_model, public_agent = make_model_and_agent(tools=[reject_me])
execution_agent = public_agent.clone()
set_public_agent(execution_agent, public_agent)
context_wrapper = make_context_wrapper()
rejected_call = make_function_tool_call(reject_me.name, call_id="obj-reject-public")
assert isinstance(rejected_call, ResponseFunctionToolCall)
rejected_item = ToolApprovalItem(agent=public_agent, raw_item=rejected_call)
context_wrapper.reject_tool(rejected_item)
run_state = make_state_with_interruptions(public_agent, [rejected_item])
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=execution_agent,
original_input="resume approvals",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=run_state,
)
rejection_outputs = [
item
for item in result.new_step_items
if isinstance(item, ToolCallOutputItem) and item.output == HITL_REJECTION_MSG
]
assert rejection_outputs
assert all(item.agent is public_agent for item in rejection_outputs)
@pytest.mark.parametrize("tool_kind", ["shell", "apply_patch"])
@pytest.mark.asyncio
async def test_resume_non_function_rejection_outputs_use_public_agent(
tool_kind: str,
) -> None:
context_wrapper = make_context_wrapper()
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
if tool_kind == "shell":
shell_tool = ShellTool(executor=lambda _req: "should_not_run", needs_approval=True)
_model, public_agent = make_model_and_agent(tools=[shell_tool])
raw_item = cast(
dict[str, Any],
make_shell_call(
"call_reject_shell_public",
id_value="shell_reject_public",
commands=["echo test"],
status="in_progress",
),
)
processed_response.shell_calls = [
ToolRunShellCall(tool_call=raw_item, shell_tool=shell_tool)
]
tool_name = shell_tool.name
else:
apply_patch_tool = ApplyPatchTool(editor=RecordingEditor(), needs_approval=True)
_model, public_agent = make_model_and_agent(tools=[apply_patch_tool])
raw_item = cast(Any, make_apply_patch_dict("call_apply_reject_public"))
processed_response.apply_patch_calls = [
ToolRunApplyPatchCall(tool_call=raw_item, apply_patch_tool=apply_patch_tool)
]
tool_name = apply_patch_tool.name
execution_agent = public_agent.clone()
set_public_agent(execution_agent, public_agent)
approval_item = ToolApprovalItem(agent=public_agent, raw_item=raw_item, tool_name=tool_name)
context_wrapper.reject_tool(approval_item)
result = await _resolve_interrupted_turn(
agent=execution_agent,
original_input="resume rejection",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=make_state_with_interruptions(public_agent, [approval_item]),
)
rejection_outputs = [
item
for item in result.new_step_items
if isinstance(item, ToolCallOutputItem) and item.output == HITL_REJECTION_MSG
]
assert rejection_outputs
assert all(item.agent is public_agent for item in rejection_outputs)
@pytest.mark.asyncio
async def test_resume_keeps_unmatched_pending_approvals_with_function_runs() -> None:
"""Pending approvals should persist even when resume has other function runs."""
@function_tool
def outer_tool() -> str:
return "outer"
@function_tool(needs_approval=True)
def inner_tool() -> str:
return "inner"
_model, agent = make_model_and_agent(tools=[outer_tool, inner_tool])
context_wrapper = make_context_wrapper()
pending_call = make_function_tool_call(inner_tool.name, call_id="call-inner")
assert isinstance(pending_call, ResponseFunctionToolCall)
pending_item = ToolApprovalItem(agent=agent, raw_item=pending_call)
run_state = make_state_with_interruptions(agent, [pending_item])
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[
ToolRunFunction(
tool_call=make_function_tool_call(outer_tool.name, call_id="call-outer"),
function_tool=outer_tool,
)
],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume approvals",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=run_state,
)
assert isinstance(result.next_step, NextStepInterruption)
assert pending_item in result.next_step.interruptions
@pytest.mark.asyncio
async def test_resume_executes_non_hitl_function_calls_without_output() -> None:
"""Non-HITL function calls should run on resume when no output exists."""
@function_tool
def already_ran() -> str:
return "done"
_, agent = make_model_and_agent(tools=[already_ran])
function_call = make_function_tool_call(already_ran.name, call_id="call-skip")
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[ToolRunFunction(tool_call=function_call, function_tool=already_ran)],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume run",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=make_context_wrapper(),
run_config=RunConfig(),
run_state=None,
)
assert isinstance(result.next_step, NextStepRunAgain)
assert any(
isinstance(item, ToolCallOutputItem) and item.output == "done"
for item in result.new_step_items
), "Non-HITL tools should run on resume when output is missing"
@pytest.mark.asyncio
async def test_resume_skips_non_hitl_function_calls_with_existing_output() -> None:
"""Non-HITL function calls with persisted outputs should not re-run on resume."""
@function_tool
def already_ran() -> str:
return "done"
model, agent = make_model_and_agent(tools=[already_ran])
function_call = make_function_tool_call(already_ran.name, call_id="call-skip")
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[ToolRunFunction(tool_call=function_call, function_tool=already_ran)],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
context_wrapper = make_context_wrapper()
context_wrapper.approve_tool(
ToolApprovalItem(agent=agent, raw_item=function_call, tool_name=already_ran.name),
always_approve=True,
)
original_pre_step_items: list[RunItem] = [
ToolCallOutputItem(
agent=agent,
raw_item={
"type": "function_call_output",
"call_id": "call-skip",
"output": "prior run",
},
output="prior run",
)
]
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume run",
original_pre_step_items=original_pre_step_items,
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=None,
)
assert isinstance(result.next_step, NextStepRunAgain)
assert not result.new_step_items, "Existing outputs should prevent re-execution on resume"
@pytest.mark.asyncio
async def test_resume_skips_shell_calls_with_existing_output() -> None:
"""Shell calls with persisted output should not execute a second time when resuming."""
shell_tool = ShellTool(executor=lambda _req: "should_not_run", needs_approval=True)
model, agent = make_model_and_agent(tools=[shell_tool])
shell_call = make_shell_call(
"call_shell_resume", id_value="shell_resume", commands=["echo done"], status="completed"
)
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[ToolRunShellCall(tool_call=shell_call, shell_tool=shell_tool)],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
original_pre_step_items = [
ToolCallOutputItem(
agent=agent,
raw_item=cast(
dict[str, Any],
{
"type": "shell_call_output",
"call_id": "call_shell_resume",
"status": "completed",
"output": "prior run",
},
),
output="prior run",
)
]
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume shell",
original_pre_step_items=cast(list[RunItem], original_pre_step_items),
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=make_context_wrapper(),
run_config=RunConfig(),
run_state=None,
)
assert isinstance(result.next_step, NextStepRunAgain)
assert not result.new_step_items, "Shell call should not run when output already exists"
@pytest.mark.asyncio
async def test_resume_keeps_approved_shell_outputs_with_pending_interruptions() -> None:
"""Approved shell outputs should be emitted even when other approvals are still pending."""
@function_tool(needs_approval=True)
def pending_tool() -> str:
return "ok"
shell_tool = ShellTool(executor=lambda _req: "shell-ok", needs_approval=True)
_model, agent = make_model_and_agent(tools=[pending_tool, shell_tool])
context_wrapper = make_context_wrapper()
function_call = make_function_tool_call(pending_tool.name, call_id="call-pending")
shell_call = make_shell_call(
"call_shell_ok", id_value="shell_ok", commands=["echo ok"], status="completed"
)
shell_approval = ToolApprovalItem(
agent=agent,
raw_item=cast(dict[str, Any], shell_call),
tool_name=shell_tool.name,
)
context_wrapper.approve_tool(shell_approval)
pending_approval = ToolApprovalItem(
agent=agent,
raw_item=function_call,
tool_name=pending_tool.name,
)
run_state = make_state_with_interruptions(agent, [pending_approval])
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[ToolRunFunction(function_tool=pending_tool, tool_call=function_call)],
computer_actions=[],
local_shell_calls=[],
shell_calls=[ToolRunShellCall(tool_call=shell_call, shell_tool=shell_tool)],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume shell with pending approval",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=run_state,
)
assert isinstance(result.next_step, NextStepInterruption)
shell_outputs = [
item
for item in result.new_step_items
if isinstance(item, ToolCallOutputItem)
and isinstance(item.raw_item, dict)
and item.raw_item.get("type") == "shell_call_output"
and item.raw_item.get("call_id") == "call_shell_ok"
]
assert shell_outputs, "Approved shell output should be included with pending interruptions"
@pytest.mark.asyncio
async def test_resume_executes_pending_computer_actions() -> None:
"""Pending computer actions should execute when resuming an interrupted turn."""
computer = TrackingComputer()
computer_tool = ComputerTool(computer=computer)
model, agent = make_model_and_agent(tools=[computer_tool])
computer_call = ResponseComputerToolCall(
type="computer_call",
id="comp_pending",
call_id="comp_pending",
status="in_progress",
action=ActionScreenshot(type="screenshot"),
pending_safety_checks=[],
)
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[
ToolRunComputerAction(tool_call=computer_call, computer_tool=computer_tool)
],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[computer_tool.name],
mcp_approval_requests=[],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume computer",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=make_context_wrapper(),
run_config=RunConfig(),
run_state=None,
)
outputs = [
item
for item in result.new_step_items
if isinstance(item, ToolCallOutputItem)
and isinstance(item.raw_item, dict)
and item.raw_item.get("type") == "computer_call_output"
]
assert outputs, "Computer action should run when resuming without prior output"
assert computer.calls, "Computer should have been invoked"
assert isinstance(result.next_step, NextStepRunAgain)
@pytest.mark.asyncio
async def test_resume_skips_computer_actions_with_existing_output() -> None:
"""Computer actions with persisted output should not execute again when resuming."""
computer = TrackingComputer()
computer_tool = ComputerTool(computer=computer)
model, agent = make_model_and_agent(tools=[computer_tool])
computer_call = ResponseComputerToolCall(
type="computer_call",
id="comp_skip",
call_id="comp_skip",
status="completed",
action=ActionScreenshot(type="screenshot"),
pending_safety_checks=[],
)
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[
ToolRunComputerAction(tool_call=computer_call, computer_tool=computer_tool)
],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[computer_tool.name],
mcp_approval_requests=[],
interruptions=[],
)
original_pre_step_items = [
ToolCallOutputItem(
agent=agent,
raw_item={
"type": "computer_call_output",
"call_id": "comp_skip",
"output": {"type": "computer_screenshot", "image_url": "data:image/png;base64,ok"},
},
output="image_url",
)
]
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume computer existing",
original_pre_step_items=cast(list[RunItem], original_pre_step_items),
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=make_context_wrapper(),
run_config=RunConfig(),
run_state=None,
)
assert not computer.calls, "Computer action should not run when output already exists"
assert not result.new_step_items, "No new items should be emitted when output exists"
assert isinstance(result.next_step, NextStepRunAgain)
@pytest.mark.asyncio
async def test_rebuild_function_runs_handles_pending_and_rejections() -> None:
"""Rebuilt function runs should surface pending approvals and emit rejections."""
@function_tool(needs_approval=True)
def reject_me(text: str = "nope") -> str:
return text
@function_tool(needs_approval=True)
def pending_me(text: str = "wait") -> str:
return text
_model, agent = make_model_and_agent(tools=[reject_me, pending_me])
context_wrapper = make_context_wrapper()
rejected_raw = {
"type": "function_call",
"name": reject_me.name,
"call_id": "call-reject",
"arguments": "{}",
}
pending_raw = {
"type": "function_call",
"name": pending_me.name,
"call_id": "call-pending",
"arguments": "{}",
}
rejected_item = ToolApprovalItem(agent=agent, raw_item=rejected_raw)
pending_item = ToolApprovalItem(agent=agent, raw_item=pending_raw)
context_wrapper.reject_tool(rejected_item)
run_state = make_state_with_interruptions(agent, [rejected_item, pending_item])
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume approvals",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=run_state,
)
assert isinstance(result.next_step, NextStepInterruption)
assert pending_item in result.next_step.interruptions
rejection_outputs = [
item
for item in result.new_step_items
if isinstance(item, ToolCallOutputItem) and item.output == HITL_REJECTION_MSG
]
assert rejection_outputs, "Rejected function call should emit rejection output"
@pytest.mark.parametrize(
"raw_item, tool_name",
[
(
make_shell_call(
"call_shell_pending_rebuild",
id_value="shell_pending_rebuild",
commands=["echo pending"],
),
"shell",
),
(cast(Any, make_apply_patch_dict("call_apply_pending_rebuild")), "apply_patch"),
(
{
"type": "function_call",
"name": "missing_tool",
"call_id": "call_missing_tool",
"arguments": "{}",
},
"missing_tool",
),
],
ids=["shell", "apply_patch", "missing_function_tool"],
)
@pytest.mark.asyncio
async def test_rebuild_preserves_unmatched_pending_approvals(
raw_item: Any,
tool_name: str,
) -> None:
"""Unmatched pending approvals should remain interruptions when rebuilding."""
_model, agent = make_model_and_agent()
approval_item = ToolApprovalItem(agent=agent, raw_item=raw_item, tool_name=tool_name)
run_state = make_state_with_interruptions(agent, [approval_item])
context_wrapper = make_context_wrapper()
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume approvals",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=run_state,
)
assert isinstance(result.next_step, NextStepInterruption)
assert approval_item in result.next_step.interruptions
@pytest.mark.asyncio
async def test_rejected_shell_calls_emit_rejection_output() -> None:
"""Shell calls should produce rejection output when already denied."""
shell_tool = ShellTool(executor=lambda _req: "should_not_run", needs_approval=True)
_model, agent = make_model_and_agent(tools=[shell_tool])
context_wrapper = make_context_wrapper()
shell_call = make_shell_call(
"call_reject_shell", id_value="shell_reject", commands=["echo test"], status="in_progress"
)
approval_item = ToolApprovalItem(
agent=agent,
raw_item=cast(dict[str, Any], shell_call),
tool_name=shell_tool.name,
)
context_wrapper.reject_tool(approval_item)
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[ToolRunShellCall(tool_call=shell_call, shell_tool=shell_tool)],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume shell rejection",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=make_state_with_interruptions(agent, [approval_item]),
)
rejection_outputs: list[ToolCallOutputItem] = []
for item in result.new_step_items:
if not isinstance(item, ToolCallOutputItem):
continue
raw = item.raw_item
if not isinstance(raw, dict) or raw.get("type") != "shell_call_output":
continue
output_value = cast(list[dict[str, Any]], raw.get("output") or [])
if not output_value:
continue
first_entry = output_value[0]
if first_entry.get("stderr") == HITL_REJECTION_MSG:
rejection_outputs.append(item)
assert rejection_outputs, "Rejected shell call should yield rejection output"
assert isinstance(result.next_step, NextStepRunAgain)
@pytest.mark.asyncio
async def test_rejected_shell_calls_with_existing_output_are_not_duplicated() -> None:
"""Rejected shell calls with persisted output should not emit duplicate rejections."""
shell_tool = ShellTool(executor=lambda _req: "should_not_run", needs_approval=True)
_model, agent = make_model_and_agent(tools=[shell_tool])
context_wrapper = make_context_wrapper()
shell_call = make_shell_call(
"call_reject_shell_dup",
id_value="shell_reject_dup",
commands=["echo test"],
status="in_progress",
)
approval_item = ToolApprovalItem(
agent=agent,
raw_item=cast(dict[str, Any], shell_call),
tool_name=shell_tool.name,
)
context_wrapper.reject_tool(approval_item)
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[ToolRunShellCall(tool_call=shell_call, shell_tool=shell_tool)],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[],
interruptions=[],
)
original_pre_step_items = [
ToolCallOutputItem(
agent=agent,
raw_item=cast(
dict[str, Any],
{
"type": "shell_call_output",
"call_id": "call_reject_shell_dup",
"output": [
{
"stdout": "",
"stderr": HITL_REJECTION_MSG,
"outcome": {"type": "exit", "exit_code": 1},
}
],
},
),
output=HITL_REJECTION_MSG,
)
]
result = await _resolve_interrupted_turn(
agent=agent,
original_input="resume shell rejection existing",
original_pre_step_items=cast(list[RunItem], original_pre_step_items),
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=None,
)
duplicate_rejections = [
item
for item in result.new_step_items
if isinstance(item, ToolCallOutputItem)
and isinstance(item.raw_item, dict)
and item.raw_item.get("type") == "shell_call_output"
and HITL_REJECTION_MSG in str(item.output)
]
assert not duplicate_rejections, "No duplicate rejection outputs should be emitted"
assert isinstance(result.next_step, NextStepRunAgain)
@pytest.mark.asyncio
async def test_mcp_callback_approvals_are_processed() -> None:
"""MCP approval requests with callbacks should emit approval responses."""
agent = make_agent()
context_wrapper = make_context_wrapper()
class DummyMcpTool:
def __init__(self) -> None:
self.on_approval_request = lambda _req: {"approve": True, "reason": "ok"}
approval_request = ToolRunMCPApprovalRequest(
request_item=McpApprovalRequest(
id="mcp-callback-1",
type="mcp_approval_request",
server_label="server",
arguments="{}",
name="hosted_mcp",
),
mcp_tool=cast(HostedMCPTool, DummyMcpTool()),
)
processed_response = ProcessedResponse(
new_items=[],
handoffs=[],
functions=[],
computer_actions=[],
local_shell_calls=[],
shell_calls=[],
apply_patch_calls=[],
tools_used=[],
mcp_approval_requests=[approval_request],
interruptions=[],
)
result = await _resolve_interrupted_turn(
agent=agent,
original_input="handle mcp",
original_pre_step_items=[],
new_response=ModelResponse(output=[], usage=Usage(), response_id="resp"),
processed_response=processed_response,
hooks=RunHooks(),
context_wrapper=context_wrapper,
run_config=RunConfig(),
run_state=None,
)
assert any(
isinstance(item, MCPApprovalResponseItem) and item.raw_item.get("approve") is True
for item in result.new_step_items
), "MCP callback approvals should emit approval responses"
assert isinstance(result.next_step, NextStepRunAgain)