openai--openai-agents-python/tests/mcp/test_runner_calls_mcp.py

import json
from typing import Any

import pytest
from pydantic import BaseModel

from agents import (
    Agent,
    FunctionTool,
    ModelBehaviorError,
    RunContextWrapper,
    Runner,
    UserError,
    default_tool_error_function,
    handoff,
)
from agents.exceptions import AgentsException

from ..fake_model import FakeModel
from ..test_responses import get_function_tool_call, get_text_message
from .helpers import FakeMCPServer


@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_calls_mcp_tool(streaming: bool):
    """Test that the runner calls an MCP tool when the model produces a tool call."""
    server = FakeMCPServer()
    server.add_tool("test_tool_1", {})
    server.add_tool("test_tool_2", {})
    server.add_tool("test_tool_3", {})
    model = FakeModel()
    agent = Agent(
        name="test",
        model=model,
        mcp_servers=[server],
    )

    model.add_multiple_turn_outputs(
        [
            # First turn: a message and tool call
            [get_text_message("a_message"), get_function_tool_call("test_tool_2", "")],
            # Second turn: text message
            [get_text_message("done")],
        ]
    )

    if streaming:
        result = Runner.run_streamed(agent, input="user_message")
        async for _ in result.stream_events():
            pass
    else:
        await Runner.run(agent, input="user_message")

    assert server.tool_calls == ["test_tool_2"]


@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_asserts_when_mcp_tool_not_found(streaming: bool):
    """Test that the runner asserts when an MCP tool is not found."""
    server = FakeMCPServer()
    server.add_tool("test_tool_1", {})
    server.add_tool("test_tool_2", {})
    server.add_tool("test_tool_3", {})
    model = FakeModel()
    agent = Agent(
        name="test",
        model=model,
        mcp_servers=[server],
    )

    model.add_multiple_turn_outputs(
        [
            # First turn: a message and tool call
            [get_text_message("a_message"), get_function_tool_call("test_tool_doesnt_exist", "")],
            # Second turn: text message
            [get_text_message("done")],
        ]
    )

    with pytest.raises(ModelBehaviorError):
        if streaming:
            result = Runner.run_streamed(agent, input="user_message")
            async for _ in result.stream_events():
                pass
        else:
            await Runner.run(agent, input="user_message")


@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_works_with_multiple_mcp_servers(streaming: bool):
    """Test that the runner works with multiple MCP servers."""
    server1 = FakeMCPServer()
    server1.add_tool("test_tool_1", {})

    server2 = FakeMCPServer()
    server2.add_tool("test_tool_2", {})
    server2.add_tool("test_tool_3", {})

    model = FakeModel()
    agent = Agent(
        name="test",
        model=model,
        mcp_servers=[server1, server2],
    )

    model.add_multiple_turn_outputs(
        [
            # First turn: a message and tool call
            [get_text_message("a_message"), get_function_tool_call("test_tool_2", "")],
            # Second turn: text message
            [get_text_message("done")],
        ]
    )

    if streaming:
        result = Runner.run_streamed(agent, input="user_message")
        async for _ in result.stream_events():
            pass
    else:
        await Runner.run(agent, input="user_message")

    assert server1.tool_calls == []
    assert server2.tool_calls == ["test_tool_2"]


@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_errors_when_mcp_tools_clash(streaming: bool):
    """Test that the runner errors when multiple servers have the same tool name."""
    server1 = FakeMCPServer()
    server1.add_tool("test_tool_1", {})
    server1.add_tool("test_tool_2", {})

    server2 = FakeMCPServer()
    server2.add_tool("test_tool_2", {})
    server2.add_tool("test_tool_3", {})

    model = FakeModel()
    agent = Agent(
        name="test",
        model=model,
        mcp_servers=[server1, server2],
    )

    model.add_multiple_turn_outputs(
        [
            # First turn: a message and tool call
            [get_text_message("a_message"), get_function_tool_call("test_tool_3", "")],
            # Second turn: text message
            [get_text_message("done")],
        ]
    )

    with pytest.raises(UserError):
        if streaming:
            result = Runner.run_streamed(agent, input="user_message")
            async for _ in result.stream_events():
                pass
        else:
            await Runner.run(agent, input="user_message")


@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_can_call_server_prefixed_mcp_tool_names(streaming: bool):
    server1 = FakeMCPServer(server_name="docs")
    server1.add_tool("search", {})

    server2 = FakeMCPServer(server_name="calendar")
    server2.add_tool("search", {})

    model = FakeModel()
    agent = Agent(
        name="test",
        model=model,
        mcp_servers=[server1, server2],
        mcp_config={"include_server_in_tool_names": True},
    )

    model.add_multiple_turn_outputs(
        [
            [get_text_message("a_message"), get_function_tool_call("mcp_calendar__search", "")],
            [get_text_message("done")],
        ]
    )

    if streaming:
        result = Runner.run_streamed(agent, input="user_message")
        async for _ in result.stream_events():
            pass
    else:
        await Runner.run(agent, input="user_message")

    assert server1.tool_calls == []
    assert server2.tool_calls == ["search"]


@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_agent_tools(streaming: bool):
    server1 = FakeMCPServer(server_name="docs")
    server1.add_tool("search", {})

    server2 = FakeMCPServer(server_name="calendar")
    server2.add_tool("search", {})

    local_tool_calls: list[str] = []

    async def invoke_local_tool(context: Any, input_json: str) -> str:
        local_tool_calls.append(input_json)
        return "local"

    local_tool = FunctionTool(
        name="mcp_calendar__search",
        description="Local tool that intentionally collides with the natural MCP prefix.",
        params_json_schema={"type": "object", "properties": {}, "additionalProperties": False},
        on_invoke_tool=invoke_local_tool,
    )

    model = FakeModel()
    agent = Agent(
        name="test",
        model=model,
        tools=[local_tool],
        mcp_servers=[server1, server2],
        mcp_config={"include_server_in_tool_names": True},
    )

    mcp_tools = await agent.get_mcp_tools(RunContextWrapper(context=None))
    calendar_search_tool_name = next(
        tool.name
        for tool in mcp_tools
        if getattr(getattr(tool, "_tool_origin", None), "mcp_server_name", None) == "calendar"
    )
    assert calendar_search_tool_name != "mcp_calendar__search"
    assert calendar_search_tool_name.startswith("mcp_calendar__search_")

    model.add_multiple_turn_outputs(
        [
            [get_text_message("a_message"), get_function_tool_call(calendar_search_tool_name, "")],
            [get_text_message("done")],
        ]
    )

    if streaming:
        result = Runner.run_streamed(agent, input="user_message")
        async for _ in result.stream_events():
            pass
    else:
        await Runner.run(agent, input="user_message")

    assert local_tool_calls == []
    assert server1.tool_calls == []
    assert server2.tool_calls == ["search"]


@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_prefixed_mcp_tool_names_do_not_collide_with_handoffs(streaming: bool):
    server = FakeMCPServer(server_name="calendar")
    server.add_tool("search", {})

    target_model = FakeModel()
    target_agent = Agent(name="calendar_agent", model=target_model)
    target_model.add_multiple_turn_outputs([[get_text_message("handoff target")]])

    model = FakeModel()
    agent = Agent(
        name="test",
        model=model,
        handoffs=[handoff(target_agent, tool_name_override="mcp_calendar__search")],
        mcp_servers=[server],
        mcp_config={"include_server_in_tool_names": True},
    )

    mcp_tools = await agent.get_mcp_tools(RunContextWrapper(context=None))
    assert len(mcp_tools) == 1
    calendar_search_tool_name = mcp_tools[0].name
    assert calendar_search_tool_name != "mcp_calendar__search"
    assert calendar_search_tool_name.startswith("mcp_calendar__search_")

    model.add_multiple_turn_outputs(
        [
            [get_text_message("a_message"), get_function_tool_call(calendar_search_tool_name, "")],
            [get_text_message("done")],
        ]
    )

    if streaming:
        result = Runner.run_streamed(agent, input="user_message")
        async for _ in result.stream_events():
            pass
    else:
        await Runner.run(agent, input="user_message")

    assert server.tool_calls == ["search"]
    assert target_model.first_turn_args is None


class Foo(BaseModel):
    bar: str
    baz: int


@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_calls_mcp_tool_with_args(streaming: bool):
    """Test that the runner calls an MCP tool when the model produces a tool call."""
    server = FakeMCPServer()
    await server.connect()
    server.add_tool("test_tool_1", {})
    server.add_tool("test_tool_2", Foo.model_json_schema())
    server.add_tool("test_tool_3", {})
    model = FakeModel()
    agent = Agent(
        name="test",
        model=model,
        mcp_servers=[server],
    )

    json_args = json.dumps(Foo(bar="baz", baz=1).model_dump())

    model.add_multiple_turn_outputs(
        [
            # First turn: a message and tool call
            [get_text_message("a_message"), get_function_tool_call("test_tool_2", json_args)],
            # Second turn: text message
            [get_text_message("done")],
        ]
    )

    if streaming:
        result = Runner.run_streamed(agent, input="user_message")
        async for _ in result.stream_events():
            pass
    else:
        await Runner.run(agent, input="user_message")

    assert server.tool_calls == ["test_tool_2"]
    assert server.tool_results == [f"result_test_tool_2_{json_args}"]

    await server.cleanup()


class CrashingFakeMCPServer(FakeMCPServer):
    async def call_tool(
        self,
        tool_name: str,
        arguments: dict[str, object] | None,
        meta: dict[str, object] | None = None,
    ):
        raise Exception("Crash!")


@pytest.mark.asyncio
@pytest.mark.parametrize("streaming", [False, True])
async def test_runner_emits_mcp_error_tool_call_output_item(streaming: bool):
    """Runner should emit tool_call_output_item with failure output when MCP tool raises."""
    server = CrashingFakeMCPServer()
    server.add_tool("crashing_tool", {})

    model = FakeModel()
    agent = Agent(
        name="test",
        model=model,
        mcp_servers=[server],
    )

    model.add_multiple_turn_outputs(
        [
            [get_text_message("a_message"), get_function_tool_call("crashing_tool", "{}")],
            [get_text_message("done")],
        ]
    )

    if streaming:
        streamed_result = Runner.run_streamed(agent, input="user_message")
        async for _ in streamed_result.stream_events():
            pass
        tool_output_items = [
            item for item in streamed_result.new_items if item.type == "tool_call_output_item"
        ]
        assert streamed_result.final_output == "done"
    else:
        non_streamed_result = await Runner.run(agent, input="user_message")
        tool_output_items = [
            item for item in non_streamed_result.new_items if item.type == "tool_call_output_item"
        ]
        assert non_streamed_result.final_output == "done"

    assert tool_output_items, "Expected tool_call_output_item for MCP failure"
    wrapped_error = AgentsException(
        "Error invoking MCP tool crashing_tool on server 'fake_mcp_server': Crash!"
    )
    expected_error_message = default_tool_error_function(
        RunContextWrapper(context=None),
        wrapped_error,
    )
    assert tool_output_items[0].output == expected_error_message