# SPDX-FileCopyrightText: 2022-present deepset GmbH # # SPDX-License-Identifier: Apache-2.0 import os from typing import Annotated, Any from unittest.mock import MagicMock import pytest from haystack import component from haystack.components.agents import Agent from haystack.components.generators.chat import OpenAIChatGenerator from haystack.dataclasses import ChatMessage, ToolCall from haystack.human_in_the_loop import ( AlwaysAskPolicy, BlockingConfirmationStrategy, ConfirmationHook, ConfirmationUIResult, NeverAskPolicy, SimpleConsoleUI, ) from haystack.human_in_the_loop.types import ConfirmationStrategy, ConfirmationUI from haystack.tools import Tool, Toolset, create_tool_from_function class MockUserInterface(ConfirmationUI): def __init__(self, ui_result: ConfirmationUIResult) -> None: self.ui_result = ui_result def get_user_confirmation( self, tool_name: str, tool_description: str, tool_params: dict[str, Any] ) -> ConfirmationUIResult: return self.ui_result def addition_tool(a: int, b: int) -> int: return a + b @pytest.fixture def tools() -> list[Tool]: tool = create_tool_from_function( function=addition_tool, name="addition_tool", description="A tool that adds two integers together." ) return [tool] @pytest.fixture def confirmation_strategies() -> dict[str, ConfirmationStrategy]: return { "addition_tool": BlockingConfirmationStrategy( confirmation_policy=NeverAskPolicy(), confirmation_ui=SimpleConsoleUI() ) } @pytest.fixture def confirmation_hook(confirmation_strategies) -> ConfirmationHook: return ConfirmationHook(confirmation_strategies=confirmation_strategies) class TestAgent: def test_confirmation_hook_accepted_on_before_tool(self, tools, confirmation_hook): agent = Agent(chat_generator=MockChatGenerator(), tools=tools, hooks={"before_tool": [confirmation_hook]}) assert agent.hooks["before_tool"] == [confirmation_hook] @pytest.mark.parametrize("bad_point", ["before_llm", "on_exit"]) def test_confirmation_hook_rejected_on_wrong_hook_point(self, tools, confirmation_hook, bad_point): with pytest.raises(ValueError, match="only supports"): Agent(chat_generator=MockChatGenerator(), tools=tools, hooks={bad_point: [confirmation_hook]}) def test_to_dict(self, tools, confirmation_hook, monkeypatch): monkeypatch.setenv("OPENAI_API_KEY", "test") agent = Agent(chat_generator=OpenAIChatGenerator(), tools=tools, hooks={"before_tool": [confirmation_hook]}) agent_dict = agent.to_dict() assert agent_dict == { "type": "haystack.components.agents.agent.Agent", "init_parameters": { "chat_generator": { "type": "haystack.components.generators.chat.openai.OpenAIChatGenerator", "init_parameters": { "model": "gpt-5-mini", "streaming_callback": None, "api_base_url": None, "organization": None, "generation_kwargs": {}, "api_key": {"type": "env_var", "env_vars": ["OPENAI_API_KEY"], "strict": True}, "timeout": None, "max_retries": None, "tools": None, "tools_strict": False, "http_client_kwargs": None, }, }, "tools": [ { "type": "haystack.tools.tool.Tool", "data": { "name": "addition_tool", "description": "A tool that adds two integers together.", "parameters": { "properties": {"a": {"type": "integer"}, "b": {"type": "integer"}}, "required": ["a", "b"], "type": "object", }, "function": "test_agent_hitl.addition_tool", "async_function": None, "outputs_to_string": None, "inputs_from_state": None, "outputs_to_state": None, }, } ], "system_prompt": None, "exit_conditions": ["text"], "state_schema": {}, "max_agent_steps": 100, "streaming_callback": None, "raise_on_tool_invocation_failure": False, "tool_concurrency_limit": 4, "tool_streaming_callback_passthrough": False, "required_variables": None, "user_prompt": None, "hooks": { "before_tool": [ { "type": "haystack.human_in_the_loop.hooks.ConfirmationHook", "init_parameters": { "confirmation_strategies": { "addition_tool": { "type": "haystack.human_in_the_loop.strategies.BlockingConfirmationStrategy", "init_parameters": { "confirmation_policy": { "type": "haystack.human_in_the_loop.policies.NeverAskPolicy", "init_parameters": {}, }, "confirmation_ui": { "type": "haystack.human_in_the_loop.user_interfaces.SimpleConsoleUI", "init_parameters": {}, }, "reject_template": "Tool execution for '{tool_name}' was rejected by " "the user.", "modify_template": "The parameters for tool '{tool_name}' were updated " "by the user to:\n{final_tool_params}", "user_feedback_template": "With user feedback: {feedback}", }, } } }, } ] }, }, } def test_from_dict(self, tools, confirmation_hook, monkeypatch): monkeypatch.setenv("OPENAI_API_KEY", "test") agent = Agent(chat_generator=OpenAIChatGenerator(), tools=tools, hooks={"before_tool": [confirmation_hook]}) deserialized_agent = Agent.from_dict(agent.to_dict()) assert deserialized_agent.to_dict() == agent.to_dict() assert isinstance(deserialized_agent.chat_generator, OpenAIChatGenerator) assert len(deserialized_agent.tools) == 1 assert deserialized_agent.tools[0].name == "addition_tool" assert deserialized_agent.tool_concurrency_limit == agent.tool_concurrency_limit assert deserialized_agent.tool_streaming_callback_passthrough == agent.tool_streaming_callback_passthrough hook = deserialized_agent.hooks["before_tool"][0] assert isinstance(hook, ConfirmationHook) assert isinstance(hook.confirmation_strategies["addition_tool"], BlockingConfirmationStrategy) assert isinstance(hook.confirmation_strategies["addition_tool"].confirmation_policy, NeverAskPolicy) assert isinstance(hook.confirmation_strategies["addition_tool"].confirmation_ui, SimpleConsoleUI) @pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set") @pytest.mark.integration def test_run_blocking_confirmation_strategy_modify(self, tools): confirmation_hook = ConfirmationHook( confirmation_strategies={ "addition_tool": BlockingConfirmationStrategy( confirmation_policy=AlwaysAskPolicy(), confirmation_ui=MockUserInterface( ConfirmationUIResult(action="modify", new_tool_params={"a": 2, "b": 3}) ), ) } ) agent = Agent( chat_generator=OpenAIChatGenerator(model="gpt-4o-mini"), tools=tools, hooks={"before_tool": [confirmation_hook]}, ) result = agent.run([ChatMessage.from_user("What is 2+2?")]) assert isinstance(result["last_message"], ChatMessage) assert result["last_message"].text is not None assert "5" in result["last_message"].text # Auto-populated run-metadata outputs: at least one tool call plus a final answer. assert result["step_count"] >= 2 assert result["tool_call_counts"]["addition_tool"] >= 1 assert result["token_usage"]["prompt_tokens"] > 0 assert result["token_usage"]["completion_tokens"] > 0 assert result["token_usage"]["total_tokens"] > 0 @pytest.mark.skipif(not os.environ.get("OPENAI_API_KEY"), reason="OPENAI_API_KEY not set") @pytest.mark.integration @pytest.mark.asyncio async def test_run_async_blocking_confirmation_strategy_modify(self, tools): confirmation_hook = ConfirmationHook( confirmation_strategies={ "addition_tool": BlockingConfirmationStrategy( confirmation_policy=AlwaysAskPolicy(), confirmation_ui=MockUserInterface( ConfirmationUIResult(action="modify", new_tool_params={"a": 2, "b": 3}) ), ) } ) agent = Agent( chat_generator=OpenAIChatGenerator(model="gpt-4o-mini"), tools=tools, hooks={"before_tool": [confirmation_hook]}, ) result = await agent.run_async([ChatMessage.from_user("What is 2+2?")]) assert isinstance(result["last_message"], ChatMessage) assert result["last_message"].text is not None assert "5" in result["last_message"].text # Auto-populated run-metadata outputs: at least one tool call plus a final answer. assert result["step_count"] >= 2 assert result["tool_call_counts"]["addition_tool"] >= 1 assert result["token_usage"]["prompt_tokens"] > 0 assert result["token_usage"]["completion_tokens"] > 0 assert result["token_usage"]["total_tokens"] > 0 @component class MockChatGenerator: @component.output_types(replies=list[ChatMessage]) def run(self, messages: list[ChatMessage], tools: list[Tool] | Toolset | None = None, **kwargs) -> dict[str, Any]: return {"replies": [ChatMessage.from_assistant("Hello")]} def _producer() -> dict[str, str]: return {"value": "PRODUCED"} def _consumer(value: Annotated[str, "the shared value"]) -> str: return f"consumed:{value}" # `producer` overwrites the `shared` state key; `consumer` reads it via inputs_from_state. Called together in one # step, the batched executor must run producer first so consumer sees the fresh value. producer_tool = Tool( name="producer", description="Produce a value into state.", parameters={"type": "object", "properties": {}, "required": []}, function=_producer, outputs_to_state={"shared": {"source": "value"}}, ) consumer_tool = Tool( name="consumer", description="Consume the shared value.", parameters={"type": "object", "properties": {"value": {"type": "string"}}, "required": ["value"]}, function=_consumer, inputs_from_state={"shared": "value"}, ) class TestConfirmationStrategyToolArgPrep: def test_confirmed_dependent_tool_runs_with_fresh_state(self): """ When a confirmation strategy is configured, a tool that reads State a same-step tool writes must still run with the freshly-produced value, not the stale step-start value. """ confirmation_hook = ConfirmationHook( confirmation_strategies={ "consumer": BlockingConfirmationStrategy( confirmation_policy=NeverAskPolicy(), confirmation_ui=SimpleConsoleUI() ) } ) agent = Agent( chat_generator=MockChatGenerator(), tools=[producer_tool, consumer_tool], state_schema={"shared": {"type": str}}, hooks={"before_tool": [confirmation_hook]}, ) agent.warm_up() # Step 1: the model calls producer and consumer together (consumer relies on inputs_from_state for `value`). # Step 2: a plain text reply ends the run. agent.chat_generator.run = MagicMock( side_effect=[ { "replies": [ ChatMessage.from_assistant(tool_calls=[ToolCall("producer", {}), ToolCall("consumer", {})]) ] }, {"replies": [ChatMessage.from_assistant("done")]}, ] ) # `shared` starts stale; producer overwrites it to "PRODUCED" before consumer runs. result = agent.run(messages=[ChatMessage.from_user("go")], shared="OLD") consumer_results = [ m.tool_call_result.result for m in result["messages"] if m.tool_call_result is not None and m.tool_call_result.origin.tool_name == "consumer" ] assert consumer_results == ["consumed:PRODUCED"] def _echo(x: Annotated[int, "the value to echo"]) -> dict[str, int]: return {"echoed": x} def _echo_tool(name: str) -> Tool: return Tool( name=name, description=f"Echo tool {name}.", parameters={"type": "object", "properties": {"x": {"type": "integer"}}, "required": ["x"]}, function=_echo, ) def _single_tool_hook(tool_name: str, ui_result: ConfirmationUIResult) -> ConfirmationHook: return ConfirmationHook( confirmation_strategies={ tool_name: BlockingConfirmationStrategy( confirmation_policy=AlwaysAskPolicy(), confirmation_ui=MockUserInterface(ui_result) ) } ) class TestMultipleConfirmationHooks: def test_multiple_hooks_each_targeting_a_different_tool(self): """ Three before_tool ConfirmationHooks, each owning a different tool of a three-call batch, compose correctly: a tool a hook does not own passes through unchanged, so reject/modify decisions from all three hooks land on the right calls in the final, re-read pending message. """ foo, bar, baz = _echo_tool("foo"), _echo_tool("bar"), _echo_tool("baz") hooks = { "before_tool": [ _single_tool_hook("foo", ConfirmationUIResult(action="reject")), _single_tool_hook("bar", ConfirmationUIResult(action="modify", new_tool_params={"x": 99})), _single_tool_hook("baz", ConfirmationUIResult(action="modify", new_tool_params={"x": 77})), ] } agent = Agent(chat_generator=MockChatGenerator(), tools=[foo, bar, baz], hooks=hooks) agent.warm_up() agent.chat_generator.run = MagicMock( side_effect=[ { "replies": [ ChatMessage.from_assistant( tool_calls=[ToolCall("foo", {"x": 1}), ToolCall("bar", {"x": 2}), ToolCall("baz", {"x": 3})] ) ] }, {"replies": [ChatMessage.from_assistant("done")]}, ] ) result = agent.run(messages=[ChatMessage.from_user("go")]) # The full transcript, in order: foo is rejected (its call + an error tool result appear before the # surviving batch), bar and baz are modified (each with an explanation message) and then executed. assert result["messages"] == [ ChatMessage.from_user("go"), ChatMessage.from_assistant(tool_calls=[ToolCall("foo", {"x": 1})]), ChatMessage.from_tool( tool_result="Tool execution for 'foo' was rejected by the user.", origin=ToolCall("foo", {"x": 1}), error=True, ), ChatMessage.from_user("The parameters for tool 'bar' were updated by the user to:\n{'x': 99}"), ChatMessage.from_user("The parameters for tool 'baz' were updated by the user to:\n{'x': 77}"), ChatMessage.from_assistant(tool_calls=[ToolCall("bar", {"x": 99}), ToolCall("baz", {"x": 77})]), ChatMessage.from_tool(tool_result='{"echoed": 99}', origin=ToolCall("bar", {"x": 99}), error=False), ChatMessage.from_tool(tool_result='{"echoed": 77}', origin=ToolCall("baz", {"x": 77}), error=False), ChatMessage.from_assistant("done"), ] # Only the two confirmed tools actually executed. assert result["tool_call_counts"] == {"foo": 0, "bar": 1, "baz": 1}