chore: import upstream snapshot with attribution
CodeQL / Analyze (csharp) (push) Waiting to run
CodeQL / Analyze (python) (push) Waiting to run

This commit is contained in:
wehub-resource-sync
2026-07-13 13:21:23 +08:00
commit b957a53def
5423 changed files with 863745 additions and 0 deletions
@@ -0,0 +1,128 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, MagicMock
import pytest
from autogen import ConversableAgent
from semantic_kernel.agents.autogen.autogen_conversable_agent import (
AutoGenConversableAgent,
AutoGenConversableAgentThread,
)
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions.agent_exceptions import AgentInvokeException
@pytest.fixture
def mock_conversable_agent():
agent = MagicMock(spec=ConversableAgent)
agent.name = "MockName"
agent.description = "MockDescription"
agent.system_message = "MockSystemMessage"
return agent
async def test_autogen_conversable_agent_initialization(mock_conversable_agent):
agent = AutoGenConversableAgent(mock_conversable_agent, id="mock_id")
assert agent.name == "MockName"
assert agent.description == "MockDescription"
assert agent.instructions == "MockSystemMessage"
assert agent.conversable_agent == mock_conversable_agent
async def test_autogen_conversable_agent_get_response(mock_conversable_agent):
mock_conversable_agent.a_generate_reply = AsyncMock(return_value="Mocked assistant response")
agent = AutoGenConversableAgent(mock_conversable_agent)
thread: AutoGenConversableAgentThread = None
response = await agent.get_response("Hello", thread=thread)
assert response.message.role == AuthorRole.ASSISTANT
assert response.message.content == "Mocked assistant response"
assert response.thread is not None
async def test_autogen_conversable_agent_get_response_exception(mock_conversable_agent):
mock_conversable_agent.a_generate_reply = AsyncMock(return_value=None)
agent = AutoGenConversableAgent(mock_conversable_agent)
with pytest.raises(AgentInvokeException):
await agent.get_response("Hello")
async def test_autogen_conversable_agent_invoke_with_recipient(mock_conversable_agent):
mock_conversable_agent.a_initiate_chat = AsyncMock()
mock_conversable_agent.a_initiate_chat.return_value = MagicMock(
chat_history=[
{"role": "user", "content": "Hello from user!"},
{"role": "assistant", "content": "Hello from assistant!"},
]
)
agent = AutoGenConversableAgent(mock_conversable_agent)
recipient_agent = MagicMock(spec=AutoGenConversableAgent)
recipient_agent.conversable_agent = MagicMock(spec=ConversableAgent)
messages = []
async for response in agent.invoke(recipient=recipient_agent, messages="Test message", arg1="arg1"):
messages.append(response)
mock_conversable_agent.a_initiate_chat.assert_awaited_once()
assert len(messages) == 2
assert messages[0].message.role == AuthorRole.USER
assert messages[0].message.content == "Hello from user!"
assert messages[1].message.role == AuthorRole.ASSISTANT
assert messages[1].message.content == "Hello from assistant!"
async def test_autogen_conversable_agent_invoke_without_recipient_string_reply(mock_conversable_agent):
mock_conversable_agent.a_generate_reply = AsyncMock(return_value="Mocked assistant response")
agent = AutoGenConversableAgent(mock_conversable_agent)
responses = []
async for response in agent.invoke(messages="Hello"):
responses.append(response)
mock_conversable_agent.a_generate_reply.assert_awaited_once()
assert len(responses) == 1
assert responses[0].message.role == AuthorRole.ASSISTANT
assert responses[0].message.content == "Mocked assistant response"
async def test_autogen_conversable_agent_invoke_without_recipient_dict_reply(mock_conversable_agent):
mock_conversable_agent.a_generate_reply = AsyncMock(
return_value={
"content": "Mocked assistant response",
"role": "assistant",
"name": "AssistantName",
}
)
agent = AutoGenConversableAgent(mock_conversable_agent)
responses = []
async for response in agent.invoke(messages="Hello"):
responses.append(response)
mock_conversable_agent.a_generate_reply.assert_awaited_once()
assert len(responses) == 1
assert responses[0].message.role == AuthorRole.ASSISTANT
assert responses[0].message.content == "Mocked assistant response"
assert responses[0].message.name == "AssistantName"
async def test_autogen_conversable_agent_invoke_without_recipient_unexpected_type(mock_conversable_agent):
mock_conversable_agent.a_generate_reply = AsyncMock(return_value=12345)
agent = AutoGenConversableAgent(mock_conversable_agent)
with pytest.raises(AgentInvokeException):
async for _ in agent.invoke(messages="Hello"):
pass
async def test_autogen_conversable_agent_invoke_with_invalid_recipient_type(mock_conversable_agent):
mock_conversable_agent.a_generate_reply = AsyncMock(return_value=12345)
agent = AutoGenConversableAgent(mock_conversable_agent)
recipient = MagicMock()
with pytest.raises(AgentInvokeException):
async for _ in agent.invoke(recipient=recipient, messages="Hello"):
pass
@@ -0,0 +1,28 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, MagicMock
import pytest
from azure.ai.agents.models import Agent as AzureAIAgentModel
from azure.ai.projects.aio import AIProjectClient
@pytest.fixture
def ai_project_client() -> AsyncMock:
client = AsyncMock(spec=AIProjectClient)
agents_mock = MagicMock()
client.agents = agents_mock
return client
@pytest.fixture
def ai_agent_definition() -> AsyncMock:
definition = AsyncMock(spec=AzureAIAgentModel)
definition.id = "agent123"
definition.name = "agentName"
definition.description = "desc"
definition.instructions = "test agent"
return definition
@@ -0,0 +1,464 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import MagicMock
from azure.ai.agents.models import (
MessageDelta,
MessageDeltaChunk,
MessageDeltaImageFileContent,
MessageDeltaImageFileContentObject,
MessageDeltaTextContent,
MessageDeltaTextContentObject,
MessageDeltaTextFileCitationAnnotation,
MessageDeltaTextFileCitationAnnotationObject,
MessageDeltaTextFilePathAnnotation,
MessageDeltaTextFilePathAnnotationObject,
MessageDeltaTextUrlCitationAnnotation,
MessageDeltaTextUrlCitationDetails,
MessageImageFileContent,
MessageImageFileDetails,
MessageTextContent,
MessageTextDetails,
MessageTextFileCitationAnnotation,
MessageTextFileCitationDetails,
MessageTextFilePathAnnotation,
MessageTextFilePathDetails,
MessageTextUrlCitationAnnotation,
MessageTextUrlCitationDetails,
RequiredFunctionToolCall,
RunStep,
RunStepBingCustomSearchToolCall,
RunStepBingGroundingToolCall,
RunStepDeltaFunction,
RunStepDeltaFunctionToolCall,
RunStepDeltaToolCallObject,
RunStepFunctionToolCall,
RunStepFunctionToolCallDetails,
ThreadMessage,
)
from semantic_kernel.agents.azure_ai.agent_content_generation import (
THREAD_MESSAGE_ID,
generate_annotation_content,
generate_bing_grounding_content,
generate_code_interpreter_content,
generate_function_call_content,
generate_function_result_content,
generate_message_content,
generate_streaming_annotation_content,
generate_streaming_code_interpreter_content,
generate_streaming_function_content,
generate_streaming_message_content,
get_function_call_contents,
get_message_contents,
)
from semantic_kernel.contents.annotation_content import AnnotationContent
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.file_reference_content import FileReferenceContent
from semantic_kernel.contents.function_call_content import FunctionCallContent
from semantic_kernel.contents.function_result_content import FunctionResultContent
from semantic_kernel.contents.image_content import ImageContent
from semantic_kernel.contents.streaming_annotation_content import StreamingAnnotationContent
from semantic_kernel.contents.streaming_file_reference_content import StreamingFileReferenceContent
from semantic_kernel.contents.streaming_text_content import StreamingTextContent
from semantic_kernel.contents.text_content import TextContent
from semantic_kernel.contents.utils.author_role import AuthorRole
def test_get_message_contents_all_types():
chat_msg = ChatMessageContent(role=AuthorRole.USER, content="")
chat_msg.items.append(TextContent(text="hello world"))
chat_msg.items.append(ImageContent(uri="http://example.com/image.png"))
chat_msg.items.append(FileReferenceContent(file_id="file123"))
chat_msg.items.append(FunctionResultContent(id="func1", result={"a": 1}))
results = get_message_contents(chat_msg)
assert len(results) == 4
assert results[0]["type"] == "text"
assert results[1]["type"] == "image_url"
assert results[2]["type"] == "image_file"
assert results[3]["type"] == "text"
def test_generate_message_content_text_and_image():
thread_msg = ThreadMessage(
content=[],
role="user",
)
image = MessageImageFileContent(image_file=MessageImageFileDetails(file_id="test_file_id"))
text = MessageTextContent(
text=MessageTextDetails(
value="some text",
annotations=[
MessageTextFileCitationAnnotation(
text="text",
file_citation=MessageTextFileCitationDetails(file_id="file_id", quote="some quote"),
start_index=0,
end_index=9,
),
MessageTextFilePathAnnotation(
text="text again",
file_path=MessageTextFilePathDetails(file_id="file_id_2"),
start_index=1,
end_index=10,
),
MessageTextUrlCitationAnnotation(
text="text",
url_citation=MessageTextUrlCitationDetails(title="some title", url="http://example.com"),
start_index=1,
end_index=10,
),
],
)
)
thread_msg.content = [image, text]
step = RunStep(id="step_id", run_id="run_id", thread_id="thread_id", agent_id="agent_id")
out = generate_message_content("assistant", thread_msg, step)
assert len(out.items) == 5
assert isinstance(out.items[0], FileReferenceContent)
assert isinstance(out.items[1], TextContent)
assert isinstance(out.items[2], AnnotationContent)
assert isinstance(out.items[3], AnnotationContent)
assert isinstance(out.items[4], AnnotationContent)
assert out.items[0].file_id == "test_file_id"
assert out.items[1].text == "some text"
assert out.items[2].file_id == "file_id"
assert out.items[2].quote == "text"
assert out.items[2].start_index == 0
assert out.items[2].end_index == 9
assert out.items[2].citation_type == "file_citation"
assert out.items[3].file_id == "file_id_2"
assert out.items[3].quote == "text again"
assert out.items[3].start_index == 1
assert out.items[3].end_index == 10
assert out.items[3].citation_type == "file_path"
assert out.items[4].url == "http://example.com"
assert out.items[4].quote == "text"
assert out.items[4].start_index == 1
assert out.items[4].end_index == 10
assert out.items[4].title == "some title"
assert out.items[4].citation_type == "url_citation"
assert out.metadata["step_id"] == "step_id"
assert out.role == AuthorRole.USER
def test_generate_annotation_content():
message_text_file_path_ann = MessageTextFilePathAnnotation(
text="some text",
file_path=MessageTextFilePathDetails(file_id="file123"),
start_index=0,
end_index=9,
)
message_text_file_citation_ann = MessageTextFileCitationAnnotation(
text="some text",
file_citation=MessageTextFileCitationDetails(file_id="file123"),
start_index=0,
end_index=9,
)
for fake_ann in [message_text_file_path_ann, message_text_file_citation_ann]:
out = generate_annotation_content(fake_ann)
assert out.file_id == "file123"
assert out.quote == "some text"
assert out.start_index == 0
assert out.end_index == 9
def test_generate_streaming_message_content_text_annotations():
message_delta_image_file_content = MessageDeltaImageFileContent(
index=0,
image_file=MessageDeltaImageFileContentObject(file_id="image_file"),
)
MessageDeltaTextFileCitationAnnotation, MessageDeltaTextFilePathAnnotation
message_delta_text_content = MessageDeltaTextContent(
index=0,
text=MessageDeltaTextContentObject(
value="some text",
annotations=[
MessageDeltaTextFileCitationAnnotation(
index=0,
file_citation=MessageDeltaTextFileCitationAnnotationObject(file_id="file123", quote="some text"),
start_index=0,
end_index=9,
text="some text",
),
MessageDeltaTextFilePathAnnotation(
index=0,
file_path=MessageDeltaTextFilePathAnnotationObject(file_id="file123"),
start_index=1,
end_index=10,
text="some text",
),
MessageDeltaTextUrlCitationAnnotation(
index=0,
url_citation=MessageDeltaTextUrlCitationDetails(
title="some title",
url="http://example.com",
),
start_index=2,
end_index=11,
),
],
),
)
delta = MessageDeltaChunk(
id="chunk123",
delta=MessageDelta(role="user", content=[message_delta_image_file_content, message_delta_text_content]),
)
out = generate_streaming_message_content("assistant", delta)
assert out is not None
assert out.content == "some text"
assert len(out.items) == 5
assert out.items[0].file_id == "image_file"
assert isinstance(out.items[0], StreamingFileReferenceContent)
assert isinstance(out.items[1], StreamingTextContent)
assert isinstance(out.items[2], StreamingAnnotationContent)
assert out.items[2].file_id == "file123"
assert out.items[2].quote == "some text"
assert out.items[2].start_index == 0
assert out.items[2].end_index == 9
assert out.items[2].citation_type == "file_citation"
assert isinstance(out.items[3], StreamingAnnotationContent)
assert out.items[3].file_id == "file123"
assert out.items[3].quote == "some text"
assert out.items[3].start_index == 1
assert out.items[3].end_index == 10
assert out.items[3].citation_type == "file_path"
assert isinstance(out.items[4], StreamingAnnotationContent)
assert out.items[4].url == "http://example.com"
assert out.items[4].title == "some title"
assert out.items[4].start_index == 2
assert out.items[4].end_index == 11
assert out.items[4].citation_type == "url_citation"
def test_generate_annotation_content_url_annotation_without_indices():
ann = MessageTextUrlCitationAnnotation(
text="url text",
url_citation=MessageTextUrlCitationDetails(title="", url="http://ex.com"),
start_index=None,
end_index=None,
)
out = generate_annotation_content(ann)
assert out.file_id is None
assert out.url == "http://ex.com"
assert out.title == "" # preserve empty title
assert out.quote == "url text"
assert out.start_index is None
assert out.end_index is None
assert out.citation_type == "url_citation"
def test_generate_streaming_annotation_content_url_quote_none_and_missing_indices():
ann = MessageDeltaTextUrlCitationAnnotation(
index=0,
url_citation=MessageDeltaTextUrlCitationDetails(title="", url="http://ex.com"),
start_index=None,
end_index=None,
)
out = generate_streaming_annotation_content(ann)
assert out.file_id is None
assert out.url == "http://ex.com"
assert out.title == ""
assert out.quote is None # no .text on URL annotation
assert out.start_index is None
assert out.end_index is None
assert out.citation_type == "url_citation"
def test_generate_streaming_message_content_text_only_no_annotations():
delta = MessageDeltaChunk(
id="c1",
delta=MessageDelta(
role="assistant",
content=[
MessageDeltaTextContent(
index=0,
text=MessageDeltaTextContentObject(value="just text", annotations=[]),
)
],
),
)
out = generate_streaming_message_content("assistant", delta, thread_msg_id="thread_1")
assert out.content == "just text"
assert len(out.items) == 1
assert isinstance(out.items[0], StreamingTextContent)
assert out.items[0].text == "just text"
assert out.metadata.get(THREAD_MESSAGE_ID) == "thread_1"
def test_generate_annotation_content_empty_title_and_url_only():
ann = MessageTextUrlCitationAnnotation(
text=None,
url_citation=MessageTextUrlCitationDetails(title=None, url="http://empty.com"),
start_index=5,
end_index=10,
)
out = generate_annotation_content(ann)
assert out.quote is None # allow None text
assert out.url == "http://empty.com"
assert out.title is None # allow None title
assert out.start_index == 5
assert out.end_index == 10
def test_generate_streaming_annotation_content_file_and_citation_have_text():
file_ann = MessageDeltaTextFileCitationAnnotation(
index=0,
file_citation=MessageDeltaTextFileCitationAnnotationObject(file_id="f1", quote="q1"),
start_index=2,
end_index=4,
text="q1",
)
out = generate_streaming_annotation_content(file_ann)
assert out.file_id == "f1"
assert out.quote == "q1"
assert out.citation_type == "file_citation"
assert out.start_index == 2
assert out.end_index == 4
def test_generate_streaming_function_content_with_function():
step_details = RunStepDeltaToolCallObject(
tool_calls=[
RunStepDeltaFunctionToolCall(
index=0, id="tool123", function=RunStepDeltaFunction(name="some_func", arguments={"arg": "val"})
)
]
)
out = generate_streaming_function_content("my_agent", step_details)
assert out is not None
assert len(out.items) == 1
assert isinstance(out.items[0], FunctionCallContent)
assert out.items[0].function_name == "some_func"
assert out.items[0].arguments == "{'arg': 'val'}"
def test_get_function_call_contents_no_action():
run = type("ThreadRunFake", (), {"required_action": None})()
fc = get_function_call_contents(run, {})
assert fc == []
def test_get_function_call_contents_submit_tool_outputs():
fake_function = MagicMock()
fake_function.name = "test_function"
fake_function.arguments = {"arg": "val"}
fake_tool_call = MagicMock(spec=RequiredFunctionToolCall)
fake_tool_call.id = "tool_id"
fake_tool_call.function = fake_function
run = MagicMock()
run.required_action.submit_tool_outputs.tool_calls = [fake_tool_call]
function_steps = {}
fc = get_function_call_contents(run, function_steps)
assert len(fc) == 1
assert fc[0].id == "tool_id"
assert fc[0].name == "test_function"
assert fc[0].arguments == {"arg": "val"}
def test_generate_function_call_content():
fcc = FunctionCallContent(id="id123", name="func_name", arguments={"x": 1})
msg = generate_function_call_content("my_agent", [fcc])
assert len(msg.items) == 1
assert msg.role == AuthorRole.ASSISTANT
def test_generate_function_result_content():
step = FunctionCallContent(id="123", name="func_name", arguments={"k": "v"})
tool_call = RunStepFunctionToolCall(
id="123",
function=RunStepFunctionToolCallDetails({
"name": "func_name",
"arguments": '{"k": "v"}',
"output": "result_data",
}),
)
msg = generate_function_result_content("my_agent", step, tool_call)
assert len(msg.items) == 1
assert msg.items[0].result == "result_data"
assert msg.role == AuthorRole.TOOL
def test_generate_code_interpreter_content():
msg = generate_code_interpreter_content("my_agent", "some_code()")
assert msg.content == "some_code()"
assert msg.metadata["code"] is True
def test_generate_streaming_code_interpreter_content_no_calls():
step_details = type("Details", (), {"tool_calls": None})
assert generate_streaming_code_interpreter_content("my_agent", step_details) is None
def test_generate_bing_grounding_content():
"""Test generate_bing_grounding_content with RunStepBingGroundingToolCall."""
bing_grounding_tool_call = RunStepBingGroundingToolCall(
id="call_gvgTmSL4hgdxWP4O7LLnwMlt",
bing_grounding={
"requesturl": "https://api.bing.microsoft.com/v7.0/search?q=search",
"response_metadata": "{'market': 'en-US', 'num_docs_retrieved': 5, 'num_docs_actually_used': 5}",
},
)
msg = generate_bing_grounding_content("my_agent", bing_grounding_tool_call)
assert len(msg.items) == 1
assert msg.role == AuthorRole.ASSISTANT
assert isinstance(msg.items[0], FunctionCallContent)
assert msg.items[0].id == "call_gvgTmSL4hgdxWP4O7LLnwMlt"
assert msg.items[0].name == "bing_grounding"
assert msg.items[0].function_name == "bing_grounding"
assert msg.items[0].arguments["requesturl"] == "https://api.bing.microsoft.com/v7.0/search?q=search"
assert msg.items[0].arguments["response_metadata"] == (
"{'market': 'en-US', 'num_docs_retrieved': 5, 'num_docs_actually_used': 5}"
)
def test_generate_bing_custom_search_content():
"""Test generate_bing_grounding_content with RunStepBingCustomSearchToolCall."""
bing_custom_search_tool_call = RunStepBingCustomSearchToolCall(
id="call_abc123def456ghi",
bing_custom_search={
"query": "semantic kernel python",
"custom_config_id": "config_123",
"search_results": "{'num_results': 10, 'top_result': 'Microsoft Semantic Kernel'}",
},
)
msg = generate_bing_grounding_content("my_agent", bing_custom_search_tool_call)
assert len(msg.items) == 1
assert msg.role == AuthorRole.ASSISTANT
assert isinstance(msg.items[0], FunctionCallContent)
assert msg.items[0].id == "call_abc123def456ghi"
assert msg.items[0].name == "bing_custom_search"
assert msg.items[0].function_name == "bing_custom_search"
assert msg.items[0].arguments["query"] == "semantic kernel python"
assert msg.items[0].arguments["custom_config_id"] == "config_123"
assert msg.items[0].arguments["search_results"] == (
"{'num_results': 10, 'top_result': 'Microsoft Semantic Kernel'}"
)
@@ -0,0 +1,662 @@
# Copyright (c) Microsoft. All rights reserved.
from datetime import datetime, timezone
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from azure.ai.agents.models import (
MessageTextContent,
MessageTextDetails,
RequiredFunctionToolCall,
RequiredFunctionToolCallDetails,
RunStep,
RunStepCodeInterpreterToolCall,
RunStepCodeInterpreterToolCallDetails,
RunStepFunctionToolCall,
RunStepFunctionToolCallDetails,
RunStepMessageCreationDetails,
RunStepMessageCreationReference,
RunStepToolCallDetails,
SubmitToolOutputsAction,
SubmitToolOutputsDetails,
ThreadMessage,
ThreadRun,
)
from azure.ai.projects.aio import AIProjectClient
from pytest import fixture
from semantic_kernel.agents.azure_ai.agent_thread_actions import AgentThreadActions
from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
from semantic_kernel.contents import FunctionCallContent, FunctionResultContent, TextContent
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions.agent_exceptions import AgentInvokeException
from semantic_kernel.functions.kernel_arguments import KernelArguments
from semantic_kernel.functions.kernel_function_decorator import kernel_function
from semantic_kernel.functions.kernel_plugin import KernelPlugin
from semantic_kernel.kernel import Kernel
@fixture
def mock_client():
mock_thread = AsyncMock()
mock_thread.id = "thread123"
mock_threads = MagicMock()
mock_threads.create = AsyncMock(return_value=mock_thread)
mock_message = AsyncMock()
mock_message.id = "message456"
mock_messages = MagicMock()
mock_messages.create = AsyncMock(return_value="someMessage")
mock_agents = MagicMock()
mock_agents.threads = mock_threads
mock_agents.messages = mock_messages
mock_client = AsyncMock(spec=AIProjectClient)
mock_client.agents = mock_agents
return mock_client
async def test_agent_thread_actions_create_thread(mock_client):
thread_id = await AgentThreadActions.create_thread(mock_client)
assert thread_id == "thread123"
async def test_agent_thread_actions_create_message(mock_client):
msg = ChatMessageContent(role=AuthorRole.USER, content="some content")
out = await AgentThreadActions.create_message(mock_client, "threadXYZ", msg)
assert out == "someMessage"
async def test_agent_thread_actions_create_message_no_content():
class FakeAgentClient:
create_message = AsyncMock(return_value="should_not_be_called")
class FakeClient:
agents = FakeAgentClient()
message = ChatMessageContent(role=AuthorRole.USER, content=" ")
out = await AgentThreadActions.create_message(FakeClient(), "threadXYZ", message)
assert out is None
assert FakeAgentClient.create_message.await_count == 0
async def test_agent_thread_actions_invoke(ai_project_client: AIProjectClient, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
# Properly construct nested mocks without re-spec'ing from a mock
mock_thread_run = ThreadRun(
id="run123",
thread_id="thread123",
status="running",
instructions="test agent",
created_at=int(datetime.now(timezone.utc).timestamp()),
model="model",
)
agent.client.agents.runs = MagicMock()
agent.client.agents.runs.create = AsyncMock(return_value=mock_thread_run)
agent.client.agents.runs.get = AsyncMock(return_value=mock_thread_run)
async def mock_poll_run_status(*args, **kwargs):
yield RunStep(
type="message_creation",
id="msg123",
thread_id="thread123",
run_id="run123",
created_at=int(datetime.now(timezone.utc).timestamp()),
completed_at=int(datetime.now(timezone.utc).timestamp()),
status="completed",
agent_id="agent123",
step_details=RunStepMessageCreationDetails(
message_creation=RunStepMessageCreationReference(
message_id="msg123",
),
),
)
agent.client.agents.run_steps = MagicMock()
agent.client.agents.run_steps.list = mock_poll_run_status
mock_message = ThreadMessage(
id="msg123",
thread_id="thread123",
run_id="run123",
created_at=int(datetime.now(timezone.utc).timestamp()),
completed_at=int(datetime.now(timezone.utc).timestamp()),
status="completed",
agent_id="agent123",
role="assistant",
content=[MessageTextContent(text=MessageTextDetails(value="some message", annotations=[]))],
)
agent.client.agents.messages = MagicMock()
agent.client.agents.messages.get = AsyncMock(return_value=mock_message)
async for is_visible, message in AgentThreadActions.invoke(
agent=agent, thread_id="thread123", kernel=AsyncMock(spec=Kernel)
):
assert str(message.content) == "some message"
break
async def test_agent_thread_actions_invoke_with_requires_action(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
agent.client.agents = MagicMock()
mock_thread_run = ThreadRun(
id="run123",
thread_id="thread123",
status="running",
instructions="test agent",
created_at=int(datetime.now(timezone.utc).timestamp()),
model="model",
)
agent.client.agents = MagicMock()
agent.client.agents.runs = MagicMock()
agent.client.agents.runs.create = AsyncMock(return_value=mock_thread_run)
agent.client.agents.runs.get = AsyncMock(return_value=mock_thread_run)
agent.client.agents.runs.submit_tool_outputs = AsyncMock()
poll_count = 0
async def mock_poll_run_status(*args, **kwargs):
nonlocal poll_count
if poll_count == 0:
mock_thread_run.status = "requires_action"
mock_thread_run.required_action = SubmitToolOutputsAction(
submit_tool_outputs=SubmitToolOutputsDetails(
tool_calls=[
RequiredFunctionToolCall(
id="tool_call_id",
function=RequiredFunctionToolCallDetails(
name="mock_function_call", arguments={"arg": "value"}
),
)
]
)
)
else:
mock_thread_run.status = "completed"
poll_count += 1
return mock_thread_run
def mock_get_function_call_contents(run: ThreadRun, function_steps: dict):
function_call_content = FunctionCallContent(
name="mock_function_call",
arguments={"arg": "value"},
id="tool_call_id",
)
function_steps[function_call_content.id] = function_call_content
return [function_call_content]
mock_run_step_tool_calls = RunStep(
type="tool_calls",
id="tool_step123",
thread_id="thread123",
run_id="run123",
created_at=int(datetime.now(timezone.utc).timestamp()),
completed_at=int(datetime.now(timezone.utc).timestamp()),
status="completed",
agent_id="agent123",
step_details=RunStepToolCallDetails(
tool_calls=[
# 1. This will yield FunctionResultContent
RunStepFunctionToolCall(
id="tool_call_id",
function=RunStepFunctionToolCallDetails({
"name": "mock_function_call",
"arguments": '{"arg": "value"}',
"output": "some output",
}),
),
# 2. This will yield TextContent
RunStepCodeInterpreterToolCall(
id="tool_call_id",
code_interpreter=RunStepCodeInterpreterToolCallDetails(
input="some code",
),
),
]
),
)
mock_run_step_message_creation = RunStep(
type="message_creation",
id="msg_step123",
thread_id="thread123",
run_id="run123",
created_at=int(datetime.now(timezone.utc).timestamp()),
completed_at=int(datetime.now(timezone.utc).timestamp()),
status="completed",
agent_id="agent123",
step_details=RunStepMessageCreationDetails(
message_creation=RunStepMessageCreationReference(message_id="msg123")
),
)
mock_run_steps = [mock_run_step_tool_calls, mock_run_step_message_creation]
async def mock_list_run_steps(*args, **kwargs):
for step in mock_run_steps:
yield step
agent.client.agents.run_steps = MagicMock()
agent.client.agents.run_steps.list = mock_list_run_steps
mock_message = ThreadMessage(
id="msg123",
thread_id="thread123",
run_id="run123",
created_at=int(datetime.now(timezone.utc).timestamp()),
completed_at=int(datetime.now(timezone.utc).timestamp()),
status="completed",
agent_id="agent123",
role="assistant",
content=[MessageTextContent(text=MessageTextDetails(value="some message", annotations=[]))],
)
agent.client.agents.runs.get = AsyncMock(return_value=mock_message)
agent.client.agents.runs.submit_tool_outputs = AsyncMock()
with (
patch.object(AgentThreadActions, "_poll_run_status", side_effect=mock_poll_run_status),
patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.get_function_call_contents",
side_effect=mock_get_function_call_contents,
),
patch.object(AgentThreadActions, "_invoke_function_calls", return_value=[None]),
):
messages = []
async for is_visible, content in AgentThreadActions.invoke(
agent=agent,
thread_id="thread123",
kernel=AsyncMock(spec=Kernel),
):
messages.append((is_visible, content))
assert len(messages) == 3, "There should be three yields in total."
assert isinstance(messages[0][1].items[0], FunctionCallContent)
assert isinstance(messages[1][1].items[0], FunctionResultContent)
assert isinstance(messages[2][1].items[0], TextContent)
agent.client.agents.runs.submit_tool_outputs.assert_awaited_once()
class MockEvent:
def __init__(self, event, data):
self.event = event
self.data = data
def __iter__(self):
return iter((self.event, self.data, None))
class MockRunData:
def __init__(self, id, status, content: str | None = None):
self.id = id
self.status = status
self.content = content
class MockAsyncIterable:
def __init__(self, items):
self.items = items.copy()
def __aiter__(self):
self._iter = iter(self.items)
return self
async def __anext__(self):
try:
return next(self._iter)
except StopIteration:
raise StopAsyncIteration
class MockStream:
def __init__(self, events):
self.events = events
async def __aenter__(self):
return MockAsyncIterable(self.events)
async def __aexit__(self, exc_type, exc_val, exc_tb):
pass
async def test_agent_thread_actions_invoke_stream(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
agent.client.agents = AsyncMock()
events = [
MockEvent("thread.run.created", MockRunData(id="run_1", status="queued")),
MockEvent("thread.message.created", MockRunData(id="msg_1", status="created", content="Hello")),
MockEvent("thread.run.in_progress", MockRunData(id="run_1", status="in_progress")),
MockEvent("thread.run.completed", MockRunData(id="run_1", status="completed")),
]
main_run_stream = MockStream(events)
agent.client.agents.create_stream.return_value = main_run_stream
with (
patch.object(AgentThreadActions, "_invoke_function_calls", return_value=None),
patch.object(AgentThreadActions, "_format_tool_outputs", return_value=[{"type": "mock_tool_output"}]),
):
collected_messages = []
async for content in AgentThreadActions.invoke_stream(
agent=agent,
thread_id="thread123",
kernel=AsyncMock(spec=Kernel),
):
collected_messages.append(content)
assert isinstance(content, ChatMessageContent)
assert content.metadata.get("message_id") == "msg_1"
# region Security tests for tools override and function_choice_behavior
async def test_validate_function_choice_behavior_rejects_required():
"""Required FCB is not supported for agent invocations."""
with pytest.raises(AgentInvokeException, match="not supported"):
AgentThreadActions._validate_function_choice_behavior(FunctionChoiceBehavior.Required())
async def test_validate_function_choice_behavior_accepts_auto():
"""Auto FCB should be accepted without error."""
AgentThreadActions._validate_function_choice_behavior(FunctionChoiceBehavior.Auto())
async def test_validate_function_choice_behavior_rejects_none_invoke():
"""NoneInvoke FCB is not supported for agent invocations."""
with pytest.raises(AgentInvokeException, match="not supported"):
AgentThreadActions._validate_function_choice_behavior(FunctionChoiceBehavior.NoneInvoke())
async def test_validate_function_choice_behavior_accepts_none():
"""None (no FCB) should be accepted."""
AgentThreadActions._validate_function_choice_behavior(None)
async def test_validate_function_choice_behavior_rejects_auto_invoke_false():
"""Auto with auto_invoke=False is not supported for agent invocations."""
with pytest.raises(AgentInvokeException, match="auto_invoke"):
AgentThreadActions._validate_function_choice_behavior(FunctionChoiceBehavior.Auto(auto_invoke=False))
async def test_validate_function_choice_behavior_rejects_empty_filters():
"""Empty filters dict should be rejected."""
fcb = FunctionChoiceBehavior.Auto()
fcb.filters = {}
with pytest.raises(AgentInvokeException, match="must not be empty"):
AgentThreadActions._validate_function_choice_behavior(fcb)
async def test_validate_function_choice_behavior_rejects_unknown_filter_keys():
"""Unknown filter keys should be rejected."""
fcb = FunctionChoiceBehavior.Auto()
# Bypass Pydantic validation to simulate a mistyped key reaching the validator
object.__setattr__(fcb, "filters", {"include_functions": ["foo"]})
with pytest.raises(AgentInvokeException, match="Unknown filter key"):
AgentThreadActions._validate_function_choice_behavior(fcb)
async def test_validate_function_choice_behavior_accepts_valid_filters():
"""Valid filter keys should be accepted."""
AgentThreadActions._validate_function_choice_behavior(
FunctionChoiceBehavior.Auto(filters={"included_functions": ["plugin-func"]})
)
async def test_get_tools_with_tools_override(ai_project_client, ai_agent_definition):
"""When tools_override is provided, it should replace agent.definition.tools."""
from azure.ai.agents.models import CodeInterpreterToolDefinition
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
kernel = MagicMock(spec=Kernel)
kernel.get_full_list_of_function_metadata.return_value = []
override_tool = CodeInterpreterToolDefinition()
tools = AgentThreadActions._get_tools(agent=agent, kernel=kernel, tools_override=[override_tool])
# Should contain the override tool, not agent.definition.tools
assert any(
(isinstance(t, CodeInterpreterToolDefinition) or (isinstance(t, dict) and t.get("type") == "code_interpreter"))
for t in tools
)
async def test_get_tools_with_fcb_filters(ai_project_client, ai_agent_definition):
"""When function_choice_behavior has filters, only matching functions should be included."""
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
kernel = MagicMock(spec=Kernel)
# Simulate filtered metadata
mock_metadata = MagicMock()
mock_metadata.fully_qualified_name = "Plugin-AllowedFunc"
mock_metadata.name = "AllowedFunc"
mock_metadata.plugin_name = "Plugin"
mock_metadata.description = "An allowed function"
mock_metadata.parameters = []
mock_metadata.is_prompt = False
mock_metadata.return_parameter = MagicMock()
mock_metadata.return_parameter.description = ""
mock_metadata.return_parameter.type_ = "str"
mock_metadata.additional_properties = {}
kernel.get_list_of_function_metadata.return_value = [mock_metadata]
kernel.get_full_list_of_function_metadata.return_value = []
fcb = FunctionChoiceBehavior.Auto(filters={"included_functions": ["Plugin-AllowedFunc"]})
AgentThreadActions._get_tools(agent=agent, kernel=kernel, function_choice_behavior=fcb)
# Should have called get_list_of_function_metadata with the filters
kernel.get_list_of_function_metadata.assert_called_once_with(fcb.filters)
async def test_get_tools_with_fcb_disable_kernel_functions(ai_project_client, ai_agent_definition):
"""When enable_kernel_functions=False, no kernel functions should be included."""
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
kernel = MagicMock(spec=Kernel)
fcb = FunctionChoiceBehavior.Auto(enable_kernel_functions=False)
AgentThreadActions._get_tools(agent=agent, kernel=kernel, function_choice_behavior=fcb)
# Full list is called for validation, but filtered list should not be called
kernel.get_full_list_of_function_metadata.assert_called_once()
kernel.get_list_of_function_metadata.assert_not_called()
async def test_invoke_function_calls_passes_function_behavior():
"""_invoke_function_calls should pass function_behavior to kernel.invoke_function_call."""
mock_kernel = AsyncMock(spec=Kernel)
mock_kernel.invoke_function_call.return_value = None
fcc = FunctionCallContent(name="Plugin-Func", arguments={}, id="call1")
from semantic_kernel.contents.chat_history import ChatHistory
chat_history = ChatHistory()
fcb = FunctionChoiceBehavior.Auto(filters={"included_functions": ["Plugin-Func"]})
await AgentThreadActions._invoke_function_calls(
kernel=mock_kernel,
fccs=[fcc],
chat_history=chat_history,
arguments=KernelArguments(),
function_choice_behavior=fcb,
)
mock_kernel.invoke_function_call.assert_awaited_once()
call_kwargs = mock_kernel.invoke_function_call.call_args
assert call_kwargs.kwargs.get("function_behavior") is fcb
async def test_invoke_function_calls_passes_disabled_kernel_functions():
"""_invoke_function_calls should pass enable_kernel_functions=False FCB to kernel."""
mock_kernel = AsyncMock(spec=Kernel)
mock_kernel.invoke_function_call.return_value = None
fcc = FunctionCallContent(name="Plugin-Func", arguments={}, id="call1")
from semantic_kernel.contents.chat_history import ChatHistory
chat_history = ChatHistory()
fcb = FunctionChoiceBehavior.Auto(enable_kernel_functions=False)
await AgentThreadActions._invoke_function_calls(
kernel=mock_kernel,
fccs=[fcc],
chat_history=chat_history,
arguments=KernelArguments(),
function_choice_behavior=fcb,
)
mock_kernel.invoke_function_call.assert_awaited_once()
call_kwargs = mock_kernel.invoke_function_call.call_args
passed_behavior = call_kwargs.kwargs.get("function_behavior")
assert passed_behavior is fcb
assert not passed_behavior.enable_kernel_functions
async def test_invoke_function_calls_blocks_disallowed_function():
"""A real Kernel should block a function call not in the FCB allowlist.
This verifies that the enforcement in kernel.invoke_function_call actually
rejects a disallowed function name when filters are provided, rather than
only asserting that the kwarg is forwarded.
"""
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.functions.kernel_function_from_method import KernelFunctionFromMethod
@kernel_function
def allowed_func() -> str:
return "allowed"
@kernel_function
def disallowed_func() -> str:
return "disallowed"
kernel = Kernel()
kernel.add_plugin(
KernelPlugin(
name="Plugin",
functions=[
KernelFunctionFromMethod(method=allowed_func, plugin_name="Plugin"),
KernelFunctionFromMethod(method=disallowed_func, plugin_name="Plugin"),
],
)
)
fcb = FunctionChoiceBehavior.Auto(filters={"included_functions": ["Plugin-allowed_func"]})
# Call a function NOT in the allowlist
fcc = FunctionCallContent(
name="Plugin-disallowed_func",
plugin_name="Plugin",
function_name="disallowed_func",
arguments={},
id="call1",
)
chat_history = ChatHistory()
result = await kernel.invoke_function_call(
function_call=fcc,
chat_history=chat_history,
function_behavior=fcb,
)
# invoke_function_call catches the FunctionExecutionException and returns None,
# adding an error message to chat_history instead of raising.
assert result is None
assert len(chat_history.messages) == 1
result_item = chat_history.messages[0].items[0]
assert "not part of the provided tools" in str(result_item.result)
async def test_invoke_function_calls_allows_permitted_function():
"""A real Kernel should allow a function call that IS in the FCB allowlist."""
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.functions.kernel_function_from_method import KernelFunctionFromMethod
@kernel_function
def allowed_func() -> str:
return "ok"
@kernel_function
def other_func() -> str:
return "other"
kernel = Kernel()
kernel.add_plugin(
KernelPlugin(
name="Plugin",
functions=[
KernelFunctionFromMethod(method=allowed_func, plugin_name="Plugin"),
KernelFunctionFromMethod(method=other_func, plugin_name="Plugin"),
],
)
)
fcb = FunctionChoiceBehavior.Auto(filters={"included_functions": ["Plugin-allowed_func"]})
fcc = FunctionCallContent(
name="Plugin-allowed_func",
plugin_name="Plugin",
function_name="allowed_func",
arguments={},
id="call1",
)
chat_history = ChatHistory()
await kernel.invoke_function_call(
function_call=fcc,
chat_history=chat_history,
function_behavior=fcb,
)
# Should succeed — the function result should be in chat_history
assert len(chat_history.messages) == 1
result_item = chat_history.messages[0].items[0]
assert "ok" in str(result_item.result)
async def test_invoke_raises_for_non_auto_fcb(ai_project_client, ai_agent_definition):
"""Calling AgentThreadActions.invoke() with a non-Auto FCB should raise before any API call."""
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
agent.client.agents = AsyncMock()
with pytest.raises(AgentInvokeException, match="not supported"):
async for _ in AgentThreadActions.invoke(
agent=agent,
thread_id="thread123",
kernel=Kernel(),
function_choice_behavior=FunctionChoiceBehavior.Required(),
):
pass
# No API calls should have been made
agent.client.agents.runs.create.assert_not_awaited()
async def test_invoke_stream_raises_for_non_auto_fcb(ai_project_client, ai_agent_definition):
"""Calling AgentThreadActions.invoke_stream() with a non-Auto FCB should raise before any API call."""
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
agent.client.agents = AsyncMock()
with pytest.raises(AgentInvokeException, match="not supported"):
async for _ in AgentThreadActions.invoke_stream(
agent=agent,
thread_id="thread123",
kernel=Kernel(),
function_choice_behavior=FunctionChoiceBehavior.NoneInvoke(),
):
pass
# No API calls should have been made
agent.client.agents.create_stream.assert_not_called()
# endregion
@@ -0,0 +1,478 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch
import pytest
from azure.ai.projects.aio import AIProjectClient
from azure.core.credentials_async import AsyncTokenCredential
from semantic_kernel.agents.agent import AgentResponseItem
from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent, AzureAIAgentThread
from semantic_kernel.agents.channels.agent_channel import AgentChannel
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.function_call_content import FunctionCallContent
from semantic_kernel.contents.function_result_content import FunctionResultContent
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions.agent_exceptions import AgentInitializationException, AgentInvokeException
async def test_azure_ai_agent_init(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
assert agent.id == "agent123"
assert agent.name == "agentName"
assert agent.description == "desc"
async def test_azure_ai_agent_init_with_plugins_via_constructor(
ai_project_client, ai_agent_definition, custom_plugin_class
):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition, plugins=[custom_plugin_class()])
assert agent.id == "agent123"
assert agent.name == "agentName"
assert agent.description == "desc"
assert agent.kernel.plugins is not None
assert len(agent.kernel.plugins) == 1
async def test_azure_ai_agent_get_response(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
thread = AsyncMock(spec=AzureAIAgentThread)
async def fake_invoke(*args, **kwargs):
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke",
side_effect=fake_invoke,
):
response = await agent.get_response(messages="message", thread=thread)
assert response.message.role == AuthorRole.ASSISTANT
assert response.message.content == "content"
assert response.thread is not None
async def test_azure_ai_agent_get_response_exception(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
thread = AsyncMock(spec=AzureAIAgentThread)
async def fake_invoke(*args, **kwargs):
yield False, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
with (
patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke",
side_effect=fake_invoke,
),
pytest.raises(AgentInvokeException),
):
await agent.get_response(messages="message", thread=thread)
async def test_azure_ai_agent_invoke(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
thread = AsyncMock(spec=AzureAIAgentThread)
results = []
async def fake_invoke(*args, **kwargs):
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke",
side_effect=fake_invoke,
):
async for item in agent.invoke(messages="message", thread=thread):
results.append(item)
assert len(results) == 1
async def test_azure_ai_agent_invoke_yields_visible_assistant_message(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
thread = AsyncMock(spec=AzureAIAgentThread)
results = []
assistant_msg = ChatMessageContent(role=AuthorRole.ASSISTANT, content="assistant says hi")
async def fake_invoke(*args, **kwargs):
yield True, assistant_msg
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke",
side_effect=fake_invoke,
):
async for item in agent.invoke(messages="message", thread=thread):
results.append(item)
assert len(results) == 1
assert results[0].message is assistant_msg
async def test_azure_ai_agent_invoke_emits_tool_message_via_callback_only(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
thread = AsyncMock(spec=AzureAIAgentThread)
callback_results = []
async def handle_callback(msg: ChatMessageContent) -> None:
callback_results.append(msg)
tool_msg = ChatMessageContent(role=AuthorRole.ASSISTANT, content="tool call")
tool_msg.items = [FunctionCallContent(name="tool", arguments="{}")]
async def fake_invoke(*args, **kwargs):
yield False, tool_msg
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke",
side_effect=fake_invoke,
):
async for _ in agent.invoke(messages="message", thread=thread, on_intermediate_message=handle_callback):
pass
assert callback_results == [tool_msg]
async def test_azure_ai_agent_invoke_suppresses_tool_message_without_callback(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
thread = AsyncMock(spec=AzureAIAgentThread)
tool_msg = ChatMessageContent(role=AuthorRole.ASSISTANT, content="tool call")
tool_msg.items = [FunctionCallContent(name="tool", arguments="{}")]
async def fake_invoke(*args, **kwargs):
yield False, tool_msg # Not visible, no callback
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke",
side_effect=fake_invoke,
):
results = [item async for item in agent.invoke(messages="message", thread=thread)]
assert results == [] # Tool message should be suppressed
async def test_azure_ai_agent_invoke_stream(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
thread = AsyncMock(spec=AzureAIAgentThread)
results = []
async def fake_invoke(*args, **kwargs):
yield ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke_stream",
side_effect=fake_invoke,
):
async for item in agent.invoke_stream(messages="message", thread=thread):
results.append(item)
assert len(results) == 1
async def test_azure_ai_agent_invoke_stream_with_on_new_message_callback(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
thread = AsyncMock(spec=AzureAIAgentThread)
thread.id = "test_thread_id"
results = []
final_chat_history = ChatHistory()
async def handle_stream_completion(message: ChatMessageContent) -> None:
final_chat_history.add_message(message)
# Fake collected messages
fake_message = StreamingChatMessageContent(role=AuthorRole.ASSISTANT, content="fake content", choice_index=0)
async def fake_invoke(*args, output_messages=None, **kwargs):
if output_messages is not None:
output_messages.append(fake_message)
yield fake_message
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke_stream",
side_effect=fake_invoke,
):
async for item in agent.invoke_stream(
messages="message", thread=thread, on_intermediate_message=handle_stream_completion
):
results.append(item)
assert len(results) == 1
assert results[0].message.content == "fake content"
assert len(final_chat_history.messages) == 1
assert final_chat_history.messages[0].content == "fake content"
async def test_azure_ai_agent_invoke_stream_tool_message_only_goes_to_callback(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
thread = AsyncMock(spec=AzureAIAgentThread)
thread.id = "test_thread_id"
received_callback_messages = []
async def async_append(msg: ChatMessageContent):
received_callback_messages.append(msg)
tool_msg = ChatMessageContent(
role=AuthorRole.ASSISTANT, content="tool call", items=[FunctionCallContent(name="ToolA", arguments="{}")]
)
streamed_msg = StreamingChatMessageContent(
role=AuthorRole.ASSISTANT, content="assistant streaming...", choice_index=0
)
async def fake_invoke_stream(*args, output_messages=None, **kwargs):
if output_messages is not None:
output_messages.append(tool_msg)
yield streamed_msg
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke_stream",
side_effect=fake_invoke_stream,
):
results = []
async for item in agent.invoke_stream(messages="message", thread=thread, on_intermediate_message=async_append):
results.append(item)
assert results == [AgentResponseItem(message=streamed_msg, thread=thread)]
assert received_callback_messages == [tool_msg]
async def test_azure_ai_agent_invoke_stream_tool_message_suppressed_without_callback(
ai_project_client, ai_agent_definition
):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
thread = AsyncMock(spec=AzureAIAgentThread)
thread.id = "test_thread_id"
tool_msg = ChatMessageContent(
role=AuthorRole.ASSISTANT,
content="tool result",
items=[FunctionResultContent(id="test-id", name="ToolA", result="result")],
)
streamed_msg = StreamingChatMessageContent(role=AuthorRole.ASSISTANT, content="assistant says hi", choice_index=0)
async def fake_invoke_stream(*args, output_messages=None, **kwargs):
if output_messages is not None:
output_messages.append(tool_msg)
yield streamed_msg
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke_stream",
side_effect=fake_invoke_stream,
):
results = []
async for item in agent.invoke_stream(messages="message", thread=thread):
results.append(item)
# Only assistant-visible content should be yielded
assert len(results) == 1
assert results[0].message == streamed_msg
async def test_azure_ai_agent_invoke_stream_mixed_messages(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
thread = AsyncMock(spec=AzureAIAgentThread)
thread.id = "test_thread_id"
callback_results = []
async def async_append(msg: ChatMessageContent):
callback_results.append(msg)
tool_msg = ChatMessageContent(
role=AuthorRole.ASSISTANT, content="tool call", items=[FunctionCallContent(name="tool", arguments="{}")]
)
text_msg = StreamingChatMessageContent(role=AuthorRole.ASSISTANT, content="streamed text", choice_index=0)
async def fake_invoke_stream(*args, output_messages: list = None, **kwargs):
if output_messages is not None:
output_messages.append(tool_msg)
yield text_msg
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke_stream",
side_effect=fake_invoke_stream,
):
results = []
async for item in agent.invoke_stream(messages="message", thread=thread, on_intermediate_message=async_append):
results.append(item)
assert callback_results == [tool_msg]
assert results == [AgentResponseItem(message=text_msg, thread=thread)]
def test_azure_ai_agent_get_channel_keys(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
keys = list(agent.get_channel_keys())
assert len(keys) >= 2
async def test_azure_ai_agent_create_channel(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
with (
patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.create_thread",
side_effect="t",
),
patch(
"semantic_kernel.agents.azure_ai.azure_ai_agent.AzureAIAgentThread.create",
new_callable=AsyncMock,
),
patch(
"semantic_kernel.agents.azure_ai.azure_ai_agent.AzureAIAgentThread.id",
new_callable=PropertyMock,
) as mock_id,
):
mock_id.return_value = "mock-thread-id"
ch = await agent.create_channel()
assert isinstance(ch, AgentChannel)
assert ch.thread_id == "mock-thread-id"
def test_create_client_with_explicit_endpoint():
credential = MagicMock(spec=AsyncTokenCredential)
with patch("semantic_kernel.agents.azure_ai.azure_ai_agent.AIProjectClient") as mock_client_cls:
mock_client = MagicMock(spec=AIProjectClient)
mock_client_cls.return_value = mock_client
result = AzureAIAgent.create_client(
credential=credential,
endpoint="https://my-endpoint",
extra_arg="extra_value",
)
mock_client_cls.assert_called_once()
_, kwargs = mock_client_cls.call_args
assert kwargs["credential"] is credential
assert kwargs["endpoint"] == "https://my-endpoint"
assert kwargs["extra_arg"] == "extra_value"
assert result is mock_client
def test_create_client_uses_settings_when_endpoint_none():
credential = MagicMock(spec=AsyncTokenCredential)
with (
patch("semantic_kernel.agents.azure_ai.azure_ai_agent.AzureAIAgentSettings") as mock_settings_cls,
patch("semantic_kernel.agents.azure_ai.azure_ai_agent.AIProjectClient") as mock_client_cls,
):
mock_settings = MagicMock()
mock_settings.endpoint = "https://configured-endpoint"
mock_settings_cls.return_value = mock_settings
mock_client = MagicMock(spec=AIProjectClient)
mock_client_cls.return_value = mock_client
result = AzureAIAgent.create_client(credential=credential)
mock_client_cls.assert_called_once()
_, kwargs = mock_client_cls.call_args
assert kwargs["endpoint"] == "https://configured-endpoint"
assert result is mock_client
def test_create_client_raises_if_no_endpoint():
credential = MagicMock(spec=AsyncTokenCredential)
with patch("semantic_kernel.agents.azure_ai.azure_ai_agent.AzureAIAgentSettings") as mock_settings_cls:
mock_settings = MagicMock()
mock_settings.endpoint = None
mock_settings_cls.return_value = mock_settings
try:
AzureAIAgent.create_client(credential=credential)
except AgentInitializationException as e:
assert "Azure AI endpoint" in str(e)
else:
assert False, "Expected AgentInitializationException to be raised"
async def test_azure_ai_agent_get_response_passes_function_choice_behavior(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
thread = AsyncMock(spec=AzureAIAgentThread)
fcb = FunctionChoiceBehavior.Auto()
captured_kwargs = {}
async def fake_invoke(*args, **kwargs):
captured_kwargs.update(kwargs)
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke",
side_effect=fake_invoke,
):
await agent.get_response(messages="message", thread=thread, function_choice_behavior=fcb)
assert captured_kwargs.get("function_choice_behavior") is fcb
async def test_azure_ai_agent_invoke_passes_function_choice_behavior(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
thread = AsyncMock(spec=AzureAIAgentThread)
fcb = FunctionChoiceBehavior.Auto()
captured_kwargs = {}
async def fake_invoke(*args, **kwargs):
captured_kwargs.update(kwargs)
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke",
side_effect=fake_invoke,
):
async for _ in agent.invoke(messages="message", thread=thread, function_choice_behavior=fcb):
pass
assert captured_kwargs.get("function_choice_behavior") is fcb
async def test_azure_ai_agent_invoke_stream_passes_function_choice_behavior(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
thread = AsyncMock(spec=AzureAIAgentThread)
fcb = FunctionChoiceBehavior.Auto()
captured_kwargs = {}
async def fake_invoke(*args, **kwargs):
captured_kwargs.update(kwargs)
yield ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke_stream",
side_effect=fake_invoke,
):
async for _ in agent.invoke_stream(messages="message", thread=thread, function_choice_behavior=fcb):
pass
assert captured_kwargs.get("function_choice_behavior") is fcb
async def test_azure_ai_agent_get_response_no_fcb_passes_none(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
thread = AsyncMock(spec=AzureAIAgentThread)
captured_kwargs = {}
async def fake_invoke(*args, **kwargs):
captured_kwargs.update(kwargs)
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke",
side_effect=fake_invoke,
):
await agent.get_response(messages="message", thread=thread)
assert captured_kwargs.get("function_choice_behavior") is None
@@ -0,0 +1,34 @@
# Copyright (c) Microsoft. All rights reserved.
import pytest
from pydantic import Field, SecretStr, ValidationError
from semantic_kernel.kernel_pydantic import KernelBaseSettings
from semantic_kernel.utils.feature_stage_decorator import experimental
@experimental
class AzureAIAgentSettings(KernelBaseSettings):
"""Slightly modified to ensure invalid data raises ValidationError."""
env_prefix = "AZURE_AI_AGENT_"
model_deployment_name: str = Field(min_length=1)
project_connection_string: SecretStr = Field(..., min_length=1)
def test_azure_ai_agent_settings_valid():
settings = AzureAIAgentSettings(
model_deployment_name="test_model",
project_connection_string="secret_value",
)
assert settings.model_deployment_name == "test_model"
assert settings.project_connection_string.get_secret_value() == "secret_value"
def test_azure_ai_agent_settings_invalid():
with pytest.raises(ValidationError):
# Should fail due to min_length=1 constraints
AzureAIAgentSettings(
model_deployment_name="", # empty => invalid
project_connection_string="",
)
@@ -0,0 +1,51 @@
# Copyright (c) Microsoft. All rights reserved.
from azure.ai.agents.models import MessageAttachment, MessageRole
from semantic_kernel.agents.azure_ai.azure_ai_agent_utils import AzureAIAgentUtils
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.file_reference_content import FileReferenceContent
from semantic_kernel.contents.utils.author_role import AuthorRole
def test_azure_ai_agent_utils_get_thread_messages_none():
msgs = AzureAIAgentUtils.get_thread_messages([])
assert msgs is None
def test_azure_ai_agent_utils_get_thread_messages():
msg1 = ChatMessageContent(role=AuthorRole.USER, content="Hello!")
msg1.items.append(FileReferenceContent(file_id="file123"))
results = AzureAIAgentUtils.get_thread_messages([msg1])
assert len(results) == 1
assert results[0].content == "Hello!"
assert results[0].role == MessageRole.USER
assert len(results[0].attachments) == 1
assert isinstance(results[0].attachments[0], MessageAttachment)
def test_azure_ai_agent_utils_get_attachments_empty():
msg1 = ChatMessageContent(role=AuthorRole.USER, content="No file items")
atts = AzureAIAgentUtils.get_attachments(msg1)
assert atts == []
def test_azure_ai_agent_utils_get_attachments_file():
msg1 = ChatMessageContent(role=AuthorRole.USER, content="One file item")
msg1.items.append(FileReferenceContent(file_id="file123"))
atts = AzureAIAgentUtils.get_attachments(msg1)
assert len(atts) == 1
assert atts[0].file_id == "file123"
def test_azure_ai_agent_utils_get_metadata():
msg1 = ChatMessageContent(role=AuthorRole.USER, content="has meta", metadata={"k": 123})
meta = AzureAIAgentUtils.get_metadata(msg1)
assert meta["k"] == "123"
def test_azure_ai_agent_utils_get_tool_definition():
gen = AzureAIAgentUtils._get_tool_definition(["file_search", "code_interpreter", "non_existent"])
# file_search & code_interpreter exist, non_existent yields nothing
tools_list = list(gen)
assert len(tools_list) == 2
@@ -0,0 +1,88 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, patch
import pytest
from azure.ai.projects.aio import AIProjectClient
from semantic_kernel.agents.azure_ai.azure_ai_agent import AzureAIAgent
from semantic_kernel.agents.azure_ai.azure_ai_channel import AzureAIChannel
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions.agent_exceptions import AgentChatException
async def test_azure_ai_channel_invoke_invalid_agent():
channel = AzureAIChannel(AsyncMock(spec=AIProjectClient), "thread123")
with pytest.raises(AgentChatException):
async for _ in channel.invoke(object()):
pass
async def test_azure_ai_channel_invoke_valid_agent(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
async def fake_invoke(*args, **kwargs):
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke",
side_effect=fake_invoke,
):
channel = AzureAIChannel(ai_project_client, "thread123")
results = []
async for is_visible, msg in channel.invoke(agent):
results.append((is_visible, msg))
assert len(results) == 1
async def test_azure_ai_channel_invoke_stream_valid_agent(ai_project_client, ai_agent_definition):
agent = AzureAIAgent(client=ai_project_client, definition=ai_agent_definition)
async def fake_invoke(*args, **kwargs):
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.invoke_stream",
side_effect=fake_invoke,
):
channel = AzureAIChannel(ai_project_client, "thread123")
results = []
async for is_visible, msg in channel.invoke_stream(agent, messages=[]):
results.append((is_visible, msg))
assert len(results) == 1
async def test_azure_ai_channel_get_history():
# We need to return an async iterable, so let's do an AsyncMock returning an _async_gen
class FakeAgentClient:
delete_thread = AsyncMock()
# We'll patch get_messages directly below
class FakeClient:
agents = FakeAgentClient()
channel = AzureAIChannel(FakeClient(), "threadXYZ")
async def fake_get_messages(client, thread_id):
# Must produce an async iterable
yield ChatMessageContent(role=AuthorRole.ASSISTANT, content="Previous msg")
with patch(
"semantic_kernel.agents.azure_ai.agent_thread_actions.AgentThreadActions.get_messages",
new=fake_get_messages, # direct replacement with a coroutine
):
results = []
async for item in channel.get_history():
results.append(item)
assert len(results) == 1
assert results[0].content == "Previous msg"
# Helper for returning an async generator
async def _async_gen(items):
for i in items:
yield i
@@ -0,0 +1,187 @@
# Copyright (c) Microsoft. All rights reserved.
from collections.abc import Callable
import pytest
from semantic_kernel.agents.bedrock.models.bedrock_agent_event_type import BedrockAgentEventType
from semantic_kernel.agents.bedrock.models.bedrock_agent_model import BedrockAgentModel
from semantic_kernel.agents.bedrock.models.bedrock_agent_status import BedrockAgentStatus
from semantic_kernel.kernel import Kernel
@pytest.fixture()
def bedrock_agent_unit_test_env(monkeypatch, exclude_list, override_env_param_dict):
"""Fixture to set environment variables for Amazon Bedrock Agent unit tests."""
if exclude_list is None:
exclude_list = []
if override_env_param_dict is None:
override_env_param_dict = {}
env_vars = {
"BEDROCK_AGENT_AGENT_RESOURCE_ROLE_ARN": "TEST_BEDROCK_AGENT_AGENT_RESOURCE_ROLE_ARN",
"BEDROCK_AGENT_FOUNDATION_MODEL": "TEST_BEDROCK_AGENT_FOUNDATION_MODEL",
}
env_vars.update(override_env_param_dict)
for key, value in env_vars.items():
if key not in exclude_list:
monkeypatch.setenv(key, value)
else:
monkeypatch.delenv(key, raising=False)
return env_vars
@pytest.fixture
def kernel_with_function(kernel: Kernel, decorated_native_function: Callable) -> Kernel:
kernel.add_function("test_plugin", function=decorated_native_function)
return kernel
@pytest.fixture
def new_agent_name():
return "test_agent_name"
@pytest.fixture
def bedrock_agent_model():
return BedrockAgentModel(
agent_name="test_agent_name",
foundation_model="test_foundation_model",
agent_status=BedrockAgentStatus.NOT_PREPARED,
)
@pytest.fixture
def bedrock_agent_model_with_id():
return BedrockAgentModel(
agent_id="test_agent_id",
agent_name="test_agent_name",
foundation_model="test_foundation_model",
agent_status=BedrockAgentStatus.NOT_PREPARED,
)
@pytest.fixture
def bedrock_agent_model_with_id_prepared_dict():
return {
"agent": {
"agentId": "test_agent_id",
"agentName": "test_agent_name",
"foundationModel": "test_foundation_model",
"agentStatus": "PREPARED",
}
}
@pytest.fixture
def bedrock_agent_model_with_id_preparing_dict():
return {
"agent": {
"agentId": "test_agent_id",
"agentName": "test_agent_name",
"foundationModel": "test_foundation_model",
"agentStatus": "PREPARING",
}
}
@pytest.fixture
def bedrock_agent_model_with_id_not_prepared_dict():
return {
"agent": {
"agentId": "test_agent_id",
"agentName": "test_agent_name",
"foundationModel": "test_foundation_model",
"agentStatus": "NOT_PREPARED",
}
}
@pytest.fixture
def existing_agent_not_prepared_model():
return BedrockAgentModel(
agent_id="test_agent_id",
agent_name="test_agent_name",
foundation_model="test_foundation_model",
agent_status=BedrockAgentStatus.NOT_PREPARED,
)
@pytest.fixture
def bedrock_action_group_mode_dict():
return {
"agentActionGroup": {
"actionGroupId": "test_action_group_id",
"actionGroupName": "test_action_group_name",
}
}
@pytest.fixture
def simple_response():
return "test response"
@pytest.fixture
def bedrock_agent_non_streaming_empty_response():
return {
"completion": [],
}
@pytest.fixture
def bedrock_agent_non_streaming_simple_response(simple_response):
return {
"completion": [
{
"chunk": {"bytes": bytes(simple_response, "utf-8")},
},
],
}
@pytest.fixture
def bedrock_agent_streaming_simple_response(simple_response):
return {
"completion": [
{
"chunk": {"bytes": bytes(chunk, "utf-8")},
}
for chunk in simple_response
]
}
@pytest.fixture
def bedrock_agent_function_call_response():
return {
"completion": [
{
BedrockAgentEventType.RETURN_CONTROL: {
"invocationId": "test_invocation_id",
"invocationInputs": [
{
"functionInvocationInput": {
"function": "test_function",
"parameters": [
{"name": "test_parameter_name", "value": "test_parameter_value"},
],
},
},
],
},
},
],
}
@pytest.fixture
def bedrock_agent_create_session_response():
return {
"sessionId": "test_session_id",
}
@@ -0,0 +1,93 @@
# Copyright (c) Microsoft. All rights reserved.
import pytest
from semantic_kernel.agents.bedrock.action_group_utils import (
BEDROCK_FUNCTION_ALLOWED_PARAMETER_TYPES,
kernel_function_parameter_type_to_bedrock_function_parameter_type,
kernel_function_to_bedrock_function_schema,
parse_function_result_contents,
parse_return_control_payload,
)
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
from semantic_kernel.contents.function_result_content import FunctionResultContent
from semantic_kernel.kernel import Kernel
def test_kernel_function_to_bedrock_function_schema(kernel_with_function: Kernel):
# Test the conversion of kernel function to bedrock function schema
function_choice_behavior = FunctionChoiceBehavior.Auto()
function_choice_configuration = function_choice_behavior.get_config(kernel_with_function)
result = kernel_function_to_bedrock_function_schema(function_choice_configuration)
assert result == {
"functions": [
{
"name": "test_plugin-getLightStatus",
"parameters": {
"arg1": {
"type": "string",
"required": True,
}
},
"requireConfirmation": "DISABLED",
}
]
}
def test_kernel_function_parameter_type_to_bedrock_function_parameter_type():
# Test the conversion of kernel function parameter type to bedrock function parameter type
schema_data = {"type": "string"}
result = kernel_function_parameter_type_to_bedrock_function_parameter_type(schema_data)
assert result == "string"
def test_kernel_function_parameter_type_to_bedrock_function_parameter_type_invalid():
# Test the conversion of invalid kernel function parameter type to bedrock function parameter type
schema_data = {"type": "invalid_type"}
with pytest.raises(
ValueError,
match="Type invalid_type is not allowed in bedrock function parameter type. "
f"Allowed types are {BEDROCK_FUNCTION_ALLOWED_PARAMETER_TYPES}.",
):
kernel_function_parameter_type_to_bedrock_function_parameter_type(schema_data)
def test_parse_return_control_payload():
# Test the parsing of return control payload to function call contents
return_control_payload = {
"invocationId": "test_invocation_id",
"invocationInputs": [
{
"functionInvocationInput": {
"function": "test_function",
"parameters": [
{"name": "param1", "value": "value1"},
{"name": "param2", "value": "value2"},
],
}
}
],
}
result = parse_return_control_payload(return_control_payload)
assert len(result) == 1
assert result[0].id == "test_invocation_id"
assert result[0].name == "test_function"
assert result[0].arguments == {"param1": "value1", "param2": "value2"}
def test_parse_function_result_contents():
# Test the parsing of function result contents to be returned to the agent
function_result_contents = [
FunctionResultContent(
id="test_id",
name="test_function",
result="test_result",
metadata={"functionInvocationInput": {"actionGroup": "test_action_group"}},
)
]
result = parse_function_result_contents(function_result_contents)
assert len(result) == 1
assert result[0]["functionResult"]["actionGroup"] == "test_action_group"
assert result[0]["functionResult"]["function"] == "test_function"
assert result[0]["functionResult"]["responseBody"]["TEXT"]["body"] == "test_result"
@@ -0,0 +1,33 @@
# Copyright (c) Microsoft. All rights reserved.
import pytest
from pydantic import ValidationError
from semantic_kernel.agents.bedrock.models.bedrock_action_group_model import BedrockActionGroupModel
def test_bedrock_action_group_model_valid():
"""Test case to verify the BedrockActionGroupModel with valid data."""
model = BedrockActionGroupModel(actionGroupId="test_id", actionGroupName="test_name")
assert model.action_group_id == "test_id"
assert model.action_group_name == "test_name"
def test_bedrock_action_group_model_missing_action_group_id():
"""Test case to verify error handling when actionGroupId is missing."""
with pytest.raises(ValidationError):
BedrockActionGroupModel(actionGroupName="test_name")
def test_bedrock_action_group_model_missing_action_group_name():
"""Test case to verify error handling when actionGroupName is missing."""
with pytest.raises(ValidationError):
BedrockActionGroupModel(actionGroupId="test_id")
def test_bedrock_action_group_model_extra_field():
"""Test case to verify the BedrockActionGroupModel with an extra field."""
model = BedrockActionGroupModel(actionGroupId="test_id", actionGroupName="test_name", extraField="extra_value")
assert model.action_group_id == "test_id"
assert model.action_group_name == "test_name"
assert model.extraField == "extra_value"
@@ -0,0 +1,751 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, Mock, PropertyMock, patch
import boto3
import pytest
from semantic_kernel.agents.bedrock.action_group_utils import (
kernel_function_to_bedrock_function_schema,
parse_function_result_contents,
)
from semantic_kernel.agents.bedrock.bedrock_agent import BedrockAgent, BedrockAgentThread
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
from semantic_kernel.contents.function_result_content import FunctionResultContent
from semantic_kernel.exceptions.agent_exceptions import AgentInitializationException, AgentInvokeException
from semantic_kernel.functions.kernel_arguments import KernelArguments
from semantic_kernel.kernel import Kernel
# region Agent Initialization Tests
# Test case to verify BedrockAgent initialization
@patch.object(boto3, "client", return_value=Mock())
async def test_bedrock_agent_initialization(client, bedrock_agent_model_with_id):
agent = BedrockAgent(bedrock_agent_model_with_id)
assert agent.name == bedrock_agent_model_with_id.agent_name
assert agent.agent_model.agent_name == bedrock_agent_model_with_id.agent_name
assert agent.agent_model.agent_id == bedrock_agent_model_with_id.agent_id
assert agent.agent_model.foundation_model == bedrock_agent_model_with_id.foundation_model
# Test case to verify error handling during BedrockAgent initialization with non-auto function choice
@patch.object(boto3, "client", return_value=Mock())
async def test_bedrock_agent_initialization_error_with_non_auto_function_choice(client, bedrock_agent_model_with_id):
with pytest.raises(ValueError, match="Only FunctionChoiceType.AUTO is supported."):
BedrockAgent(
bedrock_agent_model_with_id,
function_choice_behavior=FunctionChoiceBehavior.NoneInvoke(),
)
# Test case to verify the creation of BedrockAgent
@patch.object(boto3, "client", return_value=Mock())
@pytest.mark.parametrize(
"kernel, function_choice_behavior, arguments",
[
(None, None, None),
(Kernel(), None, None),
(Kernel(), FunctionChoiceBehavior.Auto(), None),
(Kernel(), FunctionChoiceBehavior.Auto(), KernelArguments()),
],
)
async def test_bedrock_agent_create_and_prepare_agent(
client,
bedrock_agent_model_with_id_not_prepared_dict,
bedrock_agent_unit_test_env,
kernel,
function_choice_behavior,
arguments,
):
with (
patch.object(client, "create_agent") as mock_create_agent,
patch.object(BedrockAgent, "_wait_for_agent_status", new_callable=AsyncMock),
patch.object(BedrockAgent, "prepare_agent_and_wait_until_prepared", new_callable=AsyncMock),
):
mock_create_agent.return_value = bedrock_agent_model_with_id_not_prepared_dict
agent = await BedrockAgent.create_and_prepare_agent(
name=bedrock_agent_model_with_id_not_prepared_dict["agent"]["agentName"],
instructions="test_instructions",
bedrock_client=client,
env_file_path="fake_path",
kernel=kernel,
function_choice_behavior=function_choice_behavior,
arguments=arguments,
)
mock_create_agent.assert_called_once_with(
agentName=bedrock_agent_model_with_id_not_prepared_dict["agent"]["agentName"],
foundationModel=bedrock_agent_unit_test_env["BEDROCK_AGENT_FOUNDATION_MODEL"],
agentResourceRoleArn=bedrock_agent_unit_test_env["BEDROCK_AGENT_AGENT_RESOURCE_ROLE_ARN"],
instruction="test_instructions",
)
assert agent.agent_model.agent_id == bedrock_agent_model_with_id_not_prepared_dict["agent"]["agentId"]
assert agent.id == bedrock_agent_model_with_id_not_prepared_dict["agent"]["agentId"]
assert agent.agent_model.agent_name == bedrock_agent_model_with_id_not_prepared_dict["agent"]["agentName"]
assert agent.name == bedrock_agent_model_with_id_not_prepared_dict["agent"]["agentName"]
assert (
agent.agent_model.foundation_model
== bedrock_agent_model_with_id_not_prepared_dict["agent"]["foundationModel"]
)
assert agent.kernel is not None
assert agent.function_choice_behavior is not None
if arguments:
assert agent.arguments is not None
# Test case to verify the creation of BedrockAgent
@pytest.mark.parametrize(
"exclude_list",
[
["BEDROCK_AGENT_AGENT_RESOURCE_ROLE_ARN"],
["BEDROCK_AGENT_FOUNDATION_MODEL"],
],
indirect=True,
)
@patch.object(boto3, "client", return_value=Mock())
async def test_bedrock_agent_create_and_prepare_agent_settings_validation_error(
client,
bedrock_agent_model_with_id_not_prepared_dict,
bedrock_agent_unit_test_env,
):
with pytest.raises(AgentInitializationException):
await BedrockAgent.create_and_prepare_agent(
name=bedrock_agent_model_with_id_not_prepared_dict["agent"]["agentName"],
instructions="test_instructions",
env_file_path="fake_path",
)
# Test case to verify the creation of BedrockAgent
@patch.object(boto3, "client", return_value=Mock())
async def test_bedrock_agent_create_and_prepare_agent_service_exception(
client,
bedrock_agent_model_with_id_not_prepared_dict,
bedrock_agent_unit_test_env,
):
with (
patch.object(client, "create_agent") as mock_create_agent,
patch.object(BedrockAgent, "prepare_agent_and_wait_until_prepared", new_callable=AsyncMock),
):
from botocore.exceptions import ClientError
mock_create_agent.side_effect = ClientError({}, "create_agent")
with pytest.raises(AgentInitializationException):
await BedrockAgent.create_and_prepare_agent(
name=bedrock_agent_model_with_id_not_prepared_dict["agent"]["agentName"],
instructions="test_instructions",
bedrock_client=client,
env_file_path="fake_path",
)
@patch.object(boto3, "client", return_value=Mock())
async def test_bedrock_agent_prepare_agent_and_wait_until_prepared(
client,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
bedrock_agent_model_with_id_preparing_dict,
bedrock_agent_model_with_id_prepared_dict,
):
agent = BedrockAgent(bedrock_agent_model_with_id, bedrock_client=client)
with (
patch.object(client, "get_agent") as mock_get_agent,
patch.object(client, "prepare_agent") as mock_prepare_agent,
):
mock_get_agent.side_effect = [
bedrock_agent_model_with_id_preparing_dict,
bedrock_agent_model_with_id_prepared_dict,
]
await agent.prepare_agent_and_wait_until_prepared()
mock_prepare_agent.assert_called_once_with(agentId=bedrock_agent_model_with_id.agent_id)
assert mock_get_agent.call_count == 2
assert agent.agent_model.agent_status == "PREPARED"
@patch.object(boto3, "client", return_value=Mock())
async def test_bedrock_agent_prepare_agent_and_wait_until_prepared_fail(
client,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
bedrock_agent_model_with_id_preparing_dict,
):
agent = BedrockAgent(bedrock_agent_model_with_id, bedrock_client=client)
with (
patch.object(client, "get_agent") as mock_get_agent,
patch.object(client, "prepare_agent"),
):
mock_get_agent.side_effect = [
bedrock_agent_model_with_id_preparing_dict,
bedrock_agent_model_with_id_preparing_dict,
bedrock_agent_model_with_id_preparing_dict,
bedrock_agent_model_with_id_preparing_dict,
bedrock_agent_model_with_id_preparing_dict,
bedrock_agent_model_with_id_preparing_dict,
]
with pytest.raises(TimeoutError):
await agent.prepare_agent_and_wait_until_prepared()
# Test case to verify the creation of a code interpreter action group
@patch.object(boto3, "client", return_value=Mock())
async def test_create_code_interpreter_action_group(
client,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
bedrock_action_group_mode_dict,
):
agent = BedrockAgent(bedrock_agent_model_with_id, bedrock_client=client)
with (
patch.object(client, "create_agent_action_group") as mock_create_action_group,
patch.object(
BedrockAgent, "prepare_agent_and_wait_until_prepared"
) as mock_prepare_agent_and_wait_until_prepared,
):
mock_create_action_group.return_value = bedrock_action_group_mode_dict
action_group_model = await agent.create_code_interpreter_action_group()
mock_create_action_group.assert_called_once_with(
agentId=agent.agent_model.agent_id,
agentVersion=agent.agent_model.agent_version or "DRAFT",
actionGroupName=f"{agent.agent_model.agent_name}_code_interpreter",
actionGroupState="ENABLED",
parentActionGroupSignature="AMAZON.CodeInterpreter",
)
assert action_group_model.action_group_id == bedrock_action_group_mode_dict["agentActionGroup"]["actionGroupId"]
mock_prepare_agent_and_wait_until_prepared.assert_called_once()
# Test case to verify the creation of BedrockAgent with plugins
@patch.object(boto3, "client", return_value=Mock())
async def test_bedrock_agent_create_with_plugin_via_constructor(
client,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
custom_plugin_class,
):
agent = BedrockAgent(
bedrock_agent_model_with_id,
plugins=[custom_plugin_class()],
bedrock_client=client,
)
assert agent.kernel.plugins is not None
assert len(agent.kernel.plugins) == 1
# Test case to verify the creation of a user input action group
@patch.object(boto3, "client", return_value=Mock())
async def test_create_user_input_action_group(
client,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
bedrock_action_group_mode_dict,
):
agent = BedrockAgent(bedrock_agent_model_with_id, bedrock_client=client)
with (
patch.object(agent.bedrock_client, "create_agent_action_group") as mock_create_action_group,
patch.object(
BedrockAgent, "prepare_agent_and_wait_until_prepared"
) as mock_prepare_agent_and_wait_until_prepared,
):
mock_create_action_group.return_value = bedrock_action_group_mode_dict
action_group_model = await agent.create_user_input_action_group()
mock_create_action_group.assert_called_once_with(
agentId=agent.agent_model.agent_id,
agentVersion=agent.agent_model.agent_version or "DRAFT",
actionGroupName=f"{agent.agent_model.agent_name}_user_input",
actionGroupState="ENABLED",
parentActionGroupSignature="AMAZON.UserInput",
)
assert action_group_model.action_group_id == bedrock_action_group_mode_dict["agentActionGroup"]["actionGroupId"]
mock_prepare_agent_and_wait_until_prepared.assert_called_once()
# Test case to verify the creation of a kernel function action group
@patch.object(boto3, "client", return_value=Mock())
async def test_create_kernel_function_action_group(
client,
kernel_with_function,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
bedrock_action_group_mode_dict,
):
agent = BedrockAgent(bedrock_agent_model_with_id, kernel=kernel_with_function, bedrock_client=client)
with (
patch.object(agent.bedrock_client, "create_agent_action_group") as mock_create_action_group,
patch.object(
BedrockAgent, "prepare_agent_and_wait_until_prepared"
) as mock_prepare_agent_and_wait_until_prepared,
):
mock_create_action_group.return_value = bedrock_action_group_mode_dict
action_group_model = await agent.create_kernel_function_action_group()
mock_create_action_group.assert_called_once_with(
agentId=agent.agent_model.agent_id,
agentVersion=agent.agent_model.agent_version or "DRAFT",
actionGroupName=f"{agent.agent_model.agent_name}_kernel_function",
actionGroupState="ENABLED",
actionGroupExecutor={"customControl": "RETURN_CONTROL"},
functionSchema=kernel_function_to_bedrock_function_schema(
agent.function_choice_behavior.get_config(kernel_with_function)
),
)
assert action_group_model.action_group_id == bedrock_action_group_mode_dict["agentActionGroup"]["actionGroupId"]
mock_prepare_agent_and_wait_until_prepared.assert_called_once()
# Test case to verify the association of an agent with a knowledge base
@patch.object(boto3, "client", return_value=Mock())
async def test_associate_agent_knowledge_base(
client,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
):
agent = BedrockAgent(bedrock_agent_model_with_id, bedrock_client=client)
with (
patch.object(agent.bedrock_client, "associate_agent_knowledge_base") as mock_associate_knowledge_base,
patch.object(
BedrockAgent, "prepare_agent_and_wait_until_prepared"
) as mock_prepare_agent_and_wait_until_prepared,
):
await agent.associate_agent_knowledge_base("test_knowledge_base_id")
mock_associate_knowledge_base.assert_called_once_with(
agentId=agent.agent_model.agent_id,
agentVersion=agent.agent_model.agent_version,
knowledgeBaseId="test_knowledge_base_id",
)
mock_prepare_agent_and_wait_until_prepared.assert_called_once()
# Test case to verify the disassociation of an agent with a knowledge base
@patch.object(boto3, "client", return_value=Mock())
async def test_disassociate_agent_knowledge_base(
client,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
):
agent = BedrockAgent(bedrock_agent_model_with_id, bedrock_client=client)
with (
patch.object(agent.bedrock_client, "disassociate_agent_knowledge_base") as mock_disassociate_knowledge_base,
patch.object(
BedrockAgent, "prepare_agent_and_wait_until_prepared"
) as mock_prepare_agent_and_wait_until_prepared,
):
await agent.disassociate_agent_knowledge_base("test_knowledge_base_id")
mock_disassociate_knowledge_base.assert_called_once_with(
agentId=agent.agent_model.agent_id,
agentVersion=agent.agent_model.agent_version,
knowledgeBaseId="test_knowledge_base_id",
)
mock_prepare_agent_and_wait_until_prepared.assert_called_once()
# Test case to verify listing associated knowledge bases with an agent
@patch.object(boto3, "client", return_value=Mock())
async def test_list_associated_agent_knowledge_bases(
client,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
):
agent = BedrockAgent(bedrock_agent_model_with_id, bedrock_client=client)
with patch.object(agent.bedrock_client, "list_agent_knowledge_bases") as mock_list_knowledge_bases:
await agent.list_associated_agent_knowledge_bases()
mock_list_knowledge_bases.assert_called_once_with(
agentId=agent.agent_model.agent_id,
agentVersion=agent.agent_model.agent_version,
)
# endregion
# region Agent Deletion Tests
@patch.object(boto3, "client", return_value=Mock())
async def test_delete_agent(
client,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
):
agent = BedrockAgent(bedrock_agent_model_with_id, bedrock_client=client)
agent_id = bedrock_agent_model_with_id.agent_id
with patch.object(agent.bedrock_client, "delete_agent") as mock_delete_agent:
await agent.delete_agent()
mock_delete_agent.assert_called_once_with(agentId=agent_id)
assert agent.agent_model.agent_id is None
# Test case to verify error handling when deleting an agent that does not exist
@patch.object(boto3, "client", return_value=Mock())
async def test_delete_agent_twice_error(
client,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
):
agent = BedrockAgent(bedrock_agent_model_with_id, bedrock_client=client)
with patch.object(agent.bedrock_client, "delete_agent"):
await agent.delete_agent()
with pytest.raises(ValueError):
await agent.delete_agent()
# Test case to verify error handling when there is a client error during agent deletion
@patch.object(boto3, "client", return_value=Mock())
async def test_delete_agent_client_error(
client,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
):
agent = BedrockAgent(bedrock_agent_model_with_id, bedrock_client=client)
with patch.object(agent.bedrock_client, "delete_agent") as mock_delete_agent:
from botocore.exceptions import ClientError
mock_delete_agent.side_effect = ClientError({"Error": {"Code": "500"}}, "delete_agent")
with pytest.raises(ClientError):
await agent.delete_agent()
# endregion
# region Agent Invoke Tests
# Test case to verify the `get_response` method of BedrockAgent
@pytest.mark.parametrize(
"thread",
[
None,
BedrockAgentThread(None, session_id="test_session_id"),
],
)
@patch.object(boto3, "client", return_value=Mock())
async def test_bedrock_agent_get_response(
client,
thread,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
bedrock_agent_non_streaming_simple_response,
simple_response,
):
with (
patch.object(BedrockAgent, "_invoke_agent", new_callable=AsyncMock) as mock_invoke_agent,
patch.object(BedrockAgentThread, "create", new_callable=AsyncMock) as mock_start,
patch(
"semantic_kernel.agents.bedrock.bedrock_agent.BedrockAgentThread.id",
new_callable=PropertyMock,
) as mock_id,
):
agent = BedrockAgent(bedrock_agent_model_with_id)
mock_id.return_value = "mock-thread-id"
mock_invoke_agent.return_value = bedrock_agent_non_streaming_simple_response
mock_start.return_value = "test_session_id"
response = await agent.get_response(messages="test_input_text", thread=thread)
assert response.message.content == simple_response
mock_invoke_agent.assert_called_once()
# Test case to verify the `get_response` method of BedrockAgent
@patch.object(boto3, "client", return_value=Mock())
async def test_bedrock_agent_get_response_exception(
client,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
bedrock_agent_non_streaming_empty_response,
):
with (
patch.object(BedrockAgent, "_invoke_agent", new_callable=AsyncMock) as mock_invoke_agent,
patch.object(BedrockAgentThread, "create", new_callable=AsyncMock) as mock_start,
patch(
"semantic_kernel.agents.bedrock.bedrock_agent.BedrockAgentThread.id",
new_callable=PropertyMock,
) as mock_id,
):
agent = BedrockAgent(bedrock_agent_model_with_id)
mock_id.return_value = "mock-thread-id"
mock_invoke_agent.return_value = bedrock_agent_non_streaming_empty_response
mock_start.return_value = "test_session_id"
with pytest.raises(AgentInvokeException):
await agent.get_response(messages="test_input_text")
# Test case to verify the invocation of BedrockAgent
@pytest.mark.parametrize(
"thread",
[
None,
BedrockAgentThread(None, session_id="test_session_id"),
],
)
@patch.object(boto3, "client", return_value=Mock())
async def test_bedrock_agent_invoke(
client,
thread,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
bedrock_agent_non_streaming_simple_response,
simple_response,
):
with (
patch.object(BedrockAgent, "_invoke_agent", new_callable=AsyncMock) as mock_invoke_agent,
patch.object(BedrockAgentThread, "create", new_callable=AsyncMock) as mock_start,
patch.object(BedrockAgentThread, "id", "test_session_id"),
):
agent = BedrockAgent(bedrock_agent_model_with_id)
mock_invoke_agent.return_value = bedrock_agent_non_streaming_simple_response
mock_start.return_value = "test_session_id"
async for response in agent.invoke(messages="test_input_text", thread=thread):
assert response.message.content == simple_response
mock_invoke_agent.assert_called_once_with(
"test_session_id",
"test_input_text",
None,
streamingConfigurations={"streamFinalResponse": False},
sessionState={},
)
# Test case to verify the streaming invocation of BedrockAgent
@pytest.mark.parametrize(
"thread",
[
None,
BedrockAgentThread(None, session_id="test_session_id"),
],
)
@patch.object(boto3, "client", return_value=Mock())
async def test_bedrock_agent_invoke_stream(
client,
thread,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
bedrock_agent_streaming_simple_response,
simple_response,
):
with (
patch.object(BedrockAgent, "_invoke_agent", new_callable=AsyncMock) as mock_invoke_agent,
patch.object(BedrockAgentThread, "create", new_callable=AsyncMock) as mock_start,
patch.object(BedrockAgentThread, "id", "test_session_id"),
):
agent = BedrockAgent(bedrock_agent_model_with_id)
mock_invoke_agent.return_value = bedrock_agent_streaming_simple_response
mock_start.return_value = "test_session_id"
full_message = ""
async for response in agent.invoke_stream(messages="test_input_text", thread=thread):
full_message += response.message.content
assert full_message == simple_response
mock_invoke_agent.assert_called_once_with(
"test_session_id",
"test_input_text",
None,
streamingConfigurations={"streamFinalResponse": True},
sessionState={},
)
# Test case to verify the invocation of BedrockAgent with function call
@patch.object(boto3, "client", return_value=Mock())
async def test_bedrock_agent_invoke_with_function_call(
client,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
bedrock_agent_function_call_response,
bedrock_agent_non_streaming_simple_response,
):
with (
patch.object(BedrockAgent, "_invoke_agent", new_callable=AsyncMock) as mock_invoke_agent,
patch.object(BedrockAgent, "_handle_function_call_contents") as mock_handle_function_call_contents,
patch.object(BedrockAgentThread, "create", new_callable=AsyncMock) as mock_start,
patch.object(BedrockAgentThread, "id", "test_session_id"),
):
agent = BedrockAgent(bedrock_agent_model_with_id)
function_result_contents = [
FunctionResultContent(
id="test_id",
name="test_function",
result="test_result",
metadata={"functionInvocationInput": {"actionGroup": "test_action_group"}},
)
]
mock_handle_function_call_contents.return_value = function_result_contents
agent.function_choice_behavior.maximum_auto_invoke_attempts = 2
mock_invoke_agent.side_effect = [
bedrock_agent_function_call_response,
bedrock_agent_non_streaming_simple_response,
]
mock_start.return_value = "test_session_id"
async for _ in agent.invoke(messages="test_input_text"):
mock_invoke_agent.assert_called_with(
"test_session_id",
"test_input_text",
None,
streamingConfigurations={"streamFinalResponse": False},
sessionState={
"invocationId": "test_invocation_id",
"returnControlInvocationResults": parse_function_result_contents(function_result_contents),
},
)
# Test case to verify the streaming invocation of BedrockAgent with function call
@patch.object(boto3, "client", return_value=Mock())
async def test_bedrock_agent_invoke_stream_with_function_call(
client,
bedrock_agent_unit_test_env,
bedrock_agent_model_with_id,
bedrock_agent_function_call_response,
bedrock_agent_streaming_simple_response,
):
with (
patch.object(BedrockAgent, "_invoke_agent", new_callable=AsyncMock) as mock_invoke_agent,
patch.object(BedrockAgent, "_handle_function_call_contents") as mock_handle_function_call_contents,
patch.object(BedrockAgentThread, "create", new_callable=AsyncMock) as mock_start,
patch.object(BedrockAgentThread, "id", "test_session_id"),
):
agent = BedrockAgent(bedrock_agent_model_with_id)
function_result_contents = [
FunctionResultContent(
id="test_id",
name="test_function",
result="test_result",
metadata={"functionInvocationInput": {"actionGroup": "test_action_group"}},
)
]
mock_handle_function_call_contents.return_value = function_result_contents
agent.function_choice_behavior.maximum_auto_invoke_attempts = 2
mock_invoke_agent.side_effect = [
bedrock_agent_function_call_response,
bedrock_agent_streaming_simple_response,
]
mock_start.return_value = "test_session_id"
async for _ in agent.invoke_stream(messages="test_input_text"):
mock_invoke_agent.assert_called_with(
"test_session_id",
"test_input_text",
None,
streamingConfigurations={"streamFinalResponse": True},
sessionState={
"invocationId": "test_invocation_id",
"returnControlInvocationResults": parse_function_result_contents(function_result_contents),
},
)
# endregion
# region Filename Sanitization Tests
def test_sanitize_filename_simple():
"""Test _sanitize_filename with a simple filename."""
assert BedrockAgent._sanitize_filename("file.txt") == "file.txt"
def test_sanitize_filename_with_spaces():
"""Test _sanitize_filename with spaces in filename."""
assert BedrockAgent._sanitize_filename("my file.txt") == "my file.txt"
def test_sanitize_filename_directory_traversal_unix():
"""Test _sanitize_filename strips Unix-style directory traversal."""
assert BedrockAgent._sanitize_filename("../../../etc/passwd") == "passwd"
assert BedrockAgent._sanitize_filename("../../file.txt") == "file.txt"
assert BedrockAgent._sanitize_filename("/etc/passwd") == "passwd"
def test_sanitize_filename_directory_traversal_windows():
"""Test _sanitize_filename strips Windows-style directory traversal."""
assert BedrockAgent._sanitize_filename("..\\..\\..\\Windows\\System32\\config") == "config"
assert BedrockAgent._sanitize_filename("C:\\Users\\file.txt") == "file.txt"
assert BedrockAgent._sanitize_filename("\\\\server\\share\\file.txt") == "file.txt"
def test_sanitize_filename_mixed_separators():
"""Test _sanitize_filename with mixed path separators."""
assert BedrockAgent._sanitize_filename("../path\\to/file.txt") == "file.txt"
assert BedrockAgent._sanitize_filename("..\\path/to\\file.txt") == "file.txt"
def test_sanitize_filename_null_byte():
"""Test _sanitize_filename removes null bytes."""
assert BedrockAgent._sanitize_filename("file\x00.txt") == "file.txt"
assert BedrockAgent._sanitize_filename("file.txt\x00.exe") == "file.txt.exe"
def test_sanitize_filename_empty():
"""Test _sanitize_filename returns empty string for empty result."""
assert BedrockAgent._sanitize_filename("") == ""
assert BedrockAgent._sanitize_filename("../") == ""
assert BedrockAgent._sanitize_filename("..\\") == ""
def test_sanitize_filename_only_dots():
"""Test _sanitize_filename handles edge cases with dots."""
# Note: os.path.basename("..") returns ".." which is kept as-is
# Only "../" or "..\" patterns get stripped to empty string
assert BedrockAgent._sanitize_filename(".") == "."
def test_sanitize_filename_logs_warning(caplog):
"""Test _sanitize_filename logs warning when filename is sanitized."""
import logging
with caplog.at_level(logging.WARNING):
result = BedrockAgent._sanitize_filename("../malicious/file.txt")
assert result == "file.txt"
assert "potentially malicious path components" in caplog.text
assert "../malicious/file.txt" in caplog.text
assert "file.txt" in caplog.text
def test_sanitize_filename_no_warning_for_clean_filename(caplog):
"""Test _sanitize_filename does not log warning for clean filenames."""
import logging
with caplog.at_level(logging.WARNING):
result = BedrockAgent._sanitize_filename("clean_file.txt")
assert result == "clean_file.txt"
assert "potentially malicious" not in caplog.text
# endregion
@@ -0,0 +1,331 @@
# Copyright (c) Microsoft. All rights reserved.
from collections.abc import AsyncIterable
from unittest.mock import MagicMock, Mock, patch
import boto3
import pytest
from semantic_kernel.agents.agent import Agent, AgentResponseItem, AgentThread
from semantic_kernel.agents.bedrock.models.bedrock_agent_model import BedrockAgentModel
from semantic_kernel.agents.channels.bedrock_agent_channel import BedrockAgentChannel
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions.agent_exceptions import AgentChatException
@pytest.fixture
@patch.object(boto3, "client", return_value=Mock())
def mock_channel(client):
from semantic_kernel.agents.bedrock.bedrock_agent import BedrockAgentThread
BedrockAgentChannel.model_rebuild()
thread = BedrockAgentThread(client, session_id="test_session_id")
return BedrockAgentChannel(thread=thread)
class ConcreteAgent(Agent):
async def get_response(self, *args, **kwargs) -> ChatMessageContent:
raise NotImplementedError
def invoke(self, *args, **kwargs) -> AsyncIterable[ChatMessageContent]:
raise NotImplementedError
def invoke_stream(self, *args, **kwargs) -> AsyncIterable[StreamingChatMessageContent]:
raise NotImplementedError
@pytest.fixture
def chat_history() -> list[ChatMessageContent]:
return [
ChatMessageContent(role="user", content="Hello, Bedrock!"),
ChatMessageContent(role="assistant", content="Hello, User!"),
ChatMessageContent(role="user", content="How are you?"),
ChatMessageContent(role="assistant", content="I'm good, thank you!"),
]
@pytest.fixture
def chat_history_not_alternate_role() -> list[ChatMessageContent]:
return [
ChatMessageContent(role="user", content="Hello, Bedrock!"),
ChatMessageContent(role="user", content="Hello, User!"),
ChatMessageContent(role="assistant", content="How are you?"),
ChatMessageContent(role="assistant", content="I'm good, thank you!"),
]
@pytest.fixture
def mock_agent():
"""
Fixture that creates a mock BedrockAgent.
"""
from semantic_kernel.agents.bedrock.bedrock_agent import BedrockAgent
# Create mocks
mock_agent = MagicMock(spec=BedrockAgent)
# Set the name and agent_model properties
mock_agent.name = "MockBedrockAgent"
mock_agent.agent_model = MagicMock(spec=BedrockAgentModel)
mock_agent.agent_model.foundation_model = "mock-foundation-model"
return mock_agent
async def test_receive_message(mock_channel, chat_history):
# Test to verify the receive_message functionality
await mock_channel.receive(chat_history)
assert len(mock_channel) == len(chat_history)
async def test_channel_receive_message_with_no_message(mock_channel):
# Test to verify receive_message when no message is received
await mock_channel.receive([])
assert len(mock_channel) == 0
async def test_chat_history_alternation(mock_channel, chat_history_not_alternate_role):
# Test to verify chat history alternates between user and assistant messages
await mock_channel.receive(chat_history_not_alternate_role)
assert all(
mock_channel.messages[i].role != mock_channel.messages[i + 1].role
for i in range(len(chat_history_not_alternate_role) - 1)
)
assert mock_channel.messages[1].content == mock_channel.MESSAGE_PLACEHOLDER
assert mock_channel.messages[4].content == mock_channel.MESSAGE_PLACEHOLDER
async def test_channel_reset(mock_channel, chat_history):
# Test to verify the reset functionality
await mock_channel.receive(chat_history)
assert len(mock_channel) == len(chat_history)
assert len(mock_channel.messages) == len(chat_history)
await mock_channel.reset()
assert len(mock_channel) == 0
assert len(mock_channel.messages) == 0
async def test_receive_appends_history_correctly(mock_channel):
"""Test that the receive method appends messages while ensuring they alternate in author role."""
# Provide a list of messages with identical roles to see if placeholders are inserted
incoming_messages = [
ChatMessageContent(role=AuthorRole.USER, content="User message 1"),
ChatMessageContent(role=AuthorRole.USER, content="User message 2"),
ChatMessageContent(role=AuthorRole.ASSISTANT, content="Assistant message 1"),
ChatMessageContent(role=AuthorRole.ASSISTANT, content="Assistant message 2"),
]
await mock_channel.receive(incoming_messages)
# The final channel.messages should be:
# user message 1, user placeholder, user message 2, assistant placeholder, assistant message 1,
# assistant placeholder, assistant message 2
expected_roles = [
AuthorRole.USER,
AuthorRole.ASSISTANT, # placeholder
AuthorRole.USER,
AuthorRole.ASSISTANT,
AuthorRole.USER, # placeholder
AuthorRole.ASSISTANT,
]
expected_contents = [
"User message 1",
BedrockAgentChannel.MESSAGE_PLACEHOLDER,
"User message 2",
"Assistant message 1",
BedrockAgentChannel.MESSAGE_PLACEHOLDER,
"Assistant message 2",
]
assert len(mock_channel.messages) == len(expected_roles)
for i, (msg, exp_role, exp_content) in enumerate(zip(mock_channel.messages, expected_roles, expected_contents)):
assert msg.role == exp_role, f"Role mismatch at index {i}"
assert msg.content == exp_content, f"Content mismatch at index {i}"
async def test_invoke_raises_exception_for_non_bedrock_agent(mock_channel):
"""Test invoke method raises AgentChatException if the agent provided is not a BedrockAgent."""
# Place a message in the channel so it's not empty
mock_channel.messages.append(ChatMessageContent(role=AuthorRole.USER, content="User message"))
# Create a dummy agent that is not BedrockAgent
non_bedrock_agent = ConcreteAgent()
with pytest.raises(AgentChatException) as exc_info:
_ = [msg async for msg in mock_channel.invoke(non_bedrock_agent)]
assert "Agent is not of the expected type" in str(exc_info.value)
async def test_invoke_raises_exception_if_no_history(mock_channel, mock_agent):
"""Test invoke method raises AgentChatException if no chat history is available."""
with pytest.raises(AgentChatException) as exc_info:
_ = [msg async for msg in mock_channel.invoke(mock_agent)]
assert "No chat history available" in str(exc_info.value)
async def test_invoke_inserts_placeholders_when_history_needs_to_alternate(mock_channel, mock_agent):
"""Test invoke ensures _ensure_history_alternates and _ensure_last_message_is_user are called."""
# Put messages in the channel such that the last message is an assistant's
mock_channel.messages.append(ChatMessageContent(role=AuthorRole.ASSISTANT, content="Assistant 1"))
# Mock agent.invoke to return an async generator
async def mock_invoke(messages: str, thread: AgentThread, sessionState=None, **kwargs):
# We just yield one message as if the agent responded
yield AgentResponseItem(
message=ChatMessageContent(role=AuthorRole.ASSISTANT, content="Mock Agent Response"),
thread=mock_channel.thread,
)
mock_agent.invoke = mock_invoke
# Because the last message is from the assistant, we expect a placeholder user message to be appended
# also the history might need to alternate.
# But since there's only one message, there's nothing to fix except the last message is user.
# We will now add a user message so we do not get the "No chat history available" error
mock_channel.messages.append(ChatMessageContent(role=AuthorRole.USER, content="User 1"))
# Now we do invoke
outputs = [msg async for msg in mock_channel.invoke(mock_agent)]
# We'll check if the response is appended to channel.messages
assert len(outputs) == 1
assert outputs[0][0] is True, "Expected a user-facing response"
agent_response = outputs[0][1]
assert agent_response.content == "Mock Agent Response"
# The channel messages should now have 3 messages: the assistant, the user, and the new agent message
assert len(mock_channel.messages) == 3
assert mock_channel.messages[-1].role == AuthorRole.ASSISTANT
assert mock_channel.messages[-1].content == "Mock Agent Response"
async def test_invoke_stream_raises_error_for_non_bedrock_agent(mock_channel):
"""Test invoke_stream raises AgentChatException if the agent provided is not a BedrockAgent."""
mock_channel.messages.append(ChatMessageContent(role=AuthorRole.USER, content="User message"))
non_bedrock_agent = ConcreteAgent()
with pytest.raises(AgentChatException) as exc_info:
_ = [msg async for msg in mock_channel.invoke_stream(non_bedrock_agent, [])]
assert "Agent is not of the expected type" in str(exc_info.value)
async def test_invoke_stream_raises_no_chat_history(mock_channel, mock_agent):
"""Test invoke_stream raises AgentChatException if no messages in the channel."""
with pytest.raises(AgentChatException) as exc_info:
_ = [msg async for msg in mock_channel.invoke_stream(mock_agent, [])]
assert "No chat history available." in str(exc_info.value)
async def test_invoke_stream_appends_response_message(mock_channel, mock_agent):
"""Test invoke_stream properly yields streaming content and appends an aggregated message at the end."""
# Put a user message in the channel so it won't raise No chat history
mock_channel.messages.append(ChatMessageContent(role=AuthorRole.USER, content="Last user message"))
async def mock_invoke_stream(
messages: str, thread: AgentThread, sessionState=None, **kwargs
) -> AsyncIterable[StreamingChatMessageContent]:
yield AgentResponseItem(
message=StreamingChatMessageContent(
role=AuthorRole.ASSISTANT,
choice_index=0,
content="Hello",
),
thread=mock_channel.thread,
)
yield AgentResponseItem(
message=StreamingChatMessageContent(
role=AuthorRole.ASSISTANT,
choice_index=0,
content=" World",
),
thread=mock_channel.thread,
)
mock_agent.invoke_stream = mock_invoke_stream
# Check that we get the streamed messages and that the summarized message is appended afterward
messages_param = [ChatMessageContent(role=AuthorRole.USER, content="Last user message")] # just to pass the param
streamed_content = [msg async for msg in mock_channel.invoke_stream(mock_agent, messages_param)]
# We expect two streamed chunks: "Hello" and " World"
assert len(streamed_content) == 2
assert streamed_content[0].content == "Hello"
assert streamed_content[1].content == " World"
# Then we expect the channel to append an aggregated ChatMessageContent with "Hello World"
assert len(messages_param) == 2
appended = messages_param[1]
assert appended.role == AuthorRole.ASSISTANT
assert appended.content == "Hello World"
async def test_get_history(mock_channel, chat_history):
"""Test get_history yields messages in reverse order."""
mock_channel.messages = chat_history
reversed_history = [msg async for msg in mock_channel.get_history()]
# Should be reversed
assert reversed_history[0].content == "I'm good, thank you!"
assert reversed_history[1].content == "How are you?"
assert reversed_history[2].content == "Hello, User!"
assert reversed_history[3].content == "Hello, Bedrock!"
async def test_invoke_alternates_history_and_ensures_last_user_message(mock_channel, mock_agent):
"""Test invoke method ensures history alternates and last message is user."""
mock_channel.messages = [
ChatMessageContent(role=AuthorRole.USER, content="User1"),
ChatMessageContent(role=AuthorRole.USER, content="User2"),
ChatMessageContent(role=AuthorRole.ASSISTANT, content="Assist1"),
ChatMessageContent(role=AuthorRole.ASSISTANT, content="Assist2"),
ChatMessageContent(role=AuthorRole.USER, content="User3"),
ChatMessageContent(role=AuthorRole.USER, content="User4"),
ChatMessageContent(role=AuthorRole.ASSISTANT, content="Assist3"),
]
async for _, msg in mock_channel.invoke(mock_agent):
pass
# let's define expected roles from that final structure:
expected_roles = [
AuthorRole.USER,
AuthorRole.ASSISTANT, # placeholder
AuthorRole.USER,
AuthorRole.ASSISTANT,
AuthorRole.USER, # placeholder
AuthorRole.ASSISTANT,
AuthorRole.USER,
AuthorRole.ASSISTANT, # placeholder
AuthorRole.USER,
AuthorRole.ASSISTANT,
AuthorRole.USER, # placeholder
]
expected_contents = [
"User1",
BedrockAgentChannel.MESSAGE_PLACEHOLDER,
"User2",
"Assist1",
BedrockAgentChannel.MESSAGE_PLACEHOLDER,
"Assist2",
"User3",
BedrockAgentChannel.MESSAGE_PLACEHOLDER,
"User4",
"Assist3",
BedrockAgentChannel.MESSAGE_PLACEHOLDER,
]
assert len(mock_channel.messages) == len(expected_roles)
for i, (msg, exp_role, exp_content) in enumerate(zip(mock_channel.messages, expected_roles, expected_contents)):
assert msg.role == exp_role, f"Role mismatch at index {i}. Got {msg.role}, expected {exp_role}"
assert msg.content == exp_content, f"Content mismatch at index {i}. Got {msg.content}, expected {exp_content}"
@@ -0,0 +1,27 @@
# Copyright (c) Microsoft. All rights reserved.
import pytest
from semantic_kernel.agents.bedrock.models.bedrock_agent_event_type import BedrockAgentEventType
def test_bedrock_agent_event_type_values():
"""Test case to verify the values of BedrockAgentEventType enum."""
assert BedrockAgentEventType.CHUNK.value == "chunk"
assert BedrockAgentEventType.TRACE.value == "trace"
assert BedrockAgentEventType.RETURN_CONTROL.value == "returnControl"
assert BedrockAgentEventType.FILES.value == "files"
def test_bedrock_agent_event_type_enum():
"""Test case to verify the type of BedrockAgentEventType enum members."""
assert isinstance(BedrockAgentEventType.CHUNK, BedrockAgentEventType)
assert isinstance(BedrockAgentEventType.TRACE, BedrockAgentEventType)
assert isinstance(BedrockAgentEventType.RETURN_CONTROL, BedrockAgentEventType)
assert isinstance(BedrockAgentEventType.FILES, BedrockAgentEventType)
def test_bedrock_agent_event_type_invalid():
"""Test case to verify error handling for invalid BedrockAgentEventType value."""
with pytest.raises(ValueError):
BedrockAgentEventType("invalid_value")
@@ -0,0 +1,67 @@
# Copyright (c) Microsoft. All rights reserved.
from semantic_kernel.agents.bedrock.models.bedrock_agent_model import BedrockAgentModel
def test_bedrock_agent_model_valid():
"""Test case to verify the BedrockAgentModel with valid data."""
model = BedrockAgentModel(
agentId="test_id",
agentName="test_name",
agentVersion="1.0",
foundationModel="test_model",
agentStatus="CREATING",
)
assert model.agent_id == "test_id"
assert model.agent_name == "test_name"
assert model.agent_version == "1.0"
assert model.foundation_model == "test_model"
assert model.agent_status == "CREATING"
def test_bedrock_agent_model_missing_agent_id():
"""Test case to verify the BedrockAgentModel with missing agentId."""
model = BedrockAgentModel(
agentName="test_name",
agentVersion="1.0",
foundationModel="test_model",
agentStatus="CREATING",
)
assert model.agent_id is None
assert model.agent_name == "test_name"
assert model.agent_version == "1.0"
assert model.foundation_model == "test_model"
assert model.agent_status == "CREATING"
def test_bedrock_agent_model_missing_agent_name():
"""Test case to verify the BedrockAgentModel with missing agentName."""
model = BedrockAgentModel(
agentId="test_id",
agentVersion="1.0",
foundationModel="test_model",
agentStatus="CREATING",
)
assert model.agent_id == "test_id"
assert model.agent_name is None
assert model.agent_version == "1.0"
assert model.foundation_model == "test_model"
assert model.agent_status == "CREATING"
def test_bedrock_agent_model_extra_field():
"""Test case to verify the BedrockAgentModel with an extra field."""
model = BedrockAgentModel(
agentId="test_id",
agentName="test_name",
agentVersion="1.0",
foundationModel="test_model",
agentStatus="CREATING",
extraField="extra_value",
)
assert model.agent_id == "test_id"
assert model.agent_name == "test_name"
assert model.agent_version == "1.0"
assert model.foundation_model == "test_model"
assert model.agent_status == "CREATING"
assert model.extraField == "extra_value"
@@ -0,0 +1,28 @@
# Copyright (c) Microsoft. All rights reserved.
import pytest
from pydantic import ValidationError
from semantic_kernel.agents.bedrock.bedrock_agent_settings import BedrockAgentSettings
def test_bedrock_agent_settings_from_env_vars(bedrock_agent_unit_test_env):
"""Test loading BedrockAgentSettings from environment variables."""
settings = BedrockAgentSettings(env_file_path="fake_path")
assert settings.agent_resource_role_arn == bedrock_agent_unit_test_env["BEDROCK_AGENT_AGENT_RESOURCE_ROLE_ARN"]
assert settings.foundation_model == bedrock_agent_unit_test_env["BEDROCK_AGENT_FOUNDATION_MODEL"]
@pytest.mark.parametrize(
"exclude_list",
[
["BEDROCK_AGENT_AGENT_RESOURCE_ROLE_ARN"],
["BEDROCK_AGENT_FOUNDATION_MODEL"],
],
indirect=True,
)
def test_bedrock_agent_settings_from_env_vars_missing_required(bedrock_agent_unit_test_env):
"""Test loading BedrockAgentSettings from environment variables with missing required fields."""
with pytest.raises(ValidationError):
BedrockAgentSettings(env_file_path="fake_path")
@@ -0,0 +1,23 @@
# Copyright (c) Microsoft. All rights reserved.
import pytest
from semantic_kernel.agents.bedrock.models.bedrock_agent_status import BedrockAgentStatus
def test_bedrock_agent_status_values():
"""Test case to verify the values of BedrockAgentStatus enum."""
assert BedrockAgentStatus.CREATING == "CREATING"
assert BedrockAgentStatus.PREPARING == "PREPARING"
assert BedrockAgentStatus.PREPARED == "PREPARED"
assert BedrockAgentStatus.NOT_PREPARED == "NOT_PREPARED"
assert BedrockAgentStatus.DELETING == "DELETING"
assert BedrockAgentStatus.FAILED == "FAILED"
assert BedrockAgentStatus.VERSIONING == "VERSIONING"
assert BedrockAgentStatus.UPDATING == "UPDATING"
def test_bedrock_agent_status_invalid_value():
"""Test case to verify error handling for invalid BedrockAgentStatus value."""
with pytest.raises(ValueError):
BedrockAgentStatus("INVALID_STATUS")
@@ -0,0 +1,26 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, create_autospec
import pytest
from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.kernel import Kernel
@pytest.fixture
def kernel_with_ai_service():
kernel = create_autospec(Kernel)
mock_ai_service_client = create_autospec(ChatCompletionClientBase)
mock_prompt_execution_settings = create_autospec(PromptExecutionSettings)
mock_prompt_execution_settings.function_choice_behavior = None
kernel.select_ai_service.return_value = (mock_ai_service_client, mock_prompt_execution_settings)
mock_ai_service_client.get_chat_message_contents = AsyncMock(
return_value=[ChatMessageContent(role=AuthorRole.SYSTEM, content="Processed Message")]
)
kernel.plugins = {} # Ensure plugins dict is initialized to avoid AttributeError during tests
return kernel, mock_ai_service_client
@@ -0,0 +1,471 @@
# Copyright (c) Microsoft. All rights reserved.
from collections.abc import AsyncGenerator, Callable
from types import MethodType
from unittest.mock import AsyncMock, create_autospec, patch
import pytest
from pydantic import ValidationError
from semantic_kernel.agents import ChatCompletionAgent
from semantic_kernel.agents.channels.chat_history_channel import ChatHistoryChannel
from semantic_kernel.agents.chat_completion.chat_completion_agent import ChatHistoryAgentThread
from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase
from semantic_kernel.connectors.ai.open_ai.services.azure_chat_completion import AzureChatCompletion
from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion import OpenAIChatCompletion
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.function_call_content import FunctionCallContent
from semantic_kernel.contents.function_result_content import FunctionResultContent
from semantic_kernel.contents.history_reducer.chat_history_truncation_reducer import ChatHistoryTruncationReducer
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from semantic_kernel.contents.streaming_text_content import StreamingTextContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions import KernelServiceNotFoundError
from semantic_kernel.exceptions.agent_exceptions import AgentInvokeException
from semantic_kernel.functions.kernel_arguments import KernelArguments
from semantic_kernel.kernel import Kernel
@pytest.fixture
def mock_streaming_chat_completion_response() -> Callable[..., AsyncGenerator[list[ChatMessageContent], None]]:
async def mock_response(
chat_history: ChatHistory,
settings: PromptExecutionSettings,
kernel: Kernel,
arguments: KernelArguments,
) -> AsyncGenerator[list[ChatMessageContent], None]:
content1 = ChatMessageContent(role=AuthorRole.SYSTEM, content="Processed Message 1")
content2 = ChatMessageContent(role=AuthorRole.TOOL, content="Processed Message 2")
chat_history.messages.append(content1)
chat_history.messages.append(content2)
yield [content1]
yield [content2]
return mock_response
async def test_initialization():
agent = ChatCompletionAgent(
name="TestAgent",
id="test_id",
description="Test Description",
instructions="Test Instructions",
)
assert agent.name == "TestAgent"
assert agent.id == "test_id"
assert agent.description == "Test Description"
assert agent.instructions == "Test Instructions"
async def test_initialization_invalid_name_throws():
with pytest.raises(ValidationError):
_ = ChatCompletionAgent(
name="Test Agent",
id="test_id",
description="Test Description",
instructions="Test Instructions",
)
def test_initialization_with_kernel(kernel: Kernel):
agent = ChatCompletionAgent(
kernel=kernel,
name="TestAgent",
id="test_id",
description="Test Description",
instructions="Test Instructions",
)
assert kernel == agent.kernel
assert agent.name == "TestAgent"
assert agent.id == "test_id"
assert agent.description == "Test Description"
assert agent.instructions == "Test Instructions"
def test_initialization_with_kernel_and_service(kernel: Kernel, azure_openai_unit_test_env, openai_unit_test_env):
kernel.add_service(AzureChatCompletion(service_id="test_azure"))
agent = ChatCompletionAgent(
service=OpenAIChatCompletion(),
kernel=kernel,
name="TestAgent",
id="test_id",
description="Test Description",
instructions="Test Instructions",
)
assert kernel == agent.kernel
assert len(kernel.services) == 2
assert agent.name == "TestAgent"
assert agent.id == "test_id"
assert agent.description == "Test Description"
assert agent.instructions == "Test Instructions"
def test_initialization_with_plugins_via_constructor(custom_plugin_class):
agent = ChatCompletionAgent(
name="TestAgent",
id="test_id",
description="Test Description",
instructions="Test Instructions",
plugins=[custom_plugin_class()],
)
assert agent.name == "TestAgent"
assert agent.id == "test_id"
assert agent.description == "Test Description"
assert agent.instructions == "Test Instructions"
assert agent.kernel.plugins is not None
assert len(agent.kernel.plugins) == 1
def test_initialization_with_service_via_constructor(openai_unit_test_env):
agent = ChatCompletionAgent(
name="TestAgent",
id="test_id",
description="Test Description",
instructions="Test Instructions",
service=OpenAIChatCompletion(),
)
assert agent.name == "TestAgent"
assert agent.id == "test_id"
assert agent.description == "Test Description"
assert agent.instructions == "Test Instructions"
assert agent.service is not None
assert agent.kernel.services["test_chat_model_id"] == agent.service
def test_initialize_chat_history_agent_thread_with_id():
thread = ChatHistoryAgentThread(thread_id="test_thread_id")
assert thread is not None
assert thread.id == "test_thread_id"
def test_initialize_with_base_chat_history():
base_history = ChatHistory()
thread = ChatHistoryAgentThread(chat_history=base_history, thread_id="base_test_thread")
assert thread is not None
assert thread.id == "base_test_thread"
assert isinstance(thread._chat_history, ChatHistory)
assert not isinstance(thread._chat_history, ChatHistoryTruncationReducer)
def test_initialize_with_reducer_chat_history():
reducer = ChatHistoryTruncationReducer(
service=AsyncMock(spec=ChatCompletionClientBase), target_count=10, threshold_count=2
)
thread = ChatHistoryAgentThread(chat_history=reducer, thread_id="reducer_test_thread")
assert thread is not None
assert thread.id == "reducer_test_thread"
assert isinstance(thread._chat_history, ChatHistoryTruncationReducer)
async def test_get_response(kernel_with_ai_service: tuple[Kernel, ChatCompletionClientBase]):
kernel, _ = kernel_with_ai_service
agent = ChatCompletionAgent(
kernel=kernel,
name="TestAgent",
instructions="Test Instructions",
)
thread = ChatHistoryAgentThread()
response = await agent.get_response(messages="test", thread=thread)
assert response.message.content == "Processed Message"
assert response.thread is not None
async def test_get_response_exception(kernel_with_ai_service: tuple[Kernel, ChatCompletionClientBase]):
kernel, mock_ai_service_client = kernel_with_ai_service
mock_ai_service_client.get_chat_message_contents = AsyncMock(return_value=[])
agent = ChatCompletionAgent(
kernel=kernel,
name="TestAgent",
instructions="Test Instructions",
)
thread = ChatHistoryAgentThread()
with pytest.raises(AgentInvokeException):
await agent.get_response(messages="test", thread=thread)
async def test_invoke(kernel_with_ai_service: tuple[Kernel, ChatCompletionClientBase]):
kernel, _ = kernel_with_ai_service
agent = ChatCompletionAgent(
kernel=kernel,
name="TestAgent",
instructions="Test Instructions",
)
thread = ChatHistoryAgentThread()
messages = [message async for message in agent.invoke(messages="test", thread=thread)]
assert len(messages) == 1
assert messages[0].message.content == "Processed Message"
async def test_invoke_emits_tool_call_then_result_then_text(kernel_with_ai_service):
kernel, chat_client = kernel_with_ai_service
agent = ChatCompletionAgent(kernel=kernel, name="TestAgent")
thread = ChatHistoryAgentThread()
call_msg = ChatMessageContent(
role=AuthorRole.ASSISTANT,
items=[FunctionCallContent(id="test-id", name="get_specials", arguments="{}")],
)
result_msg = ChatMessageContent(
role=AuthorRole.TOOL,
items=[FunctionResultContent(id="test-id", name="get_specials", result="Clam Chowder")],
)
final_msg = ChatMessageContent(
role=AuthorRole.ASSISTANT,
content="Clam Chowder is today's soup.",
)
chat_client.get_chat_message_contents = AsyncMock(return_value=[final_msg])
async def fake_drain(self, *_args, **_kwargs):
if not fake_drain.called:
fake_drain.called = True
return [call_msg, result_msg]
return []
fake_drain.called = False
with patch.object(ChatCompletionAgent, "_drain_mutated_messages", new=AsyncMock(side_effect=fake_drain)):
cb_messages: list[ChatMessageContent] = []
async def on_msg(m: ChatMessageContent):
cb_messages.append(m)
messages = [
m
async for m in agent.invoke(
messages="What's the special soup?", thread=thread, on_intermediate_message=on_msg
)
]
assert [type(m.items[0]) for m in cb_messages] == [
FunctionCallContent,
FunctionResultContent,
]
assert len(messages) == 1
assert isinstance(messages[0].message, ChatMessageContent)
assert messages[0].message.content.startswith("Clam Chowder")
assert messages[0].message.name == agent.name
async def test_invoke_tool_call_not_added(kernel_with_ai_service: tuple[Kernel, ChatCompletionClientBase]):
kernel, mock_ai_service_client = kernel_with_ai_service
agent = ChatCompletionAgent(
kernel=kernel,
name="TestAgent",
)
thread = ChatHistoryAgentThread()
async def mock_get_chat_message_contents(
chat_history: ChatHistory,
settings: PromptExecutionSettings,
kernel: Kernel,
arguments: KernelArguments,
):
responses = [
ChatMessageContent(
role=AuthorRole.TOOL,
items=[FunctionResultContent(result="Tool Call Result")],
),
]
chat_history.messages.extend(responses)
return responses
mock_ai_service_client.get_chat_message_contents = AsyncMock(side_effect=mock_get_chat_message_contents)
messages = [message async for message in agent.invoke(messages="test", thread=thread)]
assert len(messages) == 1
assert messages[0].message.items[0].result == "Tool Call Result"
assert messages[0].message.role == AuthorRole.TOOL
assert messages[0].message.name == "TestAgent"
thread: ChatHistoryAgentThread = messages[-1].thread
thread_messages = [message async for message in thread.get_messages()]
assert len(thread_messages) == 2
assert thread_messages[0].content == "test"
assert thread_messages[1].items[0].result == "Tool Call Result"
assert thread_messages[1].name == "TestAgent"
assert thread_messages[1].role == AuthorRole.TOOL
async def test_invoke_no_service_throws(kernel: Kernel):
agent = ChatCompletionAgent(kernel=kernel, name="TestAgent")
with pytest.raises(KernelServiceNotFoundError):
async for _ in agent.invoke(messages="test", thread=None):
pass
async def test_invoke_stream(kernel_with_ai_service: tuple[Kernel, ChatCompletionClientBase]):
kernel, _ = kernel_with_ai_service
agent = ChatCompletionAgent(kernel=kernel, name="TestAgent")
thread = ChatHistoryAgentThread()
with patch(
"semantic_kernel.connectors.ai.chat_completion_client_base.ChatCompletionClientBase.get_streaming_chat_message_contents",
return_value=AsyncMock(),
) as mock:
mock.return_value.__aiter__.return_value = [
[ChatMessageContent(role=AuthorRole.USER, content="Initial Message")]
]
async for response in agent.invoke_stream(messages="Initial Message", thread=thread):
assert response.message.role == AuthorRole.USER
assert response.message.content == "Initial Message"
async def test_invoke_stream_emits_tool_call_then_result_then_text(kernel_with_ai_service):
kernel, chat_client = kernel_with_ai_service
agent = ChatCompletionAgent(kernel=kernel, name="TestAgent")
thread = ChatHistoryAgentThread()
call_msg = ChatMessageContent(
role=AuthorRole.ASSISTANT,
items=[FunctionCallContent(id="test-id", name="get_specials", arguments="{}")],
)
result_msg = ChatMessageContent(
role=AuthorRole.TOOL,
items=[FunctionResultContent(id="test-id", name="get_specials", result="Clam Chowder")],
)
text_msg = StreamingChatMessageContent(
role=AuthorRole.ASSISTANT,
content="Clam Chowder is today's soup.",
items=[StreamingTextContent(text="Clam Chowder is today's soup.", choice_index=0)],
choice_index=0,
)
async def fake_stream(*_args, **_kwargs):
yield [StreamingChatMessageContent(role=AuthorRole.ASSISTANT, content="", items=[], choice_index=0)]
yield [text_msg]
chat_client.get_streaming_chat_message_contents = MethodType(fake_stream, chat_client)
async def fake_drain(self, *_args, **_kwargs):
if not fake_drain.called:
fake_drain.called = True
return [call_msg, result_msg]
return []
fake_drain.called = False
with patch.object(ChatCompletionAgent, "_drain_mutated_messages", new=AsyncMock(side_effect=fake_drain)):
cb_messages: list[ChatMessageContent] = []
async def on_msg(m: ChatMessageContent):
cb_messages.append(m)
yielded_text: list[StreamingChatMessageContent] = []
async for resp in agent.invoke_stream(
messages="What's the special soup?",
thread=thread,
on_intermediate_message=on_msg,
):
yielded_text.append(resp.message)
assert [type(m.items[0]) for m in cb_messages] == [
FunctionCallContent,
FunctionResultContent,
]
assert len(yielded_text) == 1
assert isinstance(yielded_text[0], StreamingChatMessageContent)
assert yielded_text[0].content.startswith("Clam Chowder")
assert yielded_text[0].name == agent.name
async def test_invoke_stream_tool_call_added(
kernel_with_ai_service: tuple[Kernel, ChatCompletionClientBase],
mock_streaming_chat_completion_response,
):
kernel, mock_ai_service_client = kernel_with_ai_service
agent = ChatCompletionAgent(kernel=kernel, name="TestAgent")
thread = ChatHistoryAgentThread()
mock_ai_service_client.get_streaming_chat_message_contents = mock_streaming_chat_completion_response
async for response in agent.invoke_stream(messages="Initial Message", thread=thread):
assert response.message.role in [AuthorRole.SYSTEM, AuthorRole.TOOL]
assert response.message.content in ["Processed Message 1", "Processed Message 2"]
async def test_invoke_stream_no_service_throws(kernel: Kernel):
agent = ChatCompletionAgent(kernel=kernel, name="TestAgent")
thread = ChatHistoryAgentThread()
with pytest.raises(KernelServiceNotFoundError):
async for _ in agent.invoke_stream(messages="test", thread=thread):
pass
def test_get_channel_keys():
agent = ChatCompletionAgent()
keys = agent.get_channel_keys()
for key in keys:
assert isinstance(key, str)
async def test_create_channel():
agent = ChatCompletionAgent()
channel = await agent.create_channel()
assert isinstance(channel, ChatHistoryChannel)
async def test_prepare_agent_chat_history_with_formatted_instructions():
agent = ChatCompletionAgent(
name="TestAgent", id="test_id", description="Test Description", instructions="Test Instructions"
)
with patch.object(
ChatCompletionAgent, "format_instructions", new=AsyncMock(return_value="Formatted instructions for testing")
) as mock_format_instructions:
dummy_kernel = create_autospec(Kernel)
dummy_args = KernelArguments(param="value")
user_message = ChatMessageContent(role=AuthorRole.USER, content="User message")
history = ChatHistory(messages=[user_message])
result_history = await agent._prepare_agent_chat_history(history, dummy_kernel, dummy_args)
mock_format_instructions.assert_awaited_once_with(dummy_kernel, dummy_args)
assert len(result_history.messages) == 2
system_message = result_history.messages[0]
assert system_message.role == AuthorRole.SYSTEM
assert system_message.content == "Formatted instructions for testing"
assert system_message.name == agent.name
assert result_history.messages[1] == user_message
async def test_prepare_agent_chat_history_without_formatted_instructions():
agent = ChatCompletionAgent(
name="TestAgent", id="test_id", description="Test Description", instructions="Test Instructions"
)
with patch.object(
ChatCompletionAgent, "format_instructions", new=AsyncMock(return_value=None)
) as mock_format_instructions:
dummy_kernel = create_autospec(Kernel)
dummy_args = KernelArguments(param="value")
user_message = ChatMessageContent(role=AuthorRole.USER, content="User message")
history = ChatHistory(messages=[user_message])
result_history = await agent._prepare_agent_chat_history(history, dummy_kernel, dummy_args)
mock_format_instructions.assert_awaited_once_with(dummy_kernel, dummy_args)
assert len(result_history.messages) == 1
assert result_history.messages[0] == user_message
@@ -0,0 +1,252 @@
# Copyright (c) Microsoft. All rights reserved.
from collections.abc import AsyncIterable
from unittest.mock import AsyncMock
import pytest
from semantic_kernel.agents.agent import AgentResponseItem
from semantic_kernel.agents.channels.chat_history_channel import ChatHistoryChannel
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.file_reference_content import FileReferenceContent
from semantic_kernel.contents.function_result_content import FunctionResultContent
from semantic_kernel.contents.streaming_file_reference_content import StreamingFileReferenceContent
from semantic_kernel.contents.utils.author_role import AuthorRole
@pytest.fixture
def chat_history_channel() -> ChatHistoryChannel:
from semantic_kernel.agents.chat_completion.chat_completion_agent import ChatHistoryAgentThread
ChatHistoryChannel.model_rebuild()
thread = ChatHistoryAgentThread()
return ChatHistoryChannel(thread=thread)
class MockChatHistoryHandler:
"""Mock agent to test chat history handling"""
async def invoke(self, history: list[ChatMessageContent]) -> AsyncIterable[ChatMessageContent]:
for message in history:
yield ChatMessageContent(role=AuthorRole.SYSTEM, content=f"Processed: {message.content}")
async def invoke_stream(self, history: list[ChatMessageContent]) -> AsyncIterable[ChatMessageContent]:
for message in history:
yield ChatMessageContent(role=AuthorRole.SYSTEM, content=f"Processed: {message.content}")
async def reduce_history(self, history: list[ChatMessageContent]) -> list[ChatMessageContent]:
return history
class MockNonChatHistoryHandler:
"""Mock agent to test incorrect instance handling."""
id: str = "mock_non_chat_history_handler"
class AsyncIterableMock:
def __init__(self, async_gen):
self.async_gen = async_gen
def __aiter__(self):
return self.async_gen()
async def test_invoke(chat_history_channel):
channel = chat_history_channel
agent = AsyncMock(spec=MockChatHistoryHandler)
async def mock_invoke(history: list[ChatMessageContent]):
for message in history:
yield AgentResponseItem(
message=ChatMessageContent(role=AuthorRole.SYSTEM, content=f"Processed: {message.content}"),
thread=channel.thread,
)
agent.invoke.return_value = AsyncIterableMock(
lambda: mock_invoke([ChatMessageContent(role=AuthorRole.USER, content="Initial message")])
)
initial_message = ChatMessageContent(role=AuthorRole.USER, content="Initial message")
channel.messages.append(initial_message)
received_messages = []
async for is_visible, message in channel.invoke(agent, thread=channel.thread):
received_messages.append(message)
assert is_visible
assert len(received_messages) == 1
assert "Processed: Initial message" in received_messages[0].content
async def test_invoke_stream(chat_history_channel):
channel = chat_history_channel
agent = AsyncMock(spec=MockChatHistoryHandler)
async def mock_invoke(history: list[ChatMessageContent]):
for message in history:
msg = AgentResponseItem(
message=ChatMessageContent(role=AuthorRole.SYSTEM, content=f"Processed: {message.content}"),
thread=channel.thread,
)
yield msg
channel.add_message(msg.message)
agent.invoke_stream.return_value = AsyncIterableMock(
lambda: mock_invoke([ChatMessageContent(role=AuthorRole.USER, content="Initial message")])
)
initial_message = ChatMessageContent(role=AuthorRole.USER, content="Initial message")
channel.messages.append(initial_message)
received_messages = []
async for message in channel.invoke_stream(agent, thread=channel.thread, messages=received_messages):
assert message is not None
assert len(received_messages) == 1
assert "Processed: Initial message" in received_messages[0].content
async def test_invoke_leftover_in_queue(chat_history_channel):
channel = chat_history_channel
agent = AsyncMock(spec=MockChatHistoryHandler)
async def mock_invoke(history: list[ChatMessageContent]):
for message in history:
yield AgentResponseItem(
message=ChatMessageContent(role=AuthorRole.SYSTEM, content=f"Processed: {message.content}"),
thread=channel.thread,
)
yield AgentResponseItem(
message=ChatMessageContent(
role=AuthorRole.SYSTEM,
content="Final message",
items=[FunctionResultContent(id="test_id", result="test")],
),
thread=channel.thread,
)
agent.invoke.return_value = AsyncIterableMock(
lambda: mock_invoke([
ChatMessageContent(
role=AuthorRole.USER,
content="Initial message",
items=[FunctionResultContent(id="test_id", result="test")],
)
])
)
initial_message = ChatMessageContent(role=AuthorRole.USER, content="Initial message")
channel.messages.append(initial_message)
received_messages = []
async for is_visible, message in channel.invoke(agent, thread=channel.thread):
received_messages.append(message)
assert is_visible
if len(received_messages) >= 3:
break
assert len(received_messages) == 3
assert "Processed: Initial message" in received_messages[0].content
assert "Final message" in received_messages[2].content
assert received_messages[2].items[0].id == "test_id"
async def test_receive(chat_history_channel):
channel = chat_history_channel
history = [
ChatMessageContent(role=AuthorRole.SYSTEM, content="test message 1"),
ChatMessageContent(role=AuthorRole.USER, content="test message 2"),
]
await channel.receive(history)
assert len(channel.messages) == 2
assert channel.messages[0].content == "test message 1"
assert channel.messages[0].role == AuthorRole.SYSTEM
assert channel.messages[1].content == "test message 2"
assert channel.messages[1].role == AuthorRole.USER
async def test_get_history(chat_history_channel):
channel = chat_history_channel
history = [
ChatMessageContent(role=AuthorRole.SYSTEM, content="test message 1"),
ChatMessageContent(role=AuthorRole.USER, content="test message 2"),
]
channel.messages.extend(history)
messages = [message async for message in channel.get_history()]
assert len(messages) == 2
assert messages[0].content == "test message 2"
assert messages[0].role == AuthorRole.USER
assert messages[1].content == "test message 1"
assert messages[1].role == AuthorRole.SYSTEM
async def test_reset_history(chat_history_channel):
channel = chat_history_channel
history = [
ChatMessageContent(role=AuthorRole.SYSTEM, content="test message 1"),
ChatMessageContent(role=AuthorRole.USER, content="test message 2"),
]
channel.messages.extend(history)
messages = [message async for message in channel.get_history()]
assert len(messages) == 2
assert messages[0].content == "test message 2"
assert messages[0].role == AuthorRole.USER
assert messages[1].content == "test message 1"
assert messages[1].role == AuthorRole.SYSTEM
await channel.reset()
assert len(channel.messages) == 0
async def test_receive_skips_file_references(chat_history_channel):
channel = chat_history_channel
file_ref_item = FileReferenceContent()
streaming_file_ref_item = StreamingFileReferenceContent()
normal_item_1 = FunctionResultContent(id="test_id", result="normal content 1")
normal_item_2 = FunctionResultContent(id="test_id_2", result="normal content 2")
msg_with_file_only = ChatMessageContent(
role=AuthorRole.USER,
content="Normal message set as TextContent",
items=[file_ref_item],
)
msg_with_mixed = ChatMessageContent(
role=AuthorRole.USER,
content="Mixed content message",
items=[streaming_file_ref_item, normal_item_1],
)
msg_with_normal = ChatMessageContent(
role=AuthorRole.USER,
content="Normal message",
items=[normal_item_2],
)
history = [msg_with_file_only, msg_with_mixed, msg_with_normal]
await channel.receive(history)
assert len(channel.messages) == 3
assert channel.messages[0].content == "Normal message set as TextContent"
assert len(channel.messages[0].items) == 1
assert channel.messages[1].content == "Mixed content message"
assert len(channel.messages[0].items) == 1
assert channel.messages[1].items[0].result == "normal content 1"
assert channel.messages[2].content == "Normal message"
assert len(channel.messages[2].items) == 2
assert channel.messages[2].items[0].result == "normal content 2"
assert channel.messages[2].items[1].text == "Normal message"
@@ -0,0 +1,359 @@
# Copyright (c) Microsoft. All rights reserved.
import logging
from collections.abc import AsyncIterator
from types import SimpleNamespace
from typing import TypeVar
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from pydantic import SecretStr, ValidationError
try:
import microsoft_agents.copilotstudio.client # type: ignore # noqa: F401
copilot_installed = True
except ImportError:
copilot_installed = False
pytestmark = pytest.mark.skipif(not copilot_installed, reason="`copilotstudio.client` is not installed")
if copilot_installed:
import semantic_kernel.agents.copilot_studio.copilot_studio_agent as csa_mod
from semantic_kernel.agents import (
CopilotStudioAgent,
CopilotStudioAgentSettings,
CopilotStudioAgentThread,
)
from semantic_kernel.agents.copilot_studio.copilot_studio_agent import (
_CopilotStudioAgentTokenFactory,
)
from semantic_kernel.agents.copilot_studio.copilot_studio_agent_settings import (
CopilotStudioAgentAuthMode,
)
from semantic_kernel.contents import AuthorRole, ChatMessageContent
from semantic_kernel.exceptions import (
AgentInitializationException,
AgentThreadInitializationException,
)
from semantic_kernel.functions.kernel_arguments import KernelArguments
from semantic_kernel.kernel import Kernel
from semantic_kernel.prompt_template import PromptTemplateConfig
T = TypeVar("T")
@pytest.fixture
def aiter():
async def _aiter(items: list[T]) -> AsyncIterator[T]:
for item in items:
yield item
return _aiter
@pytest.fixture
def DummyConversation():
class DummyConversation:
def __init__(self, cid: str):
self.id = cid
return DummyConversation
@pytest.fixture
def DummyActivity(DummyConversation):
class DummyActivity:
def __init__(self, text: str, cid: str = "conv-123"):
self.type = "message"
self.text_format = "plain"
self.text = text
self.suggested_actions = None
self.conversation = DummyConversation(cid)
return DummyActivity
def test_initialize_thread_with_no_client_throws():
with pytest.raises(AgentThreadInitializationException, match="CopilotClient cannot be None"):
CopilotStudioAgentThread(client=None)
def test_normalize_messages():
client = MagicMock(spec=csa_mod.CopilotClient)
agent = CopilotStudioAgent(client=client)
single_str = "hello"
single_chat = ChatMessageContent(role=AuthorRole.USER, content="hola")
mixed = [single_str, single_chat]
assert agent._normalize_messages(None) == []
assert agent._normalize_messages(single_str) == ["hello"]
assert agent._normalize_messages(single_chat) == ["hola"]
assert agent._normalize_messages(mixed) == ["hello", "hola"]
def test_plugin_warning_emitted_once(caplog):
caplog.set_level(logging.WARNING)
dummy_kernel = Kernel()
dummy_kernel.add_plugin(SimpleNamespace(name="MyPlugin"))
agent = CopilotStudioAgent(client=MagicMock(spec=csa_mod.CopilotClient), kernel=dummy_kernel)
warn = [rec for rec in caplog.records if rec.levelno == logging.WARNING]
assert len(warn) == 1
assert "plugins will be ignored" in warn[0].getMessage()
caplog.clear()
_ = agent.model_copy()
assert not caplog.records
async def test_inner_invoke_prompt_and_yield(monkeypatch, aiter, DummyActivity):
client = MagicMock(spec=csa_mod.CopilotClient)
prompts: dict[str, str] = {}
async def fake_ask_question(*, question: str, conversation_id: str):
prompts["sent"] = question
yield DummyActivity("Aye matey!")
client.ask_question.side_effect = fake_ask_question
client.start_conversation = lambda: aiter([DummyActivity("", "conv-123")])
monkeypatch.setattr(
CopilotStudioAgent,
"format_instructions",
AsyncMock(return_value="Respond like a pirate"),
)
agent = CopilotStudioAgent(client=client)
thread = CopilotStudioAgentThread(client=client)
replies = [
msg
async for msg in agent._inner_invoke(
thread=thread, messages=["Tell me a joke about bears", "make the joke kid friendly"]
)
]
expected_prompt = "Respond like a pirate\nTell me a joke about bears\nmake the joke kid friendly"
assert prompts["sent"] == expected_prompt
assert len(replies) == 1
assert replies[0].content == "Aye matey!"
assert replies[0].role is AuthorRole.ASSISTANT
async def test_get_response(monkeypatch, aiter, DummyActivity):
client = MagicMock(spec=csa_mod.CopilotClient)
sent: dict[str, str] = {}
async def fake_ask_question(*, question: str, conversation_id: str):
sent["cid"] = conversation_id
yield DummyActivity("first", conversation_id)
yield DummyActivity("second", conversation_id)
client.ask_question.side_effect = fake_ask_question
client.start_conversation = lambda: aiter([DummyActivity("", "conv-123")])
monkeypatch.setattr(CopilotStudioAgent, "format_instructions", AsyncMock(return_value=None))
agent = CopilotStudioAgent(client=client)
thread = CopilotStudioAgentThread(client=client)
item = await agent.get_response(messages="hi there", thread=thread)
assert item.message.content == "second"
assert thread.id == "conv-123"
assert sent["cid"] == "conv-123"
assert item.thread is thread
async def test_invoke(monkeypatch, aiter, DummyActivity):
client = MagicMock(spec=csa_mod.CopilotClient)
sent: dict[str, str] = {}
async def fake_ask_question(*, question: str, conversation_id: str):
sent["cid"] = conversation_id
yield DummyActivity("first", conversation_id)
yield DummyActivity("second", conversation_id)
client.ask_question.side_effect = fake_ask_question
client.start_conversation = lambda: aiter([DummyActivity("", "conv-123")])
monkeypatch.setattr(CopilotStudioAgent, "format_instructions", AsyncMock(return_value=None))
agent = CopilotStudioAgent(
client=client,
prompt_template_config=PromptTemplateConfig(template="Handle the message in this {{$style}}"),
)
thread = CopilotStudioAgentThread(client=client)
responses = []
async for response in agent.invoke(
messages="hi there", thread=thread, arguments=KernelArguments(style="funny")
):
responses.append(response)
item = responses[-1]
assert item.message.content == "second"
assert thread.id == "conv-123"
assert sent["cid"] == "conv-123"
assert item.thread is thread
def test_setup_resources_settings_validation_error():
sentinel_exc = ValidationError.from_exception_data("dummy", [], input_type="python")
with (
patch(
"semantic_kernel.agents.copilot_studio.copilot_studio_agent.CopilotStudioAgentSettings",
side_effect=sentinel_exc,
),
pytest.raises(AgentInitializationException, match="Failed to create Copilot Studio Agent settings"),
):
_ = CopilotStudioAgent.create_client(app_client_id="appid", tenant_id="tenantid")
def test_setup_resources_missing_ids():
dummy_settings = MagicMock(spec=CopilotStudioAgentSettings)
dummy_settings.app_client_id = None
dummy_settings.tenant_id = None
with (
patch(
"semantic_kernel.agents.copilot_studio.copilot_studio_agent.CopilotStudioAgentSettings",
return_value=dummy_settings,
),
pytest.raises(
AgentInitializationException,
match="Missing required configuration field\\(s\\): app_client_id, tenant_id",
),
):
_ = CopilotStudioAgent.create_client()
def test_setup_resources_happy_path(tmp_path, monkeypatch):
dummy_settings = MagicMock(spec=CopilotStudioAgentSettings)
dummy_settings.app_client_id = "appid"
dummy_settings.tenant_id = "tenantid"
dummy_settings.auth_mode = CopilotStudioAgentAuthMode.INTERACTIVE
dummy_settings.client_secret = SecretStr("test-secret")
monkeypatch.setattr(
"semantic_kernel.agents.copilot_studio.copilot_studio_agent.CopilotStudioAgentSettings",
lambda **_: dummy_settings,
)
monkeypatch.setattr(_CopilotStudioAgentTokenFactory, "acquire", lambda self: "fake-token")
sentinel_client = MagicMock(spec=csa_mod.CopilotClient)
with patch(
"semantic_kernel.agents.copilot_studio.copilot_studio_agent.CopilotClient", return_value=sentinel_client
) as mock_client_ctor:
cache_path = tmp_path / "cache.bin"
monkeypatch.setenv("TOKEN_CACHE_PATH", str(cache_path))
returned = CopilotStudioAgent.create_client(
app_client_id="appid",
tenant_id="tenantid",
environment_id="env-id",
agent_identifier="agent-name",
auth_mode=CopilotStudioAgentAuthMode.SERVICE,
)
mock_client_ctor.assert_called_once_with(dummy_settings, "fake-token")
assert returned is sentinel_client
class DummyCache:
pass
class FakeAppSilent:
def __init__(self, client_id, authority, token_cache, client_credential=None, **kwargs):
pass
def get_accounts(self):
return [{"home_account_id": "acct1"}]
def acquire_token_silent(self, scopes, account):
return {"access_token": "silent-token"}
def acquire_token_interactive(self, scopes):
pytest.skip("Unexpected interactive flow in silent test")
class FakeAppInteractive:
def __init__(self, client_id, authority, token_cache):
pass
def get_accounts(self):
return []
def acquire_token_silent(self, scopes, account):
pytest.skip("Unexpected silent flow in interactive test")
def acquire_token_interactive(self, scopes):
return {"access_token": "interactive-token"}
class FakeAppError:
def __init__(self, client_id, authority, token_cache):
pass
def get_accounts(self):
return []
def acquire_token_silent(self, scopes, account):
return {}
def acquire_token_interactive(self, scopes):
return {
"error": "bad",
"error_description": "failed",
"correlation_id": "cid",
}
@pytest.fixture(autouse=True)
def stub_cache(monkeypatch):
monkeypatch.setattr(
_CopilotStudioAgentTokenFactory,
"_get_msal_token_cache",
staticmethod(lambda cache_path, fallback_to_plaintext=True: DummyCache()),
)
@pytest.mark.parametrize(
"fake_app, expected_token, mode",
[
pytest.param(
FakeAppSilent,
"silent-token",
CopilotStudioAgentAuthMode.SERVICE,
marks=pytest.mark.skip(reason="Skipping SERVICE auth mode test as the mode is not yet supported."),
),
(FakeAppInteractive, "interactive-token", CopilotStudioAgentAuthMode.INTERACTIVE),
],
)
def test_acquire_token_success(monkeypatch, tmp_path, fake_app, expected_token, mode):
settings = CopilotStudioAgentSettings(app_client_id="id", tenant_id="tid")
cache_path = str(tmp_path / "cache.bin")
monkeypatch.setattr(csa_mod, "PublicClientApplication", fake_app)
monkeypatch.setattr(csa_mod, "ConfidentialClientApplication", fake_app)
client_secret = None
if fake_app == FakeAppSilent:
client_secret = "test-secret"
factory = _CopilotStudioAgentTokenFactory(
settings=settings,
cache_path=cache_path,
mode=mode,
client_secret=client_secret,
client_certificate=None,
user_assertion=None,
)
token = factory.acquire()
assert token == expected_token
def test_acquire_token_error(monkeypatch, tmp_path):
settings = CopilotStudioAgentSettings(app_client_id="id", tenant_id="tid")
cache_path = str(tmp_path / "cache.bin")
monkeypatch.setattr(csa_mod, "PublicClientApplication", FakeAppError)
monkeypatch.setattr(csa_mod, "ConfidentialClientApplication", FakeAppError)
factory = _CopilotStudioAgentTokenFactory(
settings=settings,
cache_path=cache_path,
mode=CopilotStudioAgentAuthMode.INTERACTIVE,
client_secret=None,
client_certificate=None,
user_assertion=None,
)
with pytest.raises(AgentInitializationException):
factory.acquire()
@@ -0,0 +1,481 @@
# Copyright (c) Microsoft. All rights reserved.
"""
Unit tests for OpenAI ResponsesAgent reasoning configuration.
These tests verify the reasoning functionality for OpenAI ResponsesAgent,
including priority hierarchies, parameter validation, and callback handling.
"""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from openai import AsyncOpenAI
from openai.types.responses.response_reasoning_item import ResponseReasoningItem
from openai.types.responses.response_reasoning_text_delta_event import ResponseReasoningTextDeltaEvent
from openai.types.responses.response_reasoning_text_done_event import ResponseReasoningTextDoneEvent
from semantic_kernel.agents.open_ai.openai_responses_agent import OpenAIResponsesAgent
from semantic_kernel.agents.open_ai.responses_agent_thread_actions import ResponsesAgentThreadActions
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.reasoning_content import ReasoningContent
from semantic_kernel.contents.streaming_reasoning_content import StreamingReasoningContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions import ContentAdditionException
def test_constructor_reasoning_is_stored():
"""Test that reasoning object is stored during construction"""
client = AsyncMock(spec=AsyncOpenAI)
agent = OpenAIResponsesAgent(client=client, ai_model_id="gpt-4o", reasoning={"effort": "high"})
assert agent.reasoning == {"effort": "high"}
def test_constructor_reasoning_defaults_to_none():
"""Test that constructor reasoning defaults to None when not specified."""
# Arrange & Act: Create agent without reasoning
client = AsyncMock(spec=AsyncOpenAI)
agent = OpenAIResponsesAgent(
ai_model_id="gpt-4o",
client=client,
name="TestAgent",
# No reasoning specified
)
# Assert: Default reasoning is None
assert agent.reasoning is None
def test_reasoning_priority_order_per_invocation_overrides_constructor():
"""Test per-invocation reasoning overrides constructor default."""
# Arrange: Mock agent with constructor default
agent = AsyncMock()
agent.ai_model_id = "o1"
agent.reasoning = {"effort": "low"} # Constructor setting
# Act: Override with per-invocation reasoning
options = ResponsesAgentThreadActions._generate_options(
agent=agent,
model="o1",
reasoning={"effort": "high"}, # Per-invocation override
)
# Assert: Per-invocation override wins
assert options["reasoning"] == {"effort": "high"}
def test_reasoning_priority_order_complete_hierarchy():
"""Test complete reasoning priority hierarchy: per-invocation > constructor."""
# Test 1: Per-invocation has highest priority
agent = AsyncMock()
agent.ai_model_id = "o1"
agent.reasoning = {"effort": "low"}
options = ResponsesAgentThreadActions._generate_options(
agent=agent,
model="o1",
reasoning={"effort": "medium"}, # Per-invocation
)
assert options["reasoning"] == {"effort": "medium"} # Per-invocation wins
# Test 2: Constructor has priority when no per-invocation
agent = AsyncMock()
agent.ai_model_id = "o1"
agent.reasoning = {"effort": "low"}
options = ResponsesAgentThreadActions._generate_options(
agent=agent,
model="o1",
# No per-invocation reasoning
)
assert options["reasoning"] == {"effort": "low"} # Constructor wins
# Test 3: No reasoning when neither constructor nor per-invocation provided
agent = AsyncMock()
agent.ai_model_id = "o1"
agent.reasoning = None # No constructor reasoning
options = ResponsesAgentThreadActions._generate_options(agent=agent, model="o1")
assert "reasoning" not in options # No automatic defaults
def test_multi_agent_reasoning_isolation():
"""Test multiple agents maintain separate reasoning configurations."""
client = AsyncMock(spec=AsyncOpenAI)
# Agent 1 with low reasoning
low_agent = OpenAIResponsesAgent(ai_model_id="o1", client=client, name="LowAgent", reasoning={"effort": "low"})
# Agent 2 with high reasoning
high_agent = OpenAIResponsesAgent(ai_model_id="o1", client=client, name="HighAgent", reasoning={"effort": "high"})
# Assert: Agents maintain separate defaults
assert low_agent.reasoning == {"effort": "low"}
assert high_agent.reasoning == {"effort": "high"}
# Verify isolation through options generation
low_options = ResponsesAgentThreadActions._generate_options(agent=low_agent, model="o1")
high_options = ResponsesAgentThreadActions._generate_options(agent=high_agent, model="o1")
assert low_options["reasoning"] == {"effort": "low"}
assert high_options["reasoning"] == {"effort": "high"}
def test_reasoning_validation_not_available():
"""Test that validation method was removed in simplified implementation."""
# The validation method was removed, so this test now checks that it's not available
assert not hasattr(ResponsesAgentThreadActions, "_validate_reasoning_effort_parameter")
def test_explicit_none_reasoning_disables_reasoning():
"""Test explicitly setting reasoning=None disables reasoning."""
agent = AsyncMock()
agent.ai_model_id = "o1"
agent.reasoning = None
# Explicitly disable reasoning
options = ResponsesAgentThreadActions._generate_options(
agent=agent,
model="o1",
reasoning=None, # Explicit None
)
# Explicit None should disable reasoning
assert "reasoning" not in options
def test_reasoning_object_structure_follows_openai_api():
"""Test reasoning parameter is correctly structured for OpenAI API."""
agent = AsyncMock()
agent.ai_model_id = "o1"
agent.reasoning = {"effort": "high", "summary": "auto"}
options = ResponsesAgentThreadActions._generate_options(agent=agent, model="o1")
# Verify correct OpenAI API structure
assert "reasoning" in options
reasoning = options["reasoning"]
assert isinstance(reasoning, dict)
assert reasoning == {"effort": "high", "summary": "auto"}
def test_reasoning_object_pass_through():
"""Test that reasoning objects are passed through directly."""
agent = AsyncMock()
agent.ai_model_id = "o1"
agent.reasoning = None
# Test different reasoning object structures
test_cases = [
{"effort": "low"},
{"effort": "medium", "summary": "concise"},
{"effort": "high", "summary": "detailed"},
{"effort": "minimal", "generate_summary": "auto"}, # deprecated field
]
for reasoning_obj in test_cases:
options = ResponsesAgentThreadActions._generate_options(agent=agent, model="o1", reasoning=reasoning_obj)
assert options["reasoning"] == reasoning_obj
def test_get_reasoning_items_from_output():
"""Test extraction of reasoning items from response output."""
# Create mock ResponseReasoningItem
mock_reasoning_item = MagicMock(spec=ResponseReasoningItem)
mock_reasoning_item.id = "reasoning-123"
mock_reasoning_item.content = "The model is thinking..."
mock_reasoning_item.summary = "Analyzed the problem"
mock_reasoning_item.status = "completed"
# Create mock ResponsesAgentThreadActions._create_reasoning_content_from_openai_item method
expected_reasoning_content = MagicMock(spec=ReasoningContent)
with patch.object(
ResponsesAgentThreadActions,
"_create_reasoning_content_from_openai_item",
return_value=expected_reasoning_content,
):
# Test with reasoning item in output
output_with_reasoning = [mock_reasoning_item, MagicMock()]
result = ResponsesAgentThreadActions._get_reasoning_items_from_output(output_with_reasoning)
assert len(result) == 1
assert result[0] == expected_reasoning_content
ResponsesAgentThreadActions._create_reasoning_content_from_openai_item.assert_called_once_with(
mock_reasoning_item
)
def test_get_reasoning_items_from_output_empty():
"""Test extraction with no reasoning items in output."""
# Test with no reasoning items
output_without_reasoning = [MagicMock(), MagicMock()]
result = ResponsesAgentThreadActions._get_reasoning_items_from_output(output_without_reasoning)
assert len(result) == 0
def test_get_reasoning_items_from_output_mixed():
"""Test extraction with mixed output types including reasoning."""
# Create mock items
mock_reasoning_item1 = MagicMock(spec=ResponseReasoningItem)
mock_reasoning_item2 = MagicMock(spec=ResponseReasoningItem)
mock_other_item = MagicMock()
expected_reasoning1 = MagicMock(spec=ReasoningContent)
expected_reasoning2 = MagicMock(spec=ReasoningContent)
with patch.object(
ResponsesAgentThreadActions,
"_create_reasoning_content_from_openai_item",
side_effect=[expected_reasoning1, expected_reasoning2],
):
output_mixed = [mock_other_item, mock_reasoning_item1, mock_reasoning_item2]
result = ResponsesAgentThreadActions._get_reasoning_items_from_output(output_mixed)
assert len(result) == 2
assert result[0] == expected_reasoning1
assert result[1] == expected_reasoning2
@pytest.mark.parametrize(
"reasoning_config,expected_summary",
[
({"effort": "high"}, None),
({"effort": "high", "summary": "detailed"}, "detailed"),
({"effort": "medium", "summary": "concise"}, "concise"),
({"effort": "low", "summary": "auto"}, "auto"),
],
)
def test_reasoning_summary_configuration(reasoning_config, expected_summary):
"""Test that reasoning summary configuration is properly handled."""
agent = AsyncMock()
agent.ai_model_id = "o1"
agent.reasoning = None
options = ResponsesAgentThreadActions._generate_options(agent=agent, model="o1", reasoning=reasoning_config)
assert options["reasoning"] == reasoning_config
if expected_summary:
assert options["reasoning"]["summary"] == expected_summary
def test_streaming_reasoning_content_creation():
"""Test StreamingReasoningContent creation and basic functionality."""
# Test basic creation
reasoning = StreamingReasoningContent(text="Initial reasoning", choice_index=0)
assert reasoning.text == "Initial reasoning"
assert reasoning.choice_index == 0
assert str(reasoning) == "Initial reasoning"
assert bytes(reasoning) == b"Initial reasoning"
def test_streaming_reasoning_content_addition():
"""Test StreamingReasoningContent __add__ method."""
reasoning1 = StreamingReasoningContent(
text="First part", choice_index=0, ai_model_id="gpt-4o", metadata={"key1": "value1"}
)
reasoning2 = StreamingReasoningContent(
text=" second part", choice_index=0, ai_model_id="gpt-4o", metadata={"key2": "value2"}
)
combined = reasoning1 + reasoning2
assert combined.text == "First part second part"
assert combined.choice_index == 0
assert combined.ai_model_id == "gpt-4o"
assert combined.metadata == {"key1": "value1", "key2": "value2"}
def test_streaming_reasoning_content_addition_errors():
"""Test StreamingReasoningContent addition error conditions."""
reasoning1 = StreamingReasoningContent(text="text1", choice_index=0, ai_model_id="model1")
reasoning2 = StreamingReasoningContent(text="text2", choice_index=1, ai_model_id="model1")
reasoning3 = StreamingReasoningContent(text="text3", choice_index=0, ai_model_id="model2")
# Different choice_index should raise error
with pytest.raises(ContentAdditionException, match="different choice_index"):
reasoning1 + reasoning2
# Different ai_model_id should raise error
with pytest.raises(ContentAdditionException, match="different ai_model_id"):
reasoning1 + reasoning3
def test_streaming_reasoning_content_with_regular_reasoning():
"""Test StreamingReasoningContent addition with regular ReasoningContent."""
streaming = StreamingReasoningContent(
text="Stream: ", choice_index=0, ai_model_id="gpt-4o", metadata={"stream": True}
)
regular = ReasoningContent(text="regular", ai_model_id="gpt-4o", metadata={"regular": True})
combined = streaming + regular
assert isinstance(combined, StreamingReasoningContent)
assert combined.text == "Stream: regular"
assert combined.choice_index == 0
assert combined.ai_model_id == "gpt-4o"
assert combined.metadata == {"stream": True, "regular": True}
def test_reasoning_content_from_response_item():
"""Test ReasoningContent creation from OpenAI ResponseReasoningItem via agent thread actions."""
mock_item = MagicMock(spec=ResponseReasoningItem)
mock_item.id = "reasoning-123"
mock_item.summary = [MagicMock(text="Analyzed the user's request")]
mock_item.status = "completed"
# Test the _create_reasoning_content_from_openai_item method via thread actions
reasoning = ResponsesAgentThreadActions._create_reasoning_content_from_openai_item(mock_item)
assert isinstance(reasoning, ReasoningContent)
assert reasoning.text == "Analyzed the user's request"
assert reasoning.metadata["id"] == "reasoning-123"
assert reasoning.metadata["status"] == "completed"
def test_callback_signature_validation():
"""Test that on_intermediate_message callback has correct signature."""
async def valid_callback(message: ChatMessageContent) -> None:
"""Valid callback signature."""
pass
async def invalid_callback(message: str) -> None:
"""Invalid callback signature."""
pass
# This test verifies the expected signature pattern exists
# In actual usage, the callback should accept ChatMessageContent
test_message = ChatMessageContent(role=AuthorRole.ASSISTANT, items=[ReasoningContent(text="reasoning")])
# Valid callback should work (this is a type check test)
assert callable(valid_callback)
assert test_message.role == AuthorRole.ASSISTANT
# Note: Runtime type checking would be done by the agent implementation
@patch.object(ResponsesAgentThreadActions, "_get_reasoning_items_from_output")
def test_reasoning_yield_pattern(mock_get_reasoning):
"""Test that reasoning content yields False (intermediate) while final content yields True."""
# Mock reasoning items being found
mock_reasoning_content = ReasoningContent(text="Thinking about the answer...")
mock_get_reasoning.return_value = [mock_reasoning_content]
# In actual usage, when reasoning items are found:
# yield False, reasoning_message # <- Intermediate (not visible to user)
# yield True, final_message # <- Final response (visible to user)
# This test verifies the pattern exists in the invoke method
result = ResponsesAgentThreadActions._get_reasoning_items_from_output([])
mock_get_reasoning.assert_called_once()
assert result is not None # Verify the method returns something
def test_streaming_reasoning_events():
"""Test handling of streaming reasoning events."""
# Test delta event
delta_event = MagicMock(spec=ResponseReasoningTextDeltaEvent)
delta_event.delta = "Thinking"
delta_event.item_id = "reasoning-123"
# Test done event
done_event = MagicMock(spec=ResponseReasoningTextDoneEvent)
done_event.text = "Thinking process complete"
done_event.item_id = "reasoning-123"
# Verify events have expected attributes
assert hasattr(delta_event, "delta")
assert hasattr(done_event, "text")
# These would be processed in invoke_stream method
# Delta events create StreamingReasoningContent
# Done events create ReasoningContent
def test_reasoning_metadata_handling():
"""Test that reasoning content properly handles metadata."""
reasoning = ReasoningContent(text="Analysis complete", metadata={"model": "gpt-4o", "reasoning_effort": "high"})
streaming_reasoning = StreamingReasoningContent(
text="Analyzing...", choice_index=0, metadata={"stream": True, "chunk": 1}
)
assert reasoning.metadata["model"] == "gpt-4o"
assert reasoning.metadata["reasoning_effort"] == "high"
assert streaming_reasoning.metadata["stream"] is True
assert streaming_reasoning.metadata["chunk"] == 1
@pytest.mark.parametrize(
"text_input,expected_bytes",
[
("Simple reasoning", b"Simple reasoning"),
("", b""),
("Unicode: 🤔", "Unicode: 🤔".encode()),
],
)
def test_streaming_reasoning_content_bytes_conversion(text_input, expected_bytes):
"""Test StreamingReasoningContent bytes conversion with various inputs."""
reasoning = StreamingReasoningContent(text=text_input, choice_index=0)
assert bytes(reasoning) == expected_bytes
def test_streaming_reasoning_content_default_text():
"""Test StreamingReasoningContent with default text value."""
# Test with no text parameter (should default to empty string)
reasoning_default = StreamingReasoningContent(choice_index=0)
assert reasoning_default.text is None
assert str(reasoning_default) == ""
assert bytes(reasoning_default) == b""
# Test with empty string
reasoning_empty = StreamingReasoningContent(text="", choice_index=0)
assert reasoning_empty.text == ""
assert str(reasoning_empty) == ""
assert bytes(reasoning_empty) == b""
def test_reasoning_integration_flow():
"""Test the complete flow of reasoning content through the system."""
# 1. OpenAI returns ResponseReasoningItem
mock_reasoning_item = MagicMock(spec=ResponseReasoningItem)
mock_reasoning_item.content = "Let me analyze this step by step..."
# 2. Convert to ReasoningContent
reasoning_content = ReasoningContent(text="Let me analyze this step by step...")
# 3. Create ChatMessageContent with reasoning
reasoning_message = ChatMessageContent(role=AuthorRole.ASSISTANT, items=[reasoning_content], ai_model_id="gpt-4o")
# 4. Verify message structure
assert len(reasoning_message.items) == 1
assert isinstance(reasoning_message.items[0], ReasoningContent)
assert reasoning_message.items[0].text == "Let me analyze this step by step..."
assert reasoning_message.role == AuthorRole.ASSISTANT
def test_multiple_reasoning_items_extraction():
"""Test extraction of multiple reasoning items from response output."""
# Create multiple mock reasoning items
reasoning1 = MagicMock(spec=ResponseReasoningItem)
reasoning1.content = "First reasoning step"
reasoning2 = MagicMock(spec=ResponseReasoningItem)
reasoning2.content = "Second reasoning step"
other_item = MagicMock() # Non-reasoning item
output = [reasoning1, other_item, reasoning2]
with patch.object(
ResponsesAgentThreadActions,
"_create_reasoning_content_from_openai_item",
side_effect=lambda x: ReasoningContent(text=x.content),
):
result = ResponsesAgentThreadActions._get_reasoning_items_from_output(output)
assert len(result) == 2
assert result[0].text == "First reasoning step"
assert result[1].text == "Second reasoning step"
@@ -0,0 +1,105 @@
# Copyright (c) Microsoft. All rights reserved.
from typing import Any
from unittest.mock import AsyncMock, MagicMock
import pytest
from openai import AsyncOpenAI
from openai.types.beta.assistant import Assistant
from openai.types.beta.threads.file_citation_annotation import FileCitation, FileCitationAnnotation
from openai.types.beta.threads.file_path_annotation import FilePath, FilePathAnnotation
from openai.types.beta.threads.image_file import ImageFile
from openai.types.beta.threads.image_file_content_block import ImageFileContentBlock
from openai.types.beta.threads.text import Text
from openai.types.beta.threads.text_content_block import TextContentBlock
@pytest.fixture
def mock_thread():
class MockThread:
id = "test_thread_id"
return MockThread()
@pytest.fixture
def mock_thread_messages():
class MockMessage:
def __init__(self, id, role, content, assistant_id=None):
self.id = id
self.role = role
self.content = content
self.assistant_id = assistant_id
return [
MockMessage(
id="test_message_id_1",
role="user",
content=[
TextContentBlock(
type="text",
text=Text(
value="Hello",
annotations=[
FilePathAnnotation(
type="file_path",
file_path=FilePath(file_id="test_file_id"),
end_index=5,
start_index=0,
text="Hello",
),
FileCitationAnnotation(
type="file_citation",
file_citation=FileCitation(file_id="test_file_id"),
text="Hello",
start_index=0,
end_index=5,
),
],
),
)
],
),
MockMessage(
id="test_message_id_2",
role="assistant",
content=[
ImageFileContentBlock(type="image_file", image_file=ImageFile(file_id="test_file_id", detail="auto"))
],
assistant_id="assistant_1",
),
]
@pytest.fixture
def openai_client(assistant_definition, mock_thread, mock_thread_messages) -> AsyncMock:
async def mock_list_messages(*args, **kwargs) -> Any:
return MagicMock(data=mock_thread_messages)
async def mock_retrieve_assistant(*args, **kwargs) -> Any:
asst = AsyncMock(spec=Assistant)
asst.name = "test-assistant"
return asst
client = AsyncMock(spec=AsyncOpenAI)
client.beta = MagicMock()
client.beta.assistants = MagicMock()
client.beta.assistants.create = AsyncMock(return_value=assistant_definition)
client.beta.assistants.retrieve = AsyncMock(side_effect=mock_retrieve_assistant)
client.beta.threads = MagicMock()
client.beta.threads.create = AsyncMock(return_value=mock_thread)
client.beta.threads.messages = MagicMock()
client.beta.threads.messages.list = AsyncMock(side_effect=mock_list_messages)
return client
@pytest.fixture
def assistant_definition() -> AsyncMock:
definition = AsyncMock(spec=Assistant)
definition.id = "agent123"
definition.name = "agentName"
definition.description = "desc"
definition.instructions = "test agent"
return definition
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,468 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from openai import AsyncOpenAI
from openai.types.beta.assistant import Assistant
from openai.types.beta.threads.file_citation_annotation import FileCitation, FileCitationAnnotation
from openai.types.beta.threads.file_path_annotation import FilePath, FilePathAnnotation
from openai.types.beta.threads.image_file import ImageFile
from openai.types.beta.threads.image_file_content_block import ImageFileContentBlock
from openai.types.beta.threads.text import Text
from openai.types.beta.threads.text_content_block import TextContentBlock
from pydantic import BaseModel, ValidationError
from semantic_kernel.agents import AgentRegistry
from semantic_kernel.agents.open_ai.azure_assistant_agent import AzureAssistantAgent
from semantic_kernel.agents.open_ai.openai_assistant_agent import AssistantAgentThread
from semantic_kernel.agents.open_ai.run_polling_options import RunPollingOptions
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions.agent_exceptions import AgentInitializationException
from semantic_kernel.functions.kernel_arguments import KernelArguments
from semantic_kernel.functions.kernel_function_decorator import kernel_function
from semantic_kernel.functions.kernel_plugin import KernelPlugin
from semantic_kernel.kernel import Kernel
from semantic_kernel.prompt_template.prompt_template_config import PromptTemplateConfig
@pytest.fixture
def mock_azure_openai_client_and_definition():
client = AsyncMock(spec=AsyncOpenAI)
client.beta = MagicMock()
client.beta.assistants = MagicMock()
definition = AsyncMock(spec=Assistant)
definition.id = "agent123"
definition.name = "DeclarativeAgent"
definition.description = "desc"
definition.instructions = "test agent"
definition.tools = []
definition.model = "gpt-4o"
definition.temperature = 1.0
definition.top_p = 1.0
definition.metadata = {}
client.beta.assistants.create = AsyncMock(return_value=definition)
return client, definition
class SamplePlugin:
@kernel_function
def test_plugin(self, *args, **kwargs):
pass
class ResponseModelPydantic(BaseModel):
response: str
items: list[str]
class ResponseModelNonPydantic:
response: str
items: list[str]
@pytest.fixture
def mock_thread_messages():
class MockMessage:
def __init__(self, id, role, content, assistant_id=None):
self.id = id
self.role = role
self.content = content
self.assistant_id = assistant_id
return [
MockMessage(
id="test_message_id_1",
role="user",
content=[
TextContentBlock(
type="text",
text=Text(
value="Hello",
annotations=[
FilePathAnnotation(
type="file_path",
file_path=FilePath(file_id="test_file_id"),
end_index=5,
start_index=0,
text="Hello",
),
FileCitationAnnotation(
type="file_citation",
file_citation=FileCitation(file_id="test_file_id"),
text="Hello",
start_index=0,
end_index=5,
),
],
),
)
],
),
MockMessage(
id="test_message_id_2",
role="assistant",
content=[
ImageFileContentBlock(type="image_file", image_file=ImageFile(file_id="test_file_id", detail="auto"))
],
assistant_id="assistant_1",
),
]
async def test_open_ai_assistant_agent_init():
sample_prompt_template_config = PromptTemplateConfig(
template="template",
)
kernel_plugin = KernelPlugin(name="expected_plugin_name", description="expected_plugin_description")
client = AsyncMock(spec=AsyncOpenAI)
definition = AsyncMock(spec=Assistant)
definition.id = "agent123"
definition.name = "agentName"
definition.description = "desc"
definition.instructions = "test agent"
agent = AzureAssistantAgent(
client=client,
definition=definition,
arguments=KernelArguments(test="test"),
kernel=AsyncMock(spec=Kernel),
plugins=[SamplePlugin(), kernel_plugin],
polling_options=AsyncMock(spec=RunPollingOptions),
prompt_template_config=sample_prompt_template_config, # type: ignore
other_arg="test", # type: ignore
)
assert agent.id == "agent123"
assert agent.name == "agentName"
assert agent.description == "desc"
def test_azure_open_ai_settings_create_throws(azure_openai_unit_test_env):
with patch(
"semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings.AzureOpenAISettings.__init__"
) as mock_create:
mock_create.side_effect = ValidationError.from_exception_data("test", line_errors=[], input_type="python")
with pytest.raises(AgentInitializationException, match="Failed to create Azure OpenAI settings."):
_, _ = AzureAssistantAgent.setup_resources(api_key="test_api_key")
def test_open_ai_assistant_with_code_interpreter_tool():
tools, resources = AzureAssistantAgent.configure_code_interpreter_tool(file_ids=["file_id"])
assert tools is not None
assert resources is not None
def test_open_ai_assistant_with_file_search_tool():
tools, resources = AzureAssistantAgent.configure_file_search_tool(vector_store_ids=["vector_store_id"])
assert tools is not None
assert resources is not None
@pytest.mark.parametrize(
"model, json_schema_expected",
[
pytest.param(ResponseModelPydantic, True),
pytest.param(ResponseModelNonPydantic, True),
pytest.param({"type": "json_object"}, False),
pytest.param({"type": "json_schema", "json_schema": {"schema": {}}}, False),
],
)
def test_configure_response_format(model, json_schema_expected):
response_format = AzureAssistantAgent.configure_response_format(model)
assert response_format is not None
if json_schema_expected:
assert response_format["json_schema"] is not None # type: ignore
def test_configure_response_format_unexpected_type():
with pytest.raises(AgentInitializationException) as exc_info:
AzureAssistantAgent.configure_response_format({"type": "invalid_type"})
assert "Encountered unexpected response_format type" in str(exc_info.value)
def test_configure_response_format_json_schema_invalid_schema():
with pytest.raises(AgentInitializationException) as exc_info:
AzureAssistantAgent.configure_response_format({"type": "json_schema", "json_schema": "not_a_dict"})
assert "If response_format has type 'json_schema'" in str(exc_info.value)
def test_configure_response_format_invalid_input_type():
with pytest.raises(AgentInitializationException) as exc_info:
AzureAssistantAgent.configure_response_format(3) # type: ignore
assert "response_format must be a dictionary" in str(exc_info.value)
@pytest.mark.parametrize(
"arguments, include_args",
[
pytest.param({"extra_args": "extra_args"}, True),
pytest.param(None, False),
],
)
async def test_open_ai_assistant_agent_invoke(arguments, include_args):
client = AsyncMock(spec=AsyncOpenAI)
definition = AsyncMock(spec=Assistant)
definition.id = "agent123"
definition.name = "agentName"
definition.description = "desc"
definition.instructions = "test agent"
definition.tools = []
definition.model = "gpt-4o"
definition.response_format = {"type": "json_object"}
definition.temperature = 0.1
definition.top_p = 0.9
definition.metadata = {}
agent = AzureAssistantAgent(client=client, definition=definition)
mock_thread = AsyncMock(spec=AssistantAgentThread)
results = []
async def fake_invoke(*args, **kwargs):
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
kwargs = None
if include_args:
kwargs = arguments
with patch(
"semantic_kernel.agents.open_ai.assistant_thread_actions.AssistantThreadActions.invoke",
side_effect=fake_invoke,
):
async for item in agent.invoke(messages="test", thread=mock_thread, **(kwargs or {})):
results.append(item)
assert len(results) == 1
@pytest.mark.parametrize(
"arguments, include_args",
[
pytest.param({"extra_args": "extra_args"}, True),
pytest.param(None, False),
],
)
async def test_open_ai_assistant_agent_invoke_stream(arguments, include_args):
client = AsyncMock(spec=AsyncOpenAI)
definition = AsyncMock(spec=Assistant)
definition.id = "agent123"
definition.name = "agentName"
definition.description = "desc"
definition.instructions = "test agent"
agent = AzureAssistantAgent(client=client, definition=definition)
mock_thread = AsyncMock(spec=AssistantAgentThread)
results = []
async def fake_invoke(*args, **kwargs):
yield ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
kwargs = None
if include_args:
kwargs = arguments
with patch(
"semantic_kernel.agents.open_ai.assistant_thread_actions.AssistantThreadActions.invoke_stream",
side_effect=fake_invoke,
):
async for item in agent.invoke_stream(messages="test", thread=mock_thread, **(kwargs or {})):
results.append(item)
assert len(results) == 1
def test_open_ai_assistant_agent_get_channel_keys():
client = AsyncMock(spec=AsyncOpenAI)
definition = AsyncMock(spec=Assistant)
definition.id = "agent123"
definition.name = "agentName"
definition.description = "desc"
definition.instructions = "test agent"
agent = AzureAssistantAgent(client=client, definition=definition)
keys = list(agent.get_channel_keys())
assert len(keys) >= 3
@pytest.fixture
def mock_thread():
class MockThread:
id = "test_thread_id"
return MockThread()
async def test_open_ai_assistant_agent_create_channel(mock_thread):
from semantic_kernel.agents.channels.open_ai_assistant_channel import OpenAIAssistantChannel
client = AsyncMock(spec=AsyncOpenAI)
definition = AsyncMock(spec=Assistant)
definition.id = "agent123"
definition.name = "agentName"
definition.description = "desc"
definition.instructions = "test agent"
agent = AzureAssistantAgent(client=client, definition=definition)
client.beta = MagicMock()
client.beta.assistants = MagicMock()
client.beta.assistants.create = AsyncMock(return_value=definition)
client.beta.threads = MagicMock()
client.beta.threads.create = AsyncMock(return_value=mock_thread)
ch = await agent.create_channel()
assert isinstance(ch, OpenAIAssistantChannel)
assert ch.thread_id == "test_thread_id"
def test_create_openai_client(azure_openai_unit_test_env):
client, model = AzureAssistantAgent.setup_resources(api_key="test_api_key", default_headers={"user_agent": "test"})
assert client is not None
assert client.api_key == "test_api_key"
assert model is not None
def test_create_azure_openai_client(azure_openai_unit_test_env):
client, model = AzureAssistantAgent.setup_resources(
api_key="test_api_key", endpoint="https://test_endpoint.com", default_headers={"user_agent": "test"}
)
assert model is not None
assert client is not None
assert client.api_key == "test_api_key"
assert str(client.base_url) == "https://test_endpoint.com/openai/"
@pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_ENDPOINT"]], indirect=True)
async def test_retrieve_agent_missing_endpoint_throws(kernel, azure_openai_unit_test_env):
with pytest.raises(AgentInitializationException, match="Please provide an Azure OpenAI endpoint"):
_, _ = AzureAssistantAgent.setup_resources(
env_file_path="./", api_key="test_api_key", default_headers={"user_agent": "test"}
)
@pytest.mark.parametrize("exclude_list", [["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"]], indirect=True)
async def test_retrieve_agent_missing_chat_deployment_name_throws(kernel, azure_openai_unit_test_env):
with pytest.raises(AgentInitializationException, match="Please provide an Azure OpenAI deployment name"):
_, _ = AzureAssistantAgent.setup_resources(
env_file_path="./",
api_key="test_api_key",
endpoint="https://test_endpoint.com",
default_headers={"user_agent": "test"},
)
async def test_azure_assistant_agent_from_yaml_minimal(
azure_openai_unit_test_env, mock_azure_openai_client_and_definition
):
spec = """
type: azure_assistant
name: MinimalAgent
model:
id: ${AzureOpenAI:ChatModelId}
connection:
api_key: ${AzureOpenAI:ApiKey}
endpoint: ${AzureOpenAI:Endpoint}
"""
client, definition = mock_azure_openai_client_and_definition
definition.name = "MinimalAgent"
agent = await AgentRegistry.create_from_yaml(spec, client=client)
assert isinstance(agent, AzureAssistantAgent)
assert agent.name == "MinimalAgent"
assert agent.definition.model == "gpt-4o"
async def test_azure_assistant_agent_with_tools(azure_openai_unit_test_env, mock_azure_openai_client_and_definition):
spec = """
type: azure_assistant
name: CodeAgent
description: Uses code interpreter.
model:
id: ${AzureOpenAI:ChatModelId}
connection:
api_key: ${AzureOpenAI:ApiKey}
endpoint: ${AzureOpenAI:Endpoint}
tools:
- type: code_interpreter
options:
file_ids:
- ${AzureOpenAI:FileId1}
"""
client, definition = mock_azure_openai_client_and_definition
definition.name = "CodeAgent"
definition.tools = [{"type": "code_interpreter", "options": {"file_ids": ["file-123"]}}]
agent = await AgentRegistry.create_from_yaml(spec, client=client, extras={"AzureOpenAI:FileId1": "file-123"})
assert agent.name == "CodeAgent"
assert any(t["type"] == "code_interpreter" for t in agent.definition.tools)
async def test_azure_assistant_agent_with_inputs_outputs_template(
azure_openai_unit_test_env, mock_azure_openai_client_and_definition
):
spec = """
type: azure_assistant
name: StoryAgent
model:
id: ${AzureOpenAI:ChatModelId}
connection:
api_key: ${AzureOpenAI:ApiKey}
inputs:
topic:
description: The story topic.
required: true
default: AI
length:
description: The length of story.
required: true
default: 2
outputs:
output1:
description: The story.
template:
format: semantic-kernel
"""
client, definition = mock_azure_openai_client_and_definition
definition.name = "StoryAgent"
agent: AzureAssistantAgent = await AgentRegistry.create_from_yaml(spec, client=client)
assert agent.name == "StoryAgent"
assert agent.prompt_template.prompt_template_config.template_format == "semantic-kernel"
async def test_azure_assistant_agent_from_dict_missing_type():
data = {"name": "NoType"}
with pytest.raises(AgentInitializationException, match="Missing 'type'"):
await AgentRegistry.create_from_dict(data)
async def test_azure_assistant_agent_from_yaml_missing_required_fields():
spec = """
type: azure_assistant
"""
with pytest.raises(AgentInitializationException):
await AgentRegistry.create_from_yaml(spec)
async def test_agent_from_file_success(tmp_path, azure_openai_unit_test_env, mock_azure_openai_client_and_definition):
file_path = tmp_path / "spec.yaml"
file_path.write_text(
"""
type: azure_assistant
name: DeclarativeAgent
model:
id: ${AzureOpenAI:ChatModelId}
connection:
api_key: ${AzureOpenAI:ApiKey}
""",
encoding="utf-8",
)
client, _ = mock_azure_openai_client_and_definition
agent = await AgentRegistry.create_from_file(str(file_path), client=client)
assert agent.name == "DeclarativeAgent"
assert isinstance(agent, AzureAssistantAgent)
async def test_azure_assistant_agent_from_yaml_invalid_type():
spec = """
type: not_registered
name: ShouldFail
"""
with pytest.raises(AgentInitializationException, match="not registered"):
await AgentRegistry.create_from_yaml(spec)
@@ -0,0 +1,334 @@
# Copyright (c) Microsoft. All rights reserved.
import json
from typing import Any
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from openai import AsyncOpenAI
from openai.types.beta.assistant import Assistant, ToolResources, ToolResourcesCodeInterpreter, ToolResourcesFileSearch
from openai.types.beta.threads.file_citation_annotation import FileCitation, FileCitationAnnotation
from openai.types.beta.threads.file_path_annotation import FilePath, FilePathAnnotation
from openai.types.beta.threads.image_file import ImageFile
from openai.types.beta.threads.image_file_content_block import ImageFileContentBlock
from openai.types.beta.threads.text import Text
from openai.types.beta.threads.text_content_block import TextContentBlock
from semantic_kernel.agents.chat_completion.chat_completion_agent import ChatCompletionAgent
from semantic_kernel.agents.open_ai.openai_assistant_agent import OpenAIAssistantAgent
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.function_call_content import FunctionCallContent
from semantic_kernel.contents.text_content import TextContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions.agent_exceptions import AgentChatException
from semantic_kernel.functions.kernel_arguments import KernelArguments
from semantic_kernel.kernel import Kernel
@pytest.fixture
def mock_thread_messages():
class MockMessage:
def __init__(self, role, content, assistant_id=None):
self.role = role
self.content = content
self.assistant_id = assistant_id
return [
MockMessage(
role="user",
content=[
TextContentBlock(
type="text",
text=Text(
value="Hello",
annotations=[
FilePathAnnotation(
type="file_path",
file_path=FilePath(file_id="test_file_id"),
end_index=5,
start_index=0,
text="Hello",
),
FileCitationAnnotation(
type="file_citation",
file_citation=FileCitation(file_id="test_file_id"),
text="Hello",
start_index=0,
end_index=5,
),
],
),
)
],
),
MockMessage(
role="assistant",
content=[
ImageFileContentBlock(type="image_file", image_file=ImageFile(file_id="test_file_id", detail="auto"))
],
assistant_id="assistant_1",
),
]
@pytest.fixture
def mock_assistant():
return Assistant(
created_at=123456789,
object="assistant",
metadata={
"__run_options": json.dumps({
"max_completion_tokens": 100,
"max_prompt_tokens": 50,
"parallel_tool_calls_enabled": True,
"truncation_message_count": 10,
})
},
model="test_model",
description="test_description",
id="test_id",
instructions="test_instructions",
name="test_name",
tools=[{"type": "code_interpreter"}, {"type": "file_search"}], # type: ignore
temperature=0.7,
top_p=0.9,
response_format={"type": "json_object"}, # type: ignore
tool_resources=ToolResources(
code_interpreter=ToolResourcesCodeInterpreter(file_ids=["file1", "file2"]),
file_search=ToolResourcesFileSearch(vector_store_ids=["vector_store1"]),
),
)
async def test_receive_messages():
from semantic_kernel.agents.channels.open_ai_assistant_channel import OpenAIAssistantChannel
client = MagicMock(spec=AsyncOpenAI)
client.beta = AsyncMock()
thread_id = "test_thread"
channel = OpenAIAssistantChannel(client=client, thread_id=thread_id)
history = [
MagicMock(spec=ChatMessageContent, role=AuthorRole.USER, items=[TextContent(text="test")]) for _ in range(3)
]
with patch("semantic_kernel.agents.open_ai.assistant_content_generation.create_chat_message"):
await channel.receive(history) # type: ignore
async def test_invoke_agent():
from semantic_kernel.agents.channels.open_ai_assistant_channel import OpenAIAssistantChannel
client = AsyncMock(spec=AsyncOpenAI)
definition = AsyncMock(spec=Assistant)
definition.id = "agent123"
definition.name = "agentName"
definition.description = "desc"
definition.instructions = "test agent"
agent = OpenAIAssistantAgent(
client=client,
definition=definition,
arguments=KernelArguments(test="test"),
kernel=AsyncMock(spec=Kernel),
)
channel = OpenAIAssistantChannel(client=client, thread_id="test_thread_id")
async def mock_invoke_internal(*args, **kwargs):
for _ in range(3):
yield True, MagicMock(spec=ChatMessageContent)
results = []
with patch(
"semantic_kernel.agents.channels.open_ai_assistant_channel.AssistantThreadActions.invoke",
side_effect=mock_invoke_internal,
):
async for is_visible, message in channel.invoke(agent):
results.append((is_visible, message))
assert len(results) == 3
for is_visible, message in results:
assert is_visible is True
assert isinstance(message, ChatMessageContent)
async def test_invoke_agent_invalid_instance_throws():
from semantic_kernel.agents.channels.open_ai_assistant_channel import OpenAIAssistantChannel
client = MagicMock(spec=AsyncOpenAI)
thread_id = "test_thread"
agent = MagicMock(spec=ChatCompletionAgent)
agent._is_deleted = False
channel = OpenAIAssistantChannel(client=client, thread_id=thread_id)
with pytest.raises(AgentChatException, match=f"Agent is not of the expected type {type(OpenAIAssistantAgent)}."):
async for _, _ in channel.invoke(agent):
pass
async def test_invoke_streaming_agent():
from semantic_kernel.agents.channels.open_ai_assistant_channel import OpenAIAssistantChannel
client = AsyncMock(spec=AsyncOpenAI)
definition = AsyncMock(spec=Assistant)
definition.id = "agent123"
definition.name = "agentName"
definition.description = "desc"
definition.instructions = "test agent"
agent = OpenAIAssistantAgent(
client=client,
definition=definition,
arguments=KernelArguments(test="test"),
kernel=AsyncMock(spec=Kernel),
)
channel = OpenAIAssistantChannel(client=client, thread_id="test_thread_id")
results = []
async def mock_invoke_internal(*args, **kwargs):
for _ in range(3):
msg = MagicMock(spec=ChatMessageContent)
yield msg
results.append(msg)
with patch(
"semantic_kernel.agents.channels.open_ai_assistant_channel.AssistantThreadActions.invoke_stream",
side_effect=mock_invoke_internal,
):
async for message in channel.invoke_stream(agent, results):
assert message is not None
assert len(results) == 3
for message in results:
assert isinstance(message, ChatMessageContent)
async def test_invoke_streaming_agent_invalid_instance_throws():
from semantic_kernel.agents.channels.open_ai_assistant_channel import OpenAIAssistantChannel
client = MagicMock(spec=AsyncOpenAI)
thread_id = "test_thread"
agent = MagicMock(spec=ChatCompletionAgent)
agent._is_deleted = False
channel = OpenAIAssistantChannel(client=client, thread_id=thread_id)
with pytest.raises(AgentChatException, match=f"Agent is not of the expected type {type(OpenAIAssistantAgent)}."):
async for _ in channel.invoke_stream(agent, []):
pass
async def test_invoke_agent_wrong_type():
from semantic_kernel.agents.channels.open_ai_assistant_channel import OpenAIAssistantChannel
client = MagicMock(spec=AsyncOpenAI)
thread_id = "test_thread"
agent = MagicMock()
channel = OpenAIAssistantChannel(client=client, thread_id=thread_id)
with pytest.raises(AgentChatException, match="Agent is not of the expected type"):
async for _ in channel.invoke(agent):
pass
async def test_get_history(mock_thread_messages, mock_assistant, openai_unit_test_env):
from semantic_kernel.agents.channels.open_ai_assistant_channel import OpenAIAssistantChannel
async def mock_list_messages(*args, **kwargs) -> Any:
return MagicMock(data=mock_thread_messages)
async def mock_retrieve_assistant(*args, **kwargs) -> Any:
return mock_assistant
mock_client = MagicMock(spec=AsyncOpenAI)
mock_client.beta = MagicMock()
mock_client.beta.threads = MagicMock()
mock_client.beta.threads.messages = MagicMock()
mock_client.beta.threads.messages.list = AsyncMock(side_effect=mock_list_messages)
mock_client.beta.assistants = MagicMock()
mock_client.beta.assistants.retrieve = AsyncMock(side_effect=mock_retrieve_assistant)
thread_id = "test_thread"
channel = OpenAIAssistantChannel(client=mock_client, thread_id=thread_id)
results = []
async for content in channel.get_history():
results.append(content)
assert len(results) == 2
mock_client.beta.threads.messages.list.assert_awaited_once_with(thread_id=thread_id, limit=100, order="desc")
async def test_reset_channel(mock_thread_messages, mock_assistant, openai_unit_test_env):
from semantic_kernel.agents.channels.open_ai_assistant_channel import OpenAIAssistantChannel
async def mock_list_messages(*args, **kwargs) -> Any:
return MagicMock(data=mock_thread_messages)
async def mock_retrieve_assistant(*args, **kwargs) -> Any:
return mock_assistant
mock_client = MagicMock(spec=AsyncOpenAI)
mock_client.beta = MagicMock()
mock_client.beta.threads = MagicMock()
mock_client.beta.threads.messages = MagicMock()
mock_client.beta.threads.messages.list = AsyncMock(side_effect=mock_list_messages)
mock_client.beta.assistants = MagicMock()
mock_client.beta.assistants.retrieve = AsyncMock(side_effect=mock_retrieve_assistant)
mock_client.beta.threads.delete = AsyncMock()
thread_id = "test_thread"
channel = OpenAIAssistantChannel(client=mock_client, thread_id=thread_id)
results = []
async for content in channel.get_history():
results.append(content)
assert len(results) == 2
mock_client.beta.threads.messages.list.assert_awaited_once_with(thread_id=thread_id, limit=100, order="desc")
await channel.reset()
assert channel.thread_id is not None
async def test_reset_channel_error_throws_exception(mock_thread_messages, mock_assistant, openai_unit_test_env):
from semantic_kernel.agents.channels.open_ai_assistant_channel import OpenAIAssistantChannel
async def mock_list_messages(*args, **kwargs) -> Any:
return MagicMock(data=mock_thread_messages)
async def mock_retrieve_assistant(*args, **kwargs) -> Any:
return mock_assistant
mock_client = MagicMock(spec=AsyncOpenAI)
mock_client.beta = MagicMock()
mock_client.beta.threads = MagicMock()
mock_client.beta.threads.messages = MagicMock()
mock_client.beta.threads.messages.list = AsyncMock(side_effect=mock_list_messages)
mock_client.beta.assistants = MagicMock()
mock_client.beta.assistants.retrieve = AsyncMock(side_effect=mock_retrieve_assistant)
mock_client.beta.threads.delete = AsyncMock(side_effect=Exception("Test error"))
thread_id = "test_thread"
channel = OpenAIAssistantChannel(client=mock_client, thread_id=thread_id)
results = []
async for content in channel.get_history():
results.append(content)
assert len(results) == 2
mock_client.beta.threads.messages.list.assert_awaited_once_with(thread_id=thread_id, limit=100, order="desc")
with pytest.raises(Exception, match="Test error"):
await channel.reset()
async def test_channel_receive_fcc_skipped(openai_unit_test_env):
from semantic_kernel.agents.channels.open_ai_assistant_channel import OpenAIAssistantChannel
message = ChatMessageContent(role=AuthorRole.ASSISTANT, items=[FunctionCallContent(function_name="test_function")])
client = MagicMock(spec=AsyncOpenAI)
channel = OpenAIAssistantChannel(client=client, thread_id="test_thread")
await channel.receive([message])
@@ -0,0 +1,579 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from openai import AsyncOpenAI
from openai.types.beta.assistant import Assistant
from pydantic import BaseModel, ValidationError
from semantic_kernel.agents import AgentRegistry, AgentResponseItem, OpenAIAssistantAgent
from semantic_kernel.agents.open_ai.openai_assistant_agent import AssistantAgentThread
from semantic_kernel.agents.open_ai.run_polling_options import RunPollingOptions
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.function_call_content import FunctionCallContent
from semantic_kernel.contents.function_result_content import FunctionResultContent
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions.agent_exceptions import AgentInitializationException, AgentInvokeException
from semantic_kernel.functions.kernel_arguments import KernelArguments
from semantic_kernel.functions.kernel_function_decorator import kernel_function
from semantic_kernel.functions.kernel_plugin import KernelPlugin
from semantic_kernel.kernel import Kernel
from semantic_kernel.prompt_template.prompt_template_config import PromptTemplateConfig
@pytest.fixture
def mock_openai_client_and_definition():
client = AsyncMock(spec=AsyncOpenAI)
client.beta = MagicMock()
client.beta.assistants = MagicMock()
definition = AsyncMock(spec=Assistant)
definition.id = "agent123"
definition.name = "DeclarativeAgent"
definition.description = "desc"
definition.instructions = "test agent"
definition.tools = []
definition.model = "gpt-4o"
definition.temperature = 1.0
definition.top_p = 1.0
definition.metadata = {}
client.beta.assistants.create = AsyncMock(return_value=definition)
return client, definition
class SamplePlugin:
@kernel_function
def test_plugin(self, *args, **kwargs):
pass
class ResponseModelPydantic(BaseModel):
response: str
items: list[str]
class ResponseModelNonPydantic:
response: str
items: list[str]
async def test_open_ai_assistant_agent_init(openai_client, assistant_definition):
sample_prompt_template_config = PromptTemplateConfig(
template="template",
)
kernel_plugin = KernelPlugin(name="expected_plugin_name", description="expected_plugin_description")
agent = OpenAIAssistantAgent(
client=AsyncMock(spec=AsyncOpenAI),
definition=assistant_definition,
arguments=KernelArguments(test="test"),
kernel=AsyncMock(spec=Kernel),
plugins=[SamplePlugin(), kernel_plugin],
polling_options=AsyncMock(spec=RunPollingOptions),
prompt_template_config=sample_prompt_template_config,
other_arg="test",
)
assert agent.id == "agent123"
assert agent.name == "agentName"
assert agent.description == "desc"
def test_open_ai_settings_create_throws(openai_unit_test_env):
with patch(
"semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings.OpenAISettings.__init__"
) as mock_create:
mock_create.side_effect = ValidationError.from_exception_data("test", line_errors=[], input_type="python")
with pytest.raises(AgentInitializationException, match="Failed to create OpenAI settings."):
_, _ = OpenAIAssistantAgent.setup_resources(api_key="test_api_key")
def test_open_ai_assistant_with_code_interpreter_tool():
tools, resources = OpenAIAssistantAgent.configure_code_interpreter_tool(file_ids=["file_id"])
assert tools is not None
assert resources is not None
def test_open_ai_assistant_with_file_search_tool():
tools, resources = OpenAIAssistantAgent.configure_file_search_tool(vector_store_ids=["vector_store_id"])
assert tools is not None
assert resources is not None
@pytest.mark.parametrize(
"model, json_schema_expected",
[
pytest.param(ResponseModelPydantic, True),
pytest.param(ResponseModelNonPydantic, True),
pytest.param({"type": "json_object"}, False),
pytest.param({"type": "json_schema", "json_schema": {"schema": {}}}, False),
],
)
def test_configure_response_format(model, json_schema_expected):
response_format = OpenAIAssistantAgent.configure_response_format(model)
assert response_format is not None
if json_schema_expected:
assert response_format["json_schema"] is not None # type: ignore
def test_configure_response_format_unexpected_type():
with pytest.raises(AgentInitializationException) as exc_info:
OpenAIAssistantAgent.configure_response_format({"type": "invalid_type"})
assert "Encountered unexpected response_format type" in str(exc_info.value)
def test_configure_response_format_json_schema_invalid_schema():
with pytest.raises(AgentInitializationException) as exc_info:
OpenAIAssistantAgent.configure_response_format({"type": "json_schema", "json_schema": "not_a_dict"})
assert "If response_format has type 'json_schema'" in str(exc_info.value)
def test_configure_response_format_invalid_input_type():
with pytest.raises(AgentInitializationException) as exc_info:
OpenAIAssistantAgent.configure_response_format(3) # type: ignore
assert "response_format must be a dictionary" in str(exc_info.value)
@pytest.mark.parametrize(
"arguments, include_args",
[
pytest.param({"extra_args": "extra_args"}, True),
pytest.param(None, False),
],
)
async def test_open_ai_assistant_agent_get_response(arguments, include_args, openai_client, assistant_definition):
agent = OpenAIAssistantAgent(client=openai_client, definition=assistant_definition)
mock_thread = AsyncMock(spec=AssistantAgentThread)
async def fake_invoke(*args, **kwargs):
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
kwargs = None
if include_args:
kwargs = arguments
with patch(
"semantic_kernel.agents.open_ai.assistant_thread_actions.AssistantThreadActions.invoke",
side_effect=fake_invoke,
):
response = await agent.get_response(messages="test", thread=mock_thread, **(kwargs or {}))
assert response is not None
assert response.message.content == "content"
assert response.thread is not None
@pytest.mark.parametrize(
"arguments, include_args",
[
pytest.param({"extra_args": "extra_args"}, True),
pytest.param(None, False),
],
)
async def test_open_ai_assistant_agent_get_response_exception(
arguments, include_args, openai_client, assistant_definition
):
agent = OpenAIAssistantAgent(client=openai_client, definition=assistant_definition)
mock_thread = AsyncMock(spec=AssistantAgentThread)
async def fake_invoke(*args, **kwargs):
yield False, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
kwargs = None
if include_args:
kwargs = arguments
with (
patch(
"semantic_kernel.agents.open_ai.assistant_thread_actions.AssistantThreadActions.invoke",
side_effect=fake_invoke,
),
pytest.raises(AgentInvokeException),
):
await agent.get_response(messages="test", thread=mock_thread, **(kwargs or {}))
@pytest.mark.parametrize(
"arguments, include_args",
[
pytest.param({"extra_args": "extra_args"}, True),
pytest.param(None, False),
],
)
async def test_open_ai_assistant_agent_invoke(arguments, include_args, openai_client, assistant_definition):
agent = OpenAIAssistantAgent(client=openai_client, definition=assistant_definition)
mock_thread = AsyncMock(spec=AssistantAgentThread)
results = []
async def fake_invoke(*args, **kwargs):
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
kwargs = None
if include_args:
kwargs = arguments
with patch(
"semantic_kernel.agents.open_ai.assistant_thread_actions.AssistantThreadActions.invoke",
side_effect=fake_invoke,
):
async for item in agent.invoke(messages="test", thread=mock_thread, **(kwargs or {})):
results.append(item)
assert len(results) == 1
async def test_open_ai_assistant_agent_invoke_message_ordering(openai_client, assistant_definition):
agent = OpenAIAssistantAgent(client=openai_client, definition=assistant_definition)
mock_thread = AsyncMock(spec=AssistantAgentThread)
tool_call_msg = ChatMessageContent(
role=AuthorRole.ASSISTANT,
content="",
items=[FunctionCallContent(name="ToolA", arguments="{}")],
)
tool_result_msg = ChatMessageContent(
role=AuthorRole.ASSISTANT,
content="",
items=[FunctionResultContent(name="ToolA", result="$9.99", id="func_1")],
)
assistant_msg = ChatMessageContent(role=AuthorRole.ASSISTANT, content="Here is your answer.")
emitted_callback_messages = []
yielded_messages = []
async def on_intermediate_message(msg: ChatMessageContent):
emitted_callback_messages.append(msg)
async def fake_invoke(*args, **kwargs):
yield False, tool_call_msg
yield False, tool_result_msg
yield True, assistant_msg
with patch(
"semantic_kernel.agents.open_ai.assistant_thread_actions.AssistantThreadActions.invoke",
side_effect=fake_invoke,
):
async for item in agent.invoke(
messages="test", thread=mock_thread, on_intermediate_message=on_intermediate_message
):
yielded_messages.append(item)
assert emitted_callback_messages == [tool_call_msg, tool_result_msg]
assert yielded_messages == [AgentResponseItem(message=assistant_msg, thread=mock_thread)]
@pytest.mark.parametrize(
"arguments, include_args",
[
pytest.param({"extra_args": "extra_args"}, True),
pytest.param(None, False),
],
)
async def test_open_ai_assistant_agent_invoke_stream(arguments, include_args, openai_client, assistant_definition):
agent = OpenAIAssistantAgent(client=openai_client, definition=assistant_definition)
mock_thread = AsyncMock(spec=AssistantAgentThread)
results = []
async def fake_invoke(*args, **kwargs):
yield ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
kwargs = None
if include_args:
kwargs = arguments
with patch(
"semantic_kernel.agents.open_ai.assistant_thread_actions.AssistantThreadActions.invoke_stream",
side_effect=fake_invoke,
):
async for item in agent.invoke_stream(messages="test", thread=mock_thread, **(kwargs or {})):
results.append(item)
assert len(results) == 1
@pytest.mark.parametrize(
"arguments, include_args",
[
pytest.param({"extra_args": "extra_args"}, True),
pytest.param(None, False),
],
)
async def test_open_ai_assistant_agent_invoke_stream_with_on_new_message_callback(
arguments, include_args, openai_client, assistant_definition
):
agent = OpenAIAssistantAgent(client=openai_client, definition=assistant_definition)
mock_thread = AsyncMock(spec=AssistantAgentThread)
results = []
final_chat_history = ChatHistory()
async def handle_stream_completion(message: ChatMessageContent) -> None:
final_chat_history.add_message(message)
tool_msg = ChatMessageContent(
role=AuthorRole.ASSISTANT, content="", items=[FunctionCallContent(name="ToolA", arguments="{}")]
)
streamed_msg = StreamingChatMessageContent(role=AuthorRole.ASSISTANT, content="fake content", choice_index=0)
async def fake_invoke(*args, output_messages=None, **kwargs):
if output_messages is not None:
output_messages.append(tool_msg)
yield streamed_msg
kwargs = arguments if include_args else {}
with patch(
"semantic_kernel.agents.open_ai.assistant_thread_actions.AssistantThreadActions.invoke_stream",
side_effect=fake_invoke,
):
async for item in agent.invoke_stream(
messages="test", thread=mock_thread, on_intermediate_message=handle_stream_completion, **kwargs
):
results.append(item)
assert len(results) == 1
assert results[0].message.content == "fake content"
assert len(final_chat_history.messages) == 1
assert final_chat_history.messages[0] == tool_msg
def test_open_ai_assistant_agent_get_channel_keys(openai_client, assistant_definition):
agent = OpenAIAssistantAgent(client=openai_client, definition=assistant_definition)
keys = list(agent.get_channel_keys())
assert len(keys) >= 3
async def test_open_ai_assistant_agent_create_channel(openai_client, assistant_definition):
from semantic_kernel.agents.channels.open_ai_assistant_channel import OpenAIAssistantChannel
agent = OpenAIAssistantAgent(client=openai_client, definition=assistant_definition)
ch = await agent.create_channel()
assert isinstance(ch, OpenAIAssistantChannel)
assert ch.thread_id == "test_thread_id"
def test_create_openai_client(openai_unit_test_env):
client, model = OpenAIAssistantAgent.setup_resources(env_file_path="./", default_headers={"user_agent": "test"})
assert client is not None
assert client.api_key == "test_api_key"
assert model is not None
@pytest.mark.parametrize("exclude_list", [["OPENAI_API_KEY"]], indirect=True)
async def test_open_ai_agent_missing_api_key_throws(kernel, openai_unit_test_env):
with pytest.raises(AgentInitializationException, match="The OpenAI API key is required."):
_, _ = OpenAIAssistantAgent.setup_resources(env_file_path="./", default_headers={"user_agent": "test"})
@pytest.mark.parametrize("exclude_list", [["OPENAI_CHAT_MODEL_ID"]], indirect=True)
async def test_open_ai_agent_missing_chat_deployment_name_throws(kernel, openai_unit_test_env):
with pytest.raises(AgentInitializationException, match="The OpenAI model ID is required."):
_, _ = OpenAIAssistantAgent.setup_resources(
env_file_path="./",
api_key="test_api_key",
default_headers={"user_agent": "test"},
)
async def test_openai_assistant_agent_from_yaml_minimal(openai_unit_test_env, mock_openai_client_and_definition):
spec = """
type: openai_assistant
name: MinimalAgent
model:
id: ${OpenAI:ChatModelId}
connection:
api_key: ${OpenAI:ApiKey}
"""
client, definition = mock_openai_client_and_definition
definition.name = "MinimalAgent"
agent = await AgentRegistry.create_from_yaml(spec, client=client)
assert isinstance(agent, OpenAIAssistantAgent)
assert agent.name == "MinimalAgent"
assert agent.definition.model == "gpt-4o"
async def test_openai_assistant_agent_with_tools(openai_unit_test_env, mock_openai_client_and_definition):
spec = """
type: openai_assistant
name: CodeAgent
description: Uses code interpreter.
model:
id: ${OpenAI:ChatModelId}
connection:
api_key: ${OpenAI:ApiKey}
tools:
- type: code_interpreter
options:
file_ids:
- ${OpenAI:FileId1}
"""
client, definition = mock_openai_client_and_definition
definition.name = "CodeAgent"
definition.tools = [{"type": "code_interpreter", "options": {"file_ids": ["file-123"]}}]
agent = await AgentRegistry.create_from_yaml(spec, client=client, extras={"OpenAI:FileId1": "file-123"})
assert agent.name == "CodeAgent"
assert any(t["type"] == "code_interpreter" for t in agent.definition.tools)
async def test_openai_assistant_agent_with_inputs_outputs_template(
openai_unit_test_env, mock_openai_client_and_definition
):
spec = """
type: openai_assistant
name: StoryAgent
model:
id: ${OpenAI:ChatModelId}
connection:
api_key: ${OpenAI:ApiKey}
inputs:
topic:
description: The story topic.
required: true
default: AI
length:
description: The length of story.
required: true
default: 2
outputs:
output1:
description: The story.
template:
format: semantic-kernel
"""
client, definition = mock_openai_client_and_definition
definition.name = "StoryAgent"
agent: OpenAIAssistantAgent = await AgentRegistry.create_from_yaml(spec, client=client)
assert agent.name == "StoryAgent"
assert agent.prompt_template.prompt_template_config.template_format == "semantic-kernel"
async def test_openai_assistant_agent_from_dict_missing_type():
data = {"name": "NoType"}
with pytest.raises(AgentInitializationException, match="Missing 'type'"):
await AgentRegistry.create_from_dict(data)
async def test_openai_assistant_agent_from_yaml_missing_required_fields():
spec = """
type: openai_assistant
"""
with pytest.raises(AgentInitializationException):
await AgentRegistry.create_from_yaml(spec)
async def test_agent_from_file_success(tmp_path, openai_unit_test_env, mock_openai_client_and_definition):
file_path = tmp_path / "spec.yaml"
file_path.write_text(
"""
type: openai_assistant
name: DeclarativeAgent
model:
id: ${OpenAI:ChatModelId}
connection:
api_key: ${OpenAI:ApiKey}
""",
encoding="utf-8",
)
client, _ = mock_openai_client_and_definition
agent = await AgentRegistry.create_from_file(str(file_path), client=client)
assert agent.name == "DeclarativeAgent"
assert isinstance(agent, OpenAIAssistantAgent)
async def test_openai_assistant_agent_from_yaml_invalid_type():
spec = """
type: not_registered
name: ShouldFail
"""
with pytest.raises(AgentInitializationException, match="not registered"):
await AgentRegistry.create_from_yaml(spec)
async def test_openai_assistant_agent_get_response_passes_function_choice_behavior(openai_client, assistant_definition):
agent = OpenAIAssistantAgent(client=openai_client, definition=assistant_definition)
thread = AsyncMock(spec=AssistantAgentThread)
fcb = FunctionChoiceBehavior.Auto()
captured_kwargs = {}
async def fake_invoke(*args, **kwargs):
captured_kwargs.update(kwargs)
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
with patch(
"semantic_kernel.agents.open_ai.assistant_thread_actions.AssistantThreadActions.invoke",
side_effect=fake_invoke,
):
await agent.get_response(messages="message", thread=thread, function_choice_behavior=fcb)
assert captured_kwargs.get("function_choice_behavior") is fcb
async def test_openai_assistant_agent_invoke_passes_function_choice_behavior(openai_client, assistant_definition):
agent = OpenAIAssistantAgent(client=openai_client, definition=assistant_definition)
thread = AsyncMock(spec=AssistantAgentThread)
fcb = FunctionChoiceBehavior.Auto()
captured_kwargs = {}
async def fake_invoke(*args, **kwargs):
captured_kwargs.update(kwargs)
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
with patch(
"semantic_kernel.agents.open_ai.assistant_thread_actions.AssistantThreadActions.invoke",
side_effect=fake_invoke,
):
async for _ in agent.invoke(messages="message", thread=thread, function_choice_behavior=fcb):
pass
assert captured_kwargs.get("function_choice_behavior") is fcb
async def test_openai_assistant_agent_invoke_stream_passes_function_choice_behavior(
openai_client, assistant_definition
):
agent = OpenAIAssistantAgent(client=openai_client, definition=assistant_definition)
thread = AsyncMock(spec=AssistantAgentThread)
fcb = FunctionChoiceBehavior.Auto()
captured_kwargs = {}
async def fake_invoke(*args, **kwargs):
captured_kwargs.update(kwargs)
yield ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
with patch(
"semantic_kernel.agents.open_ai.assistant_thread_actions.AssistantThreadActions.invoke_stream",
side_effect=fake_invoke,
):
async for _ in agent.invoke_stream(messages="message", thread=thread, function_choice_behavior=fcb):
pass
assert captured_kwargs.get("function_choice_behavior") is fcb
async def test_openai_assistant_agent_get_response_no_fcb_passes_none(openai_client, assistant_definition):
agent = OpenAIAssistantAgent(client=openai_client, definition=assistant_definition)
thread = AsyncMock(spec=AssistantAgentThread)
captured_kwargs = {}
async def fake_invoke(*args, **kwargs):
captured_kwargs.update(kwargs)
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
with patch(
"semantic_kernel.agents.open_ai.assistant_thread_actions.AssistantThreadActions.invoke",
side_effect=fake_invoke,
):
await agent.get_response(messages="message", thread=thread)
assert captured_kwargs.get("function_choice_behavior") is None
@@ -0,0 +1,366 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, patch
import pytest
from openai import AsyncOpenAI
from pydantic import BaseModel, ValidationError
from semantic_kernel.agents import AgentRegistry
from semantic_kernel.agents.open_ai.openai_responses_agent import OpenAIResponsesAgent, ResponsesAgentThread
from semantic_kernel.agents.open_ai.run_polling_options import RunPollingOptions
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions.agent_exceptions import AgentInitializationException, AgentInvokeException
from semantic_kernel.functions.kernel_arguments import KernelArguments
from semantic_kernel.functions.kernel_function_decorator import kernel_function
from semantic_kernel.functions.kernel_plugin import KernelPlugin
from semantic_kernel.kernel import Kernel
from semantic_kernel.prompt_template.prompt_template_config import PromptTemplateConfig
@pytest.fixture
def mock_openai_client():
return AsyncMock(spec=AsyncOpenAI)
class SamplePlugin:
@kernel_function
def test_plugin(self, *args, **kwargs):
pass
class ResponseModelPydantic(BaseModel):
response: str
items: list[str]
class ResponseModelNonPydantic:
response: str
items: list[str]
async def test_open_ai_assistant_agent_init():
sample_prompt_template_config = PromptTemplateConfig(
template="template",
)
kernel_plugin = KernelPlugin(name="expected_plugin_name", description="expected_plugin_description")
agent = OpenAIResponsesAgent(
ai_model_id="model_id",
id="agent123",
name="agentName",
description="desc",
client=AsyncMock(spec=AsyncOpenAI),
arguments=KernelArguments(test="test"),
kernel=AsyncMock(spec=Kernel),
plugins=[SamplePlugin(), kernel_plugin],
polling_options=AsyncMock(spec=RunPollingOptions),
prompt_template_config=sample_prompt_template_config,
other_arg="test",
)
assert agent.id == "agent123"
assert agent.name == "agentName"
assert agent.description == "desc"
def test_open_ai_settings_create_throws(openai_unit_test_env):
with patch(
"semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings.OpenAISettings.__init__"
) as mock_create:
mock_create.side_effect = ValidationError.from_exception_data("test", line_errors=[], input_type="python")
with pytest.raises(AgentInitializationException, match="Failed to create OpenAI settings."):
_, _ = OpenAIResponsesAgent.setup_resources(api_key="test_api_key")
def test_open_ai_assistant_with_file_search_tool():
tools, resources = OpenAIResponsesAgent.configure_file_search_tool(vector_store_ids=["vector_store_id"])
assert tools is not None
assert resources is not None
@pytest.mark.parametrize(
"model, json_schema_expected",
[
pytest.param(ResponseModelPydantic, True),
pytest.param(ResponseModelNonPydantic, True),
pytest.param({"type": "json_object"}, False),
pytest.param({"type": "json_schema", "json_schema": {"schema": {}}}, False),
],
)
def test_configure_response_format(model, json_schema_expected):
response_format = OpenAIResponsesAgent.configure_response_format(model)
assert response_format is not None
if json_schema_expected:
assert response_format["format"]["schema"] is not None # type: ignore
def test_configure_response_format_unexpected_type():
with pytest.raises(AgentInitializationException) as exc_info:
OpenAIResponsesAgent.configure_response_format({"type": "invalid_type"})
assert "Encountered unexpected response_format type" in str(exc_info.value)
def test_configure_response_format_json_schema_invalid_schema():
with pytest.raises(AgentInitializationException) as exc_info:
OpenAIResponsesAgent.configure_response_format({"type": "json_schema", "json_schema": "not_a_dict"})
assert "If response_format has type 'json_schema'" in str(exc_info.value)
def test_configure_response_format_invalid_input_type():
with pytest.raises(AgentInitializationException) as exc_info:
OpenAIResponsesAgent.configure_response_format(3) # type: ignore
assert "response_format must be a dictionary" in str(exc_info.value)
@pytest.mark.parametrize(
"arguments, include_args",
[
pytest.param({"extra_args": "extra_args"}, True),
pytest.param(None, False),
],
)
async def test_openai_responses_agent_get_response(arguments, include_args):
agent = OpenAIResponsesAgent(client=AsyncMock(spec=AsyncOpenAI), ai_model_id="model_id")
mock_thread = AsyncMock(spec=ResponsesAgentThread)
async def fake_invoke(*args, **kwargs):
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
kwargs = None
if include_args:
kwargs = arguments
with patch(
"semantic_kernel.agents.open_ai.responses_agent_thread_actions.ResponsesAgentThreadActions.invoke",
side_effect=fake_invoke,
):
response = await agent.get_response(messages="test", thread=mock_thread, **(kwargs or {}))
assert response is not None
assert response.message.content == "content"
assert response.thread is not None
@pytest.mark.parametrize(
"arguments, include_args",
[
pytest.param({"extra_args": "extra_args"}, True),
pytest.param(None, False),
],
)
async def test_openai_responses_agent_get_response_exception(arguments, include_args):
agent = OpenAIResponsesAgent(client=AsyncMock(spec=AsyncOpenAI), ai_model_id="model_id")
mock_thread = AsyncMock(spec=ResponsesAgentThread)
async def fake_invoke(*args, **kwargs):
yield False, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
kwargs = None
if include_args:
kwargs = arguments
with (
patch(
"semantic_kernel.agents.open_ai.responses_agent_thread_actions.ResponsesAgentThreadActions.invoke",
side_effect=fake_invoke,
),
pytest.raises(AgentInvokeException),
):
await agent.get_response(messages="test", thread=mock_thread, **(kwargs or {}))
@pytest.mark.parametrize(
"arguments, include_args",
[
pytest.param({"extra_args": "extra_args"}, True),
pytest.param(None, False),
],
)
async def test_openai_responses_agent_invoke(arguments, include_args):
agent = OpenAIResponsesAgent(client=AsyncMock(spec=AsyncOpenAI), ai_model_id="model_id")
mock_thread = AsyncMock(spec=ResponsesAgentThread)
results = []
async def fake_invoke(*args, **kwargs):
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
kwargs = None
if include_args:
kwargs = arguments
with patch(
"semantic_kernel.agents.open_ai.responses_agent_thread_actions.ResponsesAgentThreadActions.invoke",
side_effect=fake_invoke,
):
async for item in agent.invoke(messages="test", thread=mock_thread, **(kwargs or {})):
results.append(item)
assert len(results) == 1
@pytest.mark.parametrize(
"arguments, include_args",
[
pytest.param({"extra_args": "extra_args"}, True),
pytest.param(None, False),
],
)
async def test_openai_responses_agent_invoke_stream(arguments, include_args):
agent = OpenAIResponsesAgent(client=AsyncMock(spec=AsyncOpenAI), ai_model_id="model_id")
mock_thread = AsyncMock(spec=ResponsesAgentThread)
results = []
async def fake_invoke(*args, **kwargs):
yield ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
kwargs = None
if include_args:
kwargs = arguments
with patch(
"semantic_kernel.agents.open_ai.responses_agent_thread_actions.ResponsesAgentThreadActions.invoke_stream",
side_effect=fake_invoke,
):
async for item in agent.invoke_stream(messages="test", thread=mock_thread, **(kwargs or {})):
results.append(item)
assert len(results) == 1
def test_create_openai_client(openai_unit_test_env):
client, model = OpenAIResponsesAgent.setup_resources(env_file_path="./", default_headers={"user_agent": "test"})
assert client is not None
assert client.api_key == "test_api_key"
assert model is not None
@pytest.mark.parametrize("exclude_list", [["OPENAI_API_KEY"]], indirect=True)
async def test_open_ai_agent_missing_api_key_throws(kernel, openai_unit_test_env):
with pytest.raises(AgentInitializationException, match="The OpenAI API key is required."):
_, _ = OpenAIResponsesAgent.setup_resources(env_file_path="./", default_headers={"user_agent": "test"})
@pytest.mark.parametrize("exclude_list", [["OPENAI_RESPONSES_MODEL_ID"]], indirect=True)
async def test_open_ai_agent_missing_chat_deployment_name_throws(kernel, openai_unit_test_env):
with pytest.raises(AgentInitializationException, match="The OpenAI Responses model ID is required."):
_, _ = OpenAIResponsesAgent.setup_resources(
env_file_path="./",
api_key="test_api_key",
default_headers={"user_agent": "test"},
)
async def test_openai_assistant_agent_from_yaml_minimal(openai_unit_test_env, mock_openai_client):
spec = """
type: openai_responses
name: MinimalAgent
model:
id: ${OpenAI:ChatModelId}
connection:
api_key: ${OpenAI:ApiKey}
"""
client = mock_openai_client
agent: OpenAIResponsesAgent = await AgentRegistry.create_from_yaml(spec, client=client)
assert isinstance(agent, OpenAIResponsesAgent)
assert agent.name == "MinimalAgent"
assert agent.ai_model_id == openai_unit_test_env.get("OPENAI_RESPONSES_MODEL_ID")
async def test_openai_assistant_agent_with_tools(openai_unit_test_env, mock_openai_client):
spec = """
type: openai_responses
name: FileSearchAgent
description: Uses file search.
model:
id: ${OpenAI:ChatModelId}
connection:
api_key: ${OpenAI:ApiKey}
tools:
- type: file_search
description: File search for document retrieval.
options:
vector_store_ids:
- ${OpenAI:VectorStoreId}
"""
client = mock_openai_client
agent: OpenAIResponsesAgent = await AgentRegistry.create_from_yaml(
spec, client=client, extras={"OpenAI:VectorStoreId": "vector-store-123"}
)
assert agent.name == "FileSearchAgent"
assert any(t["type"] == "file_search" for t in agent.tools)
async def test_openai_assistant_agent_with_inputs_outputs_template(openai_unit_test_env, mock_openai_client):
spec = """
type: openai_responses
name: StoryAgent
model:
id: ${OpenAI:ChatModelId}
connection:
api_key: ${OpenAI:ApiKey}
inputs:
topic:
description: The story topic.
required: true
default: AI
length:
description: The length of story.
required: true
default: 2
outputs:
output1:
description: The story.
template:
format: semantic-kernel
"""
client = mock_openai_client
agent: OpenAIResponsesAgent = await AgentRegistry.create_from_yaml(spec, client=client)
assert agent.name == "StoryAgent"
assert agent.prompt_template.prompt_template_config.template_format == "semantic-kernel"
async def test_openai_assistant_agent_from_dict_missing_type():
data = {"name": "NoType"}
with pytest.raises(AgentInitializationException, match="Missing 'type'"):
await AgentRegistry.create_from_dict(data)
async def test_openai_assistant_agent_from_yaml_missing_required_fields():
spec = """
type: openai_responses
"""
with pytest.raises(AgentInitializationException):
await AgentRegistry.create_from_yaml(spec)
async def test_agent_from_file_success(tmp_path, openai_unit_test_env, mock_openai_client):
file_path = tmp_path / "spec.yaml"
file_path.write_text(
"""
type: openai_responses
name: DeclarativeAgent
model:
id: ${OpenAI:ChatModelId}
connection:
api_key: ${OpenAI:ApiKey}
""",
encoding="utf-8",
)
client = mock_openai_client
agent: OpenAIResponsesAgent = await AgentRegistry.create_from_file(str(file_path), client=client)
assert agent.name == "DeclarativeAgent"
assert isinstance(agent, OpenAIResponsesAgent)
async def test_openai_assistant_agent_from_yaml_invalid_type():
spec = """
type: not_registered
name: ShouldFail
"""
with pytest.raises(AgentInitializationException, match="not registered"):
await AgentRegistry.create_from_yaml(spec)
@@ -0,0 +1,568 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from openai._streaming import AsyncStream
from openai.types.responses import ResponseFunctionToolCall
from openai.types.responses.response import Response
from openai.types.responses.response_output_item_added_event import ResponseOutputItemAddedEvent
from openai.types.responses.response_output_item_done_event import ResponseOutputItemDoneEvent
from openai.types.responses.response_output_message import ResponseOutputMessage
from openai.types.responses.response_output_text import ResponseOutputText
from openai.types.responses.response_stream_event import ResponseStreamEvent
from openai.types.responses.response_text_delta_event import Logprob, ResponseTextDeltaEvent
from semantic_kernel.agents.open_ai.openai_responses_agent import OpenAIResponsesAgent
from semantic_kernel.agents.open_ai.responses_agent_thread_actions import ResponsesAgentThreadActions
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from semantic_kernel.contents.streaming_text_content import StreamingTextContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.functions import KernelArguments
@pytest.fixture
def mock_agent():
agent = AsyncMock(spec=OpenAIResponsesAgent)
agent.ai_model_id = "test-model-id"
agent.name = "test-agent"
agent.polling_options = MagicMock()
agent.polling_options.default_polling_interval.total_seconds.return_value = 0.0001
agent.tools = []
agent.text = "auto"
agent.temperature = 0.7
agent.top_p = 1.0
agent.metadata = {}
agent.format_instructions = AsyncMock(return_value="base instructions")
agent.kernel = MagicMock()
agent.polling_options.run_polling_timeout.total_seconds.return_value = 5
agent.polling_options.default_polling_interval.total_seconds.return_value = 1
return agent
@pytest.fixture
def mock_response():
response = MagicMock(spec=Response)
response.status = "completed"
response.output = []
response.id = "fake-response-id"
response.error = None
response.incomplete_details = None
response.created_at = 10303039393
response.usage = None
return response
@pytest.fixture
def mock_chat_history():
history = MagicMock()
history.messages = [ChatMessageContent(role=AuthorRole.USER, content="Hello")]
return history
@pytest.fixture
def mock_thread():
thread = MagicMock()
thread._chat_history.messages = []
return thread
async def test_invoke_no_function_calls(mock_agent, mock_response, mock_chat_history, mock_thread):
async def mock_get_response(*args, **kwargs):
return mock_response
with patch.object(ResponsesAgentThreadActions, "_get_response", new=mock_get_response):
results = []
async for is_visible, msg in ResponsesAgentThreadActions.invoke(
agent=mock_agent,
chat_history=mock_chat_history,
thread=mock_thread,
store_enabled=True,
function_choice_behavior=MagicMock(),
):
results.append((is_visible, msg))
assert len(results) == 1
is_visible, final_msg = results[0]
assert is_visible is True
assert final_msg.role == AuthorRole.ASSISTANT
async def test_invoke_raises_on_failed_response(mock_agent, mock_chat_history, mock_thread):
mock_failed_response = MagicMock(spec=Response)
mock_failed_response.status = "failed"
mock_failed_response.error = MagicMock()
mock_failed_response.error.message = "some error"
mock_failed_response.incomplete_details = None
mock_failed_response.id = "fake-failed-response-id"
async def mock_get_response(*args, **kwargs):
return mock_failed_response
with (
patch.object(ResponsesAgentThreadActions, "_get_response", new=mock_get_response),
pytest.raises(Exception, match="Run failed with status: `failed`"),
):
async for _ in ResponsesAgentThreadActions.invoke(
agent=mock_agent,
chat_history=mock_chat_history,
thread=mock_thread,
store_enabled=True,
function_choice_behavior=MagicMock(),
):
pass
async def test_invoke_reaches_maximum_attempts(mock_agent, mock_chat_history, mock_thread):
call_counter = 0
response_with_tool_call = MagicMock(spec=Response)
response_with_tool_call.status = "completed"
response_with_tool_call.id = "fake-response-id"
response_with_tool_call.output = [
ResponseFunctionToolCall(
id="tool_call_id",
call_id="call_id",
name="test_function",
arguments='{"some_arg": 123}',
type="function_call",
)
]
response_with_tool_call.error = None
response_with_tool_call.incomplete_details = None
response_with_tool_call.created_at = 123456
response_with_tool_call.usage = None
response_with_tool_call.role = "assistant"
final_response = MagicMock(spec=Response)
final_response.status = "completed"
final_response.id = "fake-final-response-id"
final_response.output = []
final_response.error = None
final_response.incomplete_details = None
final_response.created_at = 123456
final_response.usage = None
final_response.role = "assistant"
async def mock_invoke_fc(*args, **kwargs):
return MagicMock(terminate=False)
mock_agent.kernel.invoke_function_call = MagicMock(side_effect=mock_invoke_fc)
async def mock_get_response(*args, **kwargs):
nonlocal call_counter
if call_counter < 3:
call_counter += 1
return response_with_tool_call
return final_response
with patch.object(ResponsesAgentThreadActions, "_get_response", new=mock_get_response):
messages = []
async for _, msg in ResponsesAgentThreadActions.invoke(
agent=mock_agent,
chat_history=mock_chat_history,
thread=mock_thread,
store_enabled=True,
function_choice_behavior=MagicMock(maximum_auto_invoke_attempts=3),
):
messages.append(msg)
assert messages is not None
async def test_invoke_with_function_calls(mock_agent, mock_chat_history, mock_thread):
initial_response = MagicMock(spec=Response)
initial_response.status = "completed"
initial_response.id = "fake-response-id"
initial_response.output = [
ResponseFunctionToolCall(
id="tool_call_id",
call_id="call_id",
name="test_function",
arguments='{"some_arg": 123}',
type="function_call",
)
]
initial_response.error = None
initial_response.incomplete_details = None
initial_response.created_at = 123456
initial_response.usage = None
initial_response.role = "assistant"
final_response = MagicMock(spec=Response)
final_response.status = "completed"
final_response.id = "fake-final-response-id"
final_response.output = []
final_response.error = None
final_response.incomplete_details = None
final_response.created_at = 123456
final_response.usage = None
final_response.role = "assistant"
responses = [initial_response, final_response]
async def mock_invoke_fc(*args, **kwargs):
return MagicMock(terminate=False)
mock_agent.kernel.invoke_function_call = MagicMock(side_effect=mock_invoke_fc)
async def mock_get_response(*args, **kwargs):
return responses.pop(0)
with patch.object(ResponsesAgentThreadActions, "_get_response", new=mock_get_response):
messages = []
async for is_visible, msg in ResponsesAgentThreadActions.invoke(
agent=mock_agent,
chat_history=mock_chat_history,
thread=mock_thread,
store_enabled=True,
function_choice_behavior=MagicMock(maximum_auto_invoke_attempts=1),
):
messages.append(msg)
assert len(messages) == 3, f"Expected exactly 3 messages, got {len(messages)}"
async def test_invoke_passes_kernel_arguments_to_kernel(mock_agent, mock_chat_history, mock_thread):
# Prepare a response that triggers a function call
initial_response = MagicMock(spec=Response)
initial_response.status = "completed"
initial_response.id = "fake-response-id"
initial_response.output = [
ResponseFunctionToolCall(
id="tool_call_id",
call_id="call_id",
name="test_function",
arguments='{"some_arg": 123}',
type="function_call",
)
]
initial_response.error = None
initial_response.incomplete_details = None
initial_response.created_at = 123456
initial_response.usage = None
initial_response.role = "assistant"
final_response = MagicMock(spec=Response)
final_response.status = "completed"
final_response.id = "fake-final-response-id"
final_response.output = []
final_response.error = None
final_response.incomplete_details = None
final_response.created_at = 123456
final_response.usage = None
final_response.role = "assistant"
responses = [initial_response, final_response]
async def mock_invoke_fc(*args, **kwargs):
# Assert that KernelArguments were forwarded
assert isinstance(kwargs.get("arguments"), KernelArguments)
assert kwargs["arguments"].get("foo") == "bar"
return MagicMock(terminate=False)
mock_agent.kernel.invoke_function_call = MagicMock(side_effect=mock_invoke_fc)
async def mock_get_response(*args, **kwargs):
return responses.pop(0)
with patch.object(ResponsesAgentThreadActions, "_get_response", new=mock_get_response):
args = KernelArguments(foo="bar")
# Run invoke and ensure no assertion fails inside mock_invoke_fc
collected = []
async for _, msg in ResponsesAgentThreadActions.invoke(
agent=mock_agent,
chat_history=mock_chat_history,
thread=mock_thread,
store_enabled=True,
function_choice_behavior=MagicMock(maximum_auto_invoke_attempts=1),
arguments=args,
):
collected.append(msg)
assert len(collected) >= 2
async def test_invoke_stream_passes_kernel_arguments_to_kernel(mock_agent, mock_chat_history, mock_thread):
class MockStream(AsyncStream[ResponseStreamEvent]):
def __init__(self, events):
self._events = events
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
pass
def __aiter__(self):
return self
async def __anext__(self):
if not self._events:
raise StopAsyncIteration
return self._events.pop(0)
# Event that includes a function call
mock_tool_call_event = ResponseOutputItemAddedEvent(
item=ResponseFunctionToolCall(
id="fake-tool-call-id",
call_id="fake-call-id",
name="test_function",
arguments='{"arg": 123}',
type="function_call",
),
output_index=0,
type="response.output_item.added",
sequence_number=0,
)
mock_stream_event_end = ResponseOutputItemDoneEvent(
item=ResponseOutputMessage(
role="assistant",
status="completed",
id="fake-item-id",
content=[ResponseOutputText(text="Final message after tool call", type="output_text", annotations=[])],
type="message",
),
output_index=0,
sequence_number=0,
type="response.output_item.done",
)
async def mock_get_response(*args, **kwargs):
return MockStream([mock_tool_call_event, mock_stream_event_end])
async def mock_invoke_function_call(*args, **kwargs):
assert isinstance(kwargs.get("arguments"), KernelArguments)
assert kwargs["arguments"].get("foo") == "bar"
return MagicMock(terminate=False)
mock_agent.kernel.invoke_function_call = MagicMock(side_effect=mock_invoke_function_call)
with patch.object(ResponsesAgentThreadActions, "_get_response", new=mock_get_response):
args = KernelArguments(foo="bar")
collected_stream_messages = []
async for _ in ResponsesAgentThreadActions.invoke_stream(
agent=mock_agent,
chat_history=mock_chat_history,
thread=mock_thread,
store_enabled=True,
function_choice_behavior=MagicMock(maximum_auto_invoke_attempts=1),
output_messages=collected_stream_messages,
arguments=args,
):
pass
# If assertions passed in mock, arguments were forwarded
assert len(collected_stream_messages) >= 1
async def test_invoke_stream_no_function_calls(mock_agent, mock_chat_history, mock_thread):
class MockStream(AsyncStream[ResponseStreamEvent]):
def __init__(self, events):
self._events = events
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
pass
def __aiter__(self):
return self
async def __anext__(self):
if not self._events:
raise StopAsyncIteration
return self._events.pop(0)
mock_stream_event = ResponseTextDeltaEvent(
delta="Test partial content",
content_index=0,
item_id="fake-item-id",
logprobs=[Logprob(token="test_token", logprob=0.3)],
output_index=0,
type="response.output_text.delta",
sequence_number=0,
)
mock_stream_event_end = ResponseOutputItemDoneEvent(
item=ResponseOutputMessage(
role="assistant",
status="completed",
id="fake-item-id",
content=[ResponseOutputText(text="Test partial content", type="output_text", annotations=[])],
type="message",
),
output_index=0,
sequence_number=0,
type="response.output_item.done",
)
async def mock_get_response(*args, **kwargs):
return MockStream([mock_stream_event, mock_stream_event_end])
with patch.object(ResponsesAgentThreadActions, "_get_response", new=mock_get_response):
collected_stream_messages = []
received_text = ""
async for streaming_msg in ResponsesAgentThreadActions.invoke_stream(
agent=mock_agent,
chat_history=mock_chat_history,
thread=mock_thread,
store_enabled=False,
function_choice_behavior=MagicMock(),
output_messages=collected_stream_messages,
):
assert isinstance(streaming_msg, StreamingChatMessageContent)
for item in streaming_msg.items:
if isinstance(item, StreamingTextContent):
received_text += item.text
assert "Test partial content" in received_text, "Expected streamed partial content."
assert len(collected_stream_messages) == 1, "Expected exactly one final message."
assert collected_stream_messages[0].role == AuthorRole.ASSISTANT
async def test_invoke_stream_with_tool_calls(mock_agent, mock_chat_history, mock_thread):
class MockStream(AsyncStream[ResponseStreamEvent]):
def __init__(self, events):
self._events = events
async def __aenter__(self):
return self
async def __aexit__(self, exc_type, exc_val, exc_tb):
pass
def __aiter__(self):
return self
async def __anext__(self):
if not self._events:
raise StopAsyncIteration
return self._events.pop(0)
mock_tool_call_event = ResponseOutputItemAddedEvent(
item=ResponseFunctionToolCall(
id="fake-tool-call-id",
call_id="fake-call-id",
name="test_function",
arguments='{"arg": 123}',
type="function_call",
),
output_index=0,
type="response.output_item.added",
sequence_number=0,
)
mock_stream_event_end = ResponseOutputItemDoneEvent(
item=ResponseOutputMessage(
role="assistant",
status="completed",
id="fake-item-id",
content=[ResponseOutputText(text="Final message after tool call", type="output_text", annotations=[])],
type="message",
),
output_index=0,
sequence_number=0,
type="response.output_item.done",
)
async def mock_get_response(*args, **kwargs):
return MockStream([mock_tool_call_event, mock_stream_event_end])
async def mock_invoke_function_call(*args, **kwargs):
return MagicMock(terminate=False)
mock_agent.kernel.invoke_function_call = MagicMock(side_effect=mock_invoke_function_call)
with patch.object(ResponsesAgentThreadActions, "_get_response", new=mock_get_response):
collected_stream_messages = []
received_text = ""
async for streaming_msg in ResponsesAgentThreadActions.invoke_stream(
agent=mock_agent,
chat_history=mock_chat_history,
thread=mock_thread,
store_enabled=True,
function_choice_behavior=MagicMock(maximum_auto_invoke_attempts=1),
output_messages=collected_stream_messages,
):
assert isinstance(streaming_msg, StreamingChatMessageContent)
for item in streaming_msg.items:
if isinstance(item, StreamingTextContent):
received_text += item.text
assert len(collected_stream_messages) == 2, "Expected exactly two final messages after tool call."
assert collected_stream_messages[0].role == AuthorRole.ASSISTANT
def test_get_tools(mock_agent, kernel, custom_plugin_class):
kernel.add_plugin(custom_plugin_class)
fcb = FunctionChoiceBehavior()
tools = ResponsesAgentThreadActions._get_tools(
agent=mock_agent,
kernel=kernel,
function_choice_behavior=fcb,
)
assert len(tools) == len(mock_agent.tools) + len(kernel.get_full_list_of_function_metadata())
def test_prepare_chat_history_multiple_images_no_duplication():
"""Test that multiple images in a message don't get duplicated in the request."""
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.contents.image_content import ImageContent
from semantic_kernel.contents.text_content import TextContent
# Create a chat history with a message containing text and multiple images
chat_history = ChatHistory()
message_items = [
TextContent(text="How many pictures do you get?"),
ImageContent(uri="https://example.com/image1.jpg"),
ImageContent(uri="https://example.com/image2.jpg"),
ImageContent(uri="https://example.com/image3.jpg"),
ImageContent(uri="https://example.com/image4.jpg"),
]
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
message = ChatMessageContent(role=AuthorRole.USER, items=message_items)
chat_history.add_message(message)
# Call the method that was causing duplication
result = ResponsesAgentThreadActions._prepare_chat_history_for_request(chat_history, True)
# Verify we have exactly one message in the result
assert len(result) == 1, f"Expected 1 message, got {len(result)}"
# Get the content from the message
message_content = result[0]["content"]
# Count text and image items
text_items = [item for item in message_content if item["type"] == "input_text"]
image_items = [item for item in message_content if item["type"] == "input_image"]
# Verify counts
assert len(text_items) == 1, f"Expected 1 text item, got {len(text_items)}"
assert len(image_items) == 4, f"Expected 4 image items, got {len(image_items)}"
# Verify the text content
assert text_items[0]["text"] == "How many pictures do you get?"
# Verify the image URLs are correct and not duplicated
expected_urls = [
"https://example.com/image1.jpg",
"https://example.com/image2.jpg",
"https://example.com/image3.jpg",
"https://example.com/image4.jpg",
]
actual_urls = [item["image_url"] for item in image_items]
assert actual_urls == expected_urls, f"Expected {expected_urls}, got {actual_urls}"
# Verify total content items equals expected (1 text + 4 images = 5)
assert len(message_content) == 5, f"Expected 5 total content items, got {len(message_content)}"
@@ -0,0 +1,125 @@
# Copyright (c) Microsoft. All rights reserved.
import asyncio
import sys
from collections.abc import AsyncIterable, Awaitable, Callable
from semantic_kernel.agents.agent import Agent, AgentResponseItem, AgentThread
from semantic_kernel.agents.runtime.core.core_runtime import CoreRuntime
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
if sys.version_info >= (3, 12):
from typing import override # pragma: no cover
else:
from typing_extensions import override # pragma: no cover
class MockAgentThread(AgentThread):
"""A mock agent thread for testing purposes."""
@override
async def _create(self) -> str:
return "mock_thread_id"
@override
async def _delete(self) -> None:
pass
@override
async def _on_new_message(self, new_message: ChatMessageContent) -> None:
pass
class MockAgent(Agent):
"""A mock agent for testing purposes."""
@override
async def get_response(
self,
messages: str | ChatMessageContent | list[str | ChatMessageContent] | None = None,
*,
thread: AgentThread | None = None,
**kwargs,
) -> AgentResponseItem[ChatMessageContent]:
pass
@override
async def invoke(
self,
messages: str | ChatMessageContent | list[str | ChatMessageContent] | None = None,
*,
thread: AgentThread | None = None,
on_intermediate_message: Callable[[ChatMessageContent], Awaitable[None]] | None = None,
**kwargs,
) -> AgentResponseItem[ChatMessageContent]:
pass
@override
async def invoke_stream(
self,
messages: str | ChatMessageContent | list[str | ChatMessageContent] | None = None,
*,
thread: AgentThread | None = None,
on_intermediate_message: Callable[[ChatMessageContent], Awaitable[None]] | None = None,
**kwargs,
) -> AsyncIterable[AgentResponseItem[StreamingChatMessageContent]]:
"""Simulate streaming response from the agent."""
# Simulate some processing time
await asyncio.sleep(0.05)
yield AgentResponseItem[StreamingChatMessageContent](
message=StreamingChatMessageContent(
role=AuthorRole.ASSISTANT,
name=self.name,
content="mock",
choice_index=0,
),
thread=thread or MockAgentThread(),
)
# Simulate some processing time before sending the next part of the response
await asyncio.sleep(0.05)
yield AgentResponseItem[StreamingChatMessageContent](
message=StreamingChatMessageContent(
role=AuthorRole.ASSISTANT,
name=self.name,
content="_response",
choice_index=0,
),
thread=thread or MockAgentThread(),
)
class MockAgentWithException(MockAgent):
"""A mock agent that raises an exception for testing purposes."""
@override
async def invoke_stream(
self,
messages: str | ChatMessageContent | list[str | ChatMessageContent] | None = None,
*,
thread: AgentThread | None = None,
on_intermediate_message: Callable[[ChatMessageContent], Awaitable[None]] | None = None,
**kwargs,
) -> AsyncIterable[AgentResponseItem[StreamingChatMessageContent]]:
"""Simulate streaming response from the agent that raises an exception."""
# Simulate some processing time
await asyncio.sleep(0.05)
yield AgentResponseItem[StreamingChatMessageContent](
message=StreamingChatMessageContent(
role=AuthorRole.ASSISTANT,
name=self.name,
content="mock",
choice_index=0,
),
thread=thread or MockAgentThread(),
)
raise RuntimeError("Mock agent exception")
class MockRuntime(CoreRuntime):
"""A mock agent runtime for testing purposes."""
pass
@@ -0,0 +1,204 @@
# Copyright (c) Microsoft. All rights reserved.
import asyncio
import sys
from unittest.mock import patch
import pytest
from semantic_kernel.agents.orchestration.concurrent import ConcurrentOrchestration
from semantic_kernel.agents.orchestration.orchestration_base import DefaultTypeAlias, OrchestrationResult
from semantic_kernel.agents.runtime.in_process.in_process_runtime import InProcessRuntime
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from tests.unit.agents.orchestration.conftest import MockAgent, MockAgentWithException, MockRuntime
async def test_prepare():
"""Test the prepare method of the ConcurrentOrchestration."""
agent_a = MockAgent()
agent_b = MockAgent()
runtime = MockRuntime()
package_path = "semantic_kernel.agents.orchestration.concurrent"
with (
patch(f"{package_path}.ConcurrentOrchestration._start"),
patch(f"{package_path}.ConcurrentAgentActor.register") as mock_agent_actor_register,
patch(f"{package_path}.CollectionActor.register") as mock_collection_actor_register,
patch.object(runtime, "add_subscription") as mock_add_subscription,
):
orchestration = ConcurrentOrchestration(members=[agent_a, agent_b])
await orchestration.invoke(task="test_message", runtime=runtime)
assert mock_agent_actor_register.call_count == 2
assert mock_collection_actor_register.call_count == 1
assert mock_add_subscription.call_count == 2
async def test_invoke():
"""Test the invoke method of the ConcurrentOrchestration."""
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = ConcurrentOrchestration(members=[agent_a, agent_b])
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
result = await orchestration_result.get(1.0)
assert isinstance(orchestration_result, OrchestrationResult)
assert isinstance(result, list)
assert len(result) == 2
assert all(isinstance(item, ChatMessageContent) for item in result)
finally:
await runtime.stop_when_idle()
async def test_invoke_with_response_callback():
"""Test the invoke method of the ConcurrentOrchestration with a response callback."""
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
responses: list[DefaultTypeAlias] = []
try:
orchestration = ConcurrentOrchestration(
members=[agent_a, agent_b],
agent_response_callback=lambda x: responses.append(x),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
assert len(responses) == 2
assert all(isinstance(item, ChatMessageContent) for item in responses)
assert all(item.content == "mock_response" for item in responses)
finally:
await runtime.stop_when_idle()
async def test_invoke_with_streaming_response_callback():
"""Test the invoke method of the ConcurrentOrchestration with a streaming response callback."""
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
responses: dict[str, list[StreamingChatMessageContent]] = {}
try:
orchestration = ConcurrentOrchestration(
members=[agent_a, agent_b],
streaming_agent_response_callback=lambda x, _: responses.setdefault(x.name, []).append(x),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
assert len(responses[agent_a.name]) == 2
assert len(responses[agent_b.name]) == 2
assert all(isinstance(item, StreamingChatMessageContent) for item in responses[agent_a.name])
assert all(isinstance(item, StreamingChatMessageContent) for item in responses[agent_b.name])
agent_a_response = sum(responses[agent_a.name][1:], responses[agent_a.name][0])
assert agent_a_response.content == "mock_response"
agent_b_response = sum(responses[agent_b.name][1:], responses[agent_b.name][0])
assert agent_b_response.content == "mock_response"
finally:
await runtime.stop_when_idle()
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke_cancel_before_completion():
"""Test the invoke method of the ConcurrentOrchestration with cancellation before completion."""
with (
patch.object(MockAgent, "invoke_stream", wraps=MockAgent.invoke_stream, autospec=True) as mock_invoke_stream,
):
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = ConcurrentOrchestration(members=[agent_a, agent_b])
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
# Cancel before the collection agent gets the responses
await asyncio.sleep(0.05)
orchestration_result.cancel()
finally:
await runtime.stop_when_idle()
assert mock_invoke_stream.call_count == 2
async def test_invoke_cancel_after_completion():
"""Test the invoke method of the ConcurrentOrchestration with cancellation after completion."""
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = ConcurrentOrchestration(members=[agent_a, agent_b])
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
# Wait for the orchestration to complete
await orchestration_result.get(1.0)
with pytest.raises(RuntimeError, match="The invocation has already been completed."):
orchestration_result.cancel()
finally:
await runtime.stop_when_idle()
async def test_invoke_with_double_get_result():
"""Test the invoke method of the ConcurrentOrchestration with double get result."""
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = ConcurrentOrchestration(members=[agent_a, agent_b])
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
# Get result before completion
with pytest.raises(asyncio.TimeoutError):
await orchestration_result.get(0.1)
# The invocation should still be in progress and getting the result again should not raise an error
result = await orchestration_result.get(1.0)
assert isinstance(result, list)
assert len(result) == 2
finally:
await runtime.stop_when_idle()
async def test_invoke_with_agent_raising_exception():
"""Test the invoke method of the ConcurrentOrchestration with an agent raising an exception."""
agent_a = MockAgent()
agent_b = MockAgentWithException()
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = ConcurrentOrchestration(members=[agent_a, agent_b])
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
with pytest.raises(RuntimeError, match="Mock agent exception"):
await orchestration_result.get(1.0)
assert orchestration_result.exception is not None
finally:
await runtime.stop_when_idle()
@@ -0,0 +1,399 @@
# Copyright (c) Microsoft. All rights reserved.
import asyncio
import sys
from unittest.mock import patch
import pytest
from semantic_kernel.agents.orchestration.group_chat import (
BooleanResult,
GroupChatOrchestration,
RoundRobinGroupChatManager,
)
from semantic_kernel.agents.orchestration.orchestration_base import DefaultTypeAlias, OrchestrationResult
from semantic_kernel.agents.runtime.in_process.in_process_runtime import InProcessRuntime
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from tests.unit.agents.orchestration.conftest import MockAgent, MockAgentWithException, MockRuntime
if sys.version_info >= (3, 12):
from typing import override # pragma: no cover
else:
from typing_extensions import override # pragma: no cover
class RoundRobinGroupChatManagerWithUserInput(RoundRobinGroupChatManager):
@override
async def should_request_user_input(self, chat_history: ChatHistory) -> BooleanResult:
"""Check if the group chat should request user input."""
return BooleanResult(
result=True,
reason="Allow user input for testing purposes.",
)
# region GroupChatOrchestration
async def test_init_member_without_description_throws():
"""Test the prepare method of the GroupChatOrchestration with a member without description."""
agent_a = MockAgent()
agent_b = MockAgent()
with pytest.raises(ValueError):
GroupChatOrchestration(members=[agent_a, agent_b], manager=RoundRobinGroupChatManager())
async def test_prepare():
"""Test the prepare method of the GroupChatOrchestration."""
agent_a = MockAgent(description="test agent")
agent_b = MockAgent(description="test agent")
runtime = MockRuntime()
package_path = "semantic_kernel.agents.orchestration.group_chat"
with (
patch(f"{package_path}.GroupChatOrchestration._start"),
patch(f"{package_path}.GroupChatAgentActor.register") as mock_agent_actor_register,
patch(f"{package_path}.GroupChatManagerActor.register") as mock_manager_actor_register,
patch.object(runtime, "add_subscription") as mock_add_subscription,
):
orchestration = GroupChatOrchestration(members=[agent_a, agent_b], manager=RoundRobinGroupChatManager())
await orchestration.invoke(task="test_message", runtime=runtime)
assert mock_agent_actor_register.call_count == 2
assert mock_manager_actor_register.call_count == 1
assert mock_add_subscription.call_count == 3
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke():
"""Test the invoke method of the GroupChatOrchestration."""
with (
patch.object(MockAgent, "invoke_stream", wraps=MockAgent.invoke_stream, autospec=True) as mock_invoke_stream,
):
agent_a = MockAgent(description="test agent")
agent_b = MockAgent(description="test agent")
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = GroupChatOrchestration(
members=[agent_a, agent_b],
manager=RoundRobinGroupChatManager(max_rounds=3),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
result = await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert isinstance(orchestration_result, OrchestrationResult)
assert isinstance(result, ChatMessageContent)
assert result.role == AuthorRole.ASSISTANT
assert result.content == "mock_response"
assert mock_invoke_stream.call_count == 3
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke_with_list():
"""Test the invoke method of the GroupChatOrchestration with a list of messages."""
with (
patch.object(MockAgent, "invoke_stream", wraps=MockAgent.invoke_stream, autospec=True) as mock_invoke_stream,
):
agent_a = MockAgent(description="test agent")
agent_b = MockAgent(description="test agent")
runtime = InProcessRuntime()
runtime.start()
messages = [
ChatMessageContent(role=AuthorRole.USER, content="test_message_1"),
ChatMessageContent(role=AuthorRole.USER, content="test_message_2"),
]
try:
orchestration = GroupChatOrchestration(
members=[agent_a, agent_b],
manager=RoundRobinGroupChatManager(max_rounds=2),
)
orchestration_result = await orchestration.invoke(task=messages, runtime=runtime)
await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert mock_invoke_stream.call_count == 2
# Two messages
assert len(mock_invoke_stream.call_args_list[0][0][1]) == 2
# Two messages + response from agent A
assert len(mock_invoke_stream.call_args_list[1][0][1]) == 3
async def test_invoke_with_response_callback():
"""Test the invoke method of the GroupChatOrchestration with a response callback."""
agent_a = MockAgent(description="test agent")
agent_b = MockAgent(description="test agent")
runtime = InProcessRuntime()
runtime.start()
responses: list[DefaultTypeAlias] = []
try:
orchestration = GroupChatOrchestration(
members=[agent_a, agent_b],
manager=RoundRobinGroupChatManager(max_rounds=3),
agent_response_callback=lambda x: responses.append(x),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert len(responses) == 3
assert all(isinstance(item, ChatMessageContent) for item in responses)
assert all(item.content == "mock_response" for item in responses)
async def test_invoke_with_streaming_response_callback():
"""Test the invoke method of the GroupChatOrchestration with a streaming_response callback."""
agent_a = MockAgent(description="test agent")
agent_b = MockAgent(description="test agent")
runtime = InProcessRuntime()
runtime.start()
responses: dict[str, list[StreamingChatMessageContent]] = {}
try:
orchestration = GroupChatOrchestration(
members=[agent_a, agent_b],
manager=RoundRobinGroupChatManager(max_rounds=3),
streaming_agent_response_callback=lambda x, _: responses.setdefault(x.name, []).append(x),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert len(responses[agent_a.name]) == 4 # Invoke twice, each with 2 chunks
assert len(responses[agent_b.name]) == 2 # Invoke once, with 2 chunks
assert all(isinstance(item, StreamingChatMessageContent) for item in responses[agent_a.name])
assert all(isinstance(item, StreamingChatMessageContent) for item in responses[agent_b.name])
agent_a_response = sum(responses[agent_a.name][1:2], responses[agent_a.name][0])
assert agent_a_response.content == "mock_response"
agent_b_response = sum(responses[agent_b.name][1:], responses[agent_b.name][0])
assert agent_b_response.content == "mock_response"
async def test_invoke_with_human_response_function():
"""Test the invoke method of the GroupChatOrchestration with a human response function."""
agent_a = MockAgent(description="test agent")
agent_b = MockAgent(description="test agent")
user_input_count = 0
def human_response_function(chat_history: ChatHistory) -> ChatMessageContent:
# Simulate user input
nonlocal user_input_count
user_input_count += 1
return ChatMessageContent(
role=AuthorRole.USER,
content=f"user_input_{user_input_count}",
)
orchestration_manager = RoundRobinGroupChatManagerWithUserInput(
max_rounds=3,
human_response_function=human_response_function,
)
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = GroupChatOrchestration(
members=[agent_a, agent_b],
manager=orchestration_manager,
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert user_input_count == 4 # 3 rounds + 1 initial user input
@pytest.mark.skip(
reason="Unreliable test due to timing issues in CI environment. To be fixed later.",
)
async def test_invoke_cancel_before_completion():
"""Test the invoke method of the GroupChatOrchestration with cancellation before completion."""
with (
patch.object(MockAgent, "invoke_stream", wraps=MockAgent.invoke_stream, autospec=True) as mock_invoke_stream,
):
agent_a = MockAgent(description="test agent")
agent_b = MockAgent(description="test agent")
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = GroupChatOrchestration(
members=[agent_a, agent_b],
manager=RoundRobinGroupChatManager(max_rounds=3),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
# Cancel before the second agent responds
await asyncio.sleep(0.19)
orchestration_result.cancel()
finally:
await runtime.stop_when_idle()
assert mock_invoke_stream.call_count == 2
async def test_invoke_cancel_after_completion():
"""Test the invoke method of the GroupChatOrchestration with cancellation after completion."""
agent_a = MockAgent(description="test agent")
agent_b = MockAgent(description="test agent")
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = GroupChatOrchestration(
members=[agent_a, agent_b],
manager=RoundRobinGroupChatManager(max_rounds=3),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
# Wait for the orchestration to complete
await orchestration_result.get(1.0)
with pytest.raises(RuntimeError, match="The invocation has already been completed."):
orchestration_result.cancel()
finally:
await runtime.stop_when_idle()
async def test_invoke_with_agent_raising_exception():
"""Test the invoke method of the GroupChatOrchestration with an agent raising an exception."""
agent_a = MockAgent(description="test agent")
agent_b = MockAgentWithException(description="test agent")
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = GroupChatOrchestration(
members=[agent_a, agent_b],
manager=RoundRobinGroupChatManager(max_rounds=3),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
with pytest.raises(RuntimeError, match="Mock agent exception"):
await orchestration_result.get(1.0)
assert orchestration_result.exception is not None
finally:
await runtime.stop_when_idle()
# endregion GroupChatOrchestration
# region RoundRobinGroupChatManager
def test_round_robin_group_chat_manager_init():
"""Test the initialization of the RoundRobinGroupChatManager."""
manager = RoundRobinGroupChatManager()
assert manager.max_rounds is None
assert manager.current_round == 0
assert manager.current_index == 0
assert manager.human_response_function is None
def test_round_robin_group_chat_manager_init_with_max_rounds():
"""Test the initialization of the RoundRobinGroupChatManager with max_rounds."""
manager = RoundRobinGroupChatManager(max_rounds=5)
assert manager.max_rounds == 5
assert manager.current_round == 0
assert manager.current_index == 0
assert manager.human_response_function is None
def test_round_robin_group_chat_manager_init_with_human_response_function():
"""Test the initialization of the RoundRobinGroupChatManager with human_response_function."""
async def human_response_function(chat_history: ChatHistory) -> str:
# Simulate user input
await asyncio.sleep(0.1)
return "user_input"
manager = RoundRobinGroupChatManager(human_response_function=human_response_function)
assert manager.max_rounds is None
assert manager.current_round == 0
assert manager.current_index == 0
assert manager.human_response_function == human_response_function
async def test_round_robin_group_chat_manager_should_terminate():
"""Test the should_terminate method of the RoundRobinGroupChatManager."""
manager = RoundRobinGroupChatManager(max_rounds=3)
result = await manager.should_terminate(ChatHistory())
assert result.result is False
result = await manager.should_terminate(ChatHistory())
assert result.result is False
result = await manager.should_terminate(ChatHistory())
assert result.result is False
result = await manager.should_terminate(ChatHistory())
assert result.result is True
async def test_round_robin_group_chat_manager_should_terminate_without_max_rounds():
"""Test the should_terminate method of the RoundRobinGroupChatManager without max_rounds."""
manager = RoundRobinGroupChatManager()
result = await manager.should_terminate(ChatHistory())
assert result.result is False
async def test_round_robin_group_chat_manager_select_next_agent():
"""Test the select_next_agent method of the RoundRobinGroupChatManager."""
manager = RoundRobinGroupChatManager(max_rounds=3)
participant_descriptions = {
"agent_1": "Agent 1",
"agent_2": "Agent 2",
"agent_3": "Agent 3",
}
await manager.should_terminate(ChatHistory())
result = await manager.select_next_agent(ChatHistory(), participant_descriptions)
assert result.result == "agent_1"
await manager.should_terminate(ChatHistory())
result = await manager.select_next_agent(ChatHistory(), participant_descriptions)
assert result.result == "agent_2"
await manager.should_terminate(ChatHistory())
result = await manager.select_next_agent(ChatHistory(), participant_descriptions)
assert result.result == "agent_3"
assert manager.current_round == 3
# endregion RoundRobinGroupChatManager
@@ -0,0 +1,721 @@
# Copyright (c) Microsoft. All rights reserved.
import asyncio
import sys
from collections.abc import AsyncIterable, Awaitable, Callable
from unittest.mock import patch
import pytest
from semantic_kernel.agents.agent import Agent, AgentResponseItem, AgentThread
from semantic_kernel.agents.orchestration.handoffs import (
HANDOFF_PLUGIN_NAME,
HandoffAgentActor,
HandoffOrchestration,
OrchestrationHandoffs,
)
from semantic_kernel.agents.orchestration.orchestration_base import DefaultTypeAlias
from semantic_kernel.agents.runtime.in_process.in_process_runtime import InProcessRuntime
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.function_call_content import FunctionCallContent
from semantic_kernel.contents.function_result_content import FunctionResultContent
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.kernel import Kernel
from tests.unit.agents.orchestration.conftest import MockAgent, MockAgentWithException, MockRuntime
if sys.version_info >= (3, 12):
from typing import override # pragma: no cover
else:
from typing_extensions import override # pragma: no cover
class MockAgentWithHandoffFunctionCall(Agent):
"""A mock agent with handoff function call for testing purposes."""
target_agent: Agent
def __init__(self, target_agent: Agent):
super().__init__(target_agent=target_agent)
@override
async def get_response(
self,
messages: str | ChatMessageContent | list[str | ChatMessageContent] | None = None,
*,
thread: AgentThread | None = None,
kernel: Kernel | None = None,
**kwargs,
) -> AgentResponseItem[ChatMessageContent]:
pass
@override
async def invoke(
self,
messages: str | ChatMessageContent | list[str | ChatMessageContent] | None = None,
*,
thread: AgentThread | None = None,
on_intermediate_message: Callable[[ChatMessageContent], Awaitable[None]] | None = None,
**kwargs,
) -> AgentResponseItem[ChatMessageContent]:
pass
@override
async def invoke_stream(
self,
messages: str | ChatMessageContent | list[str | ChatMessageContent] | None = None,
*,
thread: AgentThread | None = None,
on_intermediate_message: Callable[[ChatMessageContent], Awaitable[None]] | None = None,
kernel: Kernel | None = None,
**kwargs,
) -> AsyncIterable[AgentResponseItem[StreamingChatMessageContent]]:
"""Simulate streaming response from the agent."""
function_call = FunctionCallContent(
function_name=f"transfer_to_{self.target_agent.name}",
plugin_name=HANDOFF_PLUGIN_NAME,
call_id="test_call_id",
id="test_id",
)
# Simulate some processing time
await asyncio.sleep(0.1)
await kernel.invoke_function_call(
function_call=function_call,
chat_history=ChatHistory(),
)
# Do not yield any messages, as the agent doesn't yield any tool related messages from the streaming API.
# Nevertheless, the method needs have a `yield` code path to satisfy the AsyncIterable interface.
if False:
yield
# Simulate on_intermediate_message callback
await on_intermediate_message(
StreamingChatMessageContent(
role=AuthorRole.ASSISTANT,
name=self.name,
choice_index=0,
items=[function_call],
)
)
await on_intermediate_message(
StreamingChatMessageContent(
role=AuthorRole.ASSISTANT,
name=self.name,
choice_index=0,
items=[
FunctionResultContent(
function_name=function_call.function_name,
plugin_name=function_call.plugin_name,
call_id=function_call.call_id,
id=function_call.id,
result=None,
)
],
)
)
class MockAgentWithCompleteTaskFunctionCall(Agent):
"""A mock agent with complete_task function call for testing purposes."""
@override
async def get_response(
self,
messages: str | ChatMessageContent | list[str | ChatMessageContent] | None = None,
*,
thread: AgentThread | None = None,
kernel: Kernel | None = None,
**kwargs,
) -> AgentResponseItem[ChatMessageContent]:
pass
@override
async def invoke(
self,
messages: str | ChatMessageContent | list[str | ChatMessageContent] | None = None,
*,
thread: AgentThread | None = None,
on_intermediate_message: Callable[[ChatMessageContent], Awaitable[None]] | None = None,
**kwargs,
) -> AgentResponseItem[ChatMessageContent]:
pass
@override
async def invoke_stream(
self,
messages: str | ChatMessageContent | list[str | ChatMessageContent] | None = None,
*,
thread: AgentThread | None = None,
on_intermediate_message: Callable[[ChatMessageContent], Awaitable[None]] | None = None,
kernel: Kernel | None = None,
**kwargs,
) -> AsyncIterable[AgentResponseItem[StreamingChatMessageContent]]:
"""Simulate streaming response from the agent."""
# Simulate some processing time
await asyncio.sleep(0.1)
await kernel.invoke_function_call(
function_call=FunctionCallContent(
function_name="complete_task",
plugin_name=HANDOFF_PLUGIN_NAME,
call_id="test_call_id",
id="test_id",
arguments={"task_summary": "test_summary"},
),
chat_history=ChatHistory(),
)
# Do not yield any messages, as the agent doesn't yield any tool related messages from the streaming API.
# Nevertheless, the method needs have a `yield` code path to satisfy the AsyncIterable interface.
if False:
yield
# region HandoffOrchestration
def test_init_without_handoffs():
"""Test the initialization of HandoffOrchestration without handoffs."""
agent_a = MockAgent()
agent_b = MockAgent()
with pytest.raises(ValueError):
HandoffOrchestration(members=[agent_a, agent_b], handoffs={})
def test_init_with_invalid_handoff():
"""Test the initialization of HandoffOrchestration with invalid handoff."""
agent_a = MockAgent()
agent_b = MockAgent()
# Invalid handoff agent name
with pytest.raises(ValueError):
HandoffOrchestration(
members=[agent_a, agent_b],
handoffs={
agent_a.name: {agent_b.name: "test", "invalid_agent_name": "test"},
agent_b.name: {agent_a.name: "test"},
},
)
# Invalid handoff agent name (not in members)
with pytest.raises(ValueError):
HandoffOrchestration(
members=[agent_a, agent_b],
handoffs={
"invalid_agent_name": {agent_b.name: "test"},
agent_b.name: {agent_a.name: "test"},
},
)
# Cannot handoff to self
with pytest.raises(ValueError):
HandoffOrchestration(
members=[agent_a, agent_b],
handoffs={
agent_a.name: {agent_a.name: "test"},
agent_b.name: {agent_a.name: "test"},
},
)
def test_init_with_duplicate_handoffs():
"""Test the initialization of HandoffOrchestration with duplicate handoffs."""
agent_a = MockAgent()
agent_b = MockAgent()
# Uniqueness guarantee
orchestration = HandoffOrchestration(
members=[agent_a, agent_b],
handoffs={
agent_a.name: {agent_b.name: "test 1", agent_b.name: "test 2"},
},
)
assert len(orchestration._handoffs[agent_a.name]) == 1
def test_init_with_dictionary_handoffs():
"""Test the initialization of HandoffOrchestration with dictionary handoffs."""
agent_a = MockAgent()
agent_b = MockAgent()
orchestration_handoffs = OrchestrationHandoffs(
{
agent_a.name: {agent_b.name: "test 1"},
agent_b.name: {agent_a.name: "test 2"},
},
)
assert len(orchestration_handoffs) == 2
for handoff_agent_name, handoff_description in orchestration_handoffs[agent_a.name].items():
assert handoff_agent_name == agent_b.name
assert handoff_description == "test 1"
for handoff_agent_name, handoff_description in orchestration_handoffs[agent_b.name].items():
assert handoff_agent_name == agent_a.name
assert handoff_description == "test 2"
def test_orchestration_handoff_add():
"""Test the add method of the OrchestrationHandoffs."""
agent_a = MockAgent()
agent_b = MockAgent()
orchestration_handoffs = OrchestrationHandoffs().add(agent_a, agent_b).add(agent_b, agent_a)
assert isinstance(orchestration_handoffs, OrchestrationHandoffs)
assert len(orchestration_handoffs) == 2
assert len(orchestration_handoffs[agent_a.name]) == 1
assert len(orchestration_handoffs[agent_b.name]) == 1
for handoff_agent_name, handoff_description in orchestration_handoffs[agent_a.name].items():
assert handoff_agent_name == agent_b.name
assert handoff_description == ""
for handoff_agent_name, handoff_description in orchestration_handoffs[agent_b.name].items():
assert handoff_agent_name == agent_a.name
assert handoff_description == ""
def test_orchestration_handoff_add_many():
"""Test the add_many method of the OrchestrationHandoffs."""
agent_a = MockAgent(description="agent_a")
agent_b = MockAgent(description="agent_b")
agent_c = MockAgent(description="agent_c")
# Case 1: Agent instance as source and dictionary as handoffs
orchestration_handoffs = OrchestrationHandoffs().add_many(
agent_a,
{agent_b.name: "test 1", agent_c.name: "test 2"},
)
assert isinstance(orchestration_handoffs, OrchestrationHandoffs)
assert len(orchestration_handoffs) == 1
assert len(orchestration_handoffs[agent_a.name]) == 2
for handoff_agent_name, handoff_description in orchestration_handoffs[agent_a.name].items():
assert handoff_agent_name in [agent_b.name, agent_c.name]
assert handoff_description in ["test 1", "test 2"]
# Case 2: Agent name as source and list of agents as handoffs
orchestration_handoffs = OrchestrationHandoffs().add_many(
agent_a.name,
{agent_b.name: "test 1", agent_c.name: "test 2"},
)
assert isinstance(orchestration_handoffs, OrchestrationHandoffs)
assert len(orchestration_handoffs) == 1
assert len(orchestration_handoffs[agent_a.name]) == 2
for handoff_agent_name, handoff_description in orchestration_handoffs[agent_a.name].items():
assert handoff_agent_name in [agent_b.name, agent_c.name]
assert handoff_description in ["test 1", "test 2"]
# Case 3: Agent instance as source and list of agents as handoffs
orchestration_handoffs = OrchestrationHandoffs().add_many(agent_a, [agent_b, agent_c])
assert isinstance(orchestration_handoffs, OrchestrationHandoffs)
assert len(orchestration_handoffs) == 1
assert len(orchestration_handoffs[agent_a.name]) == 2
for handoff_agent_name, handoff_description in orchestration_handoffs[agent_a.name].items():
assert handoff_agent_name in [agent_b.name, agent_c.name]
assert handoff_description in [agent_b.description, agent_c.description]
# Case 4: Agent name as source and list of agent names as handoffs
orchestration_handoffs = OrchestrationHandoffs().add_many(agent_a.name, [agent_b.name, agent_c.name])
assert isinstance(orchestration_handoffs, OrchestrationHandoffs)
assert len(orchestration_handoffs) == 1
assert len(orchestration_handoffs[agent_a.name]) == 2
for handoff_agent_name, handoff_description in orchestration_handoffs[agent_a.name].items():
assert handoff_agent_name in [agent_b.name, agent_c.name]
assert handoff_description == ""
async def test_prepare():
"""Test the prepare method of the HandoffOrchestration."""
agent_a = MockAgent()
agent_b = MockAgent()
agent_c = MockAgent()
runtime = MockRuntime()
package_path = "semantic_kernel.agents.orchestration.handoffs"
with (
patch(f"{package_path}.HandoffOrchestration._start"),
patch(f"{package_path}.HandoffAgentActor.register") as mock_agent_actor_register,
patch.object(runtime, "add_subscription") as mock_add_subscription,
):
orchestration = HandoffOrchestration(
members=[agent_a, agent_b, agent_c],
handoffs={
agent_a.name: {agent_b.name: "test"},
agent_b.name: {agent_c.name: "test"},
agent_c.name: {agent_a.name: "test"},
},
)
await orchestration.invoke(task="test_message", runtime=runtime)
assert mock_agent_actor_register.call_count == 3
assert mock_add_subscription.call_count == 3
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke():
"""Test the prepare method of the HandoffOrchestration."""
with (
patch.object(HandoffAgentActor, "__init__", wraps=HandoffAgentActor.__init__, autospec=True) as mock_init,
patch.object(MockAgent, "invoke_stream", wraps=MockAgent.invoke_stream, autospec=True) as mock_invoke_stream,
):
agent_a = MockAgent()
agent_b = MockAgent()
agent_c = MockAgent()
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = HandoffOrchestration(
members=[agent_a, agent_b, agent_c],
handoffs={
agent_a.name: {agent_b.name: "test", agent_c.name: "test"},
agent_b.name: {agent_a.name: "test"},
},
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
assert mock_init.call_args_list[0][0][3] == {agent_b.name: "test", agent_c.name: "test"}
assert isinstance(mock_invoke_stream.call_args_list[0][1]["kernel"], Kernel)
kernel = mock_invoke_stream.call_args_list[0][1]["kernel"]
assert HANDOFF_PLUGIN_NAME in kernel.plugins
assert (
len(kernel.plugins[HANDOFF_PLUGIN_NAME].functions) == 3
) # two handoff functions + complete task function
# The kernel in the agent should not be modified
assert len(agent_a.kernel.plugins) == 0
assert len(agent_b.kernel.plugins) == 0
assert len(agent_c.kernel.plugins) == 0
finally:
await runtime.stop_when_idle()
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke_with_list():
"""Test the invoke method of the HandoffOrchestration with a list of messages."""
with (
patch.object(MockAgent, "invoke_stream", wraps=MockAgent.invoke_stream, autospec=True) as mock_invoke_stream,
):
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
messages = [
ChatMessageContent(role=AuthorRole.USER, content="test_message_1"),
ChatMessageContent(role=AuthorRole.USER, content="test_message_2"),
]
try:
orchestration = HandoffOrchestration(
members=[agent_a, agent_b],
handoffs={agent_a.name: {agent_b.name: "test"}},
)
orchestration_result = await orchestration.invoke(task=messages, runtime=runtime)
await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert mock_invoke_stream.call_count == 1
# Two messages
assert len(mock_invoke_stream.call_args_list[0][0][1]) == 2
# The kernel in the agent should not be modified
assert len(agent_a.kernel.plugins) == 0
assert len(agent_b.kernel.plugins) == 0
async def test_invoke_with_response_callback():
"""Test the invoke method of the HandoffOrchestration with a response callback."""
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
responses: list[DefaultTypeAlias] = []
try:
orchestration = HandoffOrchestration(
members=[agent_a, agent_b],
handoffs={agent_a.name: {agent_b.name: "test"}},
agent_response_callback=lambda x: responses.append(x),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert len(responses) == 1
assert all(isinstance(item, ChatMessageContent) for item in responses)
assert all(item.content == "mock_response" for item in responses)
# The kernel in the agent should not be modified
assert len(agent_a.kernel.plugins) == 0
assert len(agent_b.kernel.plugins) == 0
async def test_invoke_with_streaming_response_callback():
"""Test the invoke method of the HandoffOrchestration with a streaming response callback."""
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
responses: dict[str, list[StreamingChatMessageContent]] = {}
try:
orchestration = HandoffOrchestration(
members=[agent_a, agent_b],
handoffs={agent_a.name: {agent_b.name: "test"}},
streaming_agent_response_callback=lambda x, _: responses.setdefault(x.name, []).append(x),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert len(responses[agent_a.name]) == 2
assert all(isinstance(item, StreamingChatMessageContent) for item in responses[agent_a.name])
agent_a_response = sum(responses[agent_a.name][1:], responses[agent_a.name][0])
assert agent_a_response.content == "mock_response"
# Agent B was not invoked, so it should not have any responses
assert agent_b.name not in responses or len(responses[agent_b.name]) == 0
# The kernel in the agent should not be modified
assert len(agent_a.kernel.plugins) == 0
assert len(agent_b.kernel.plugins) == 0
async def test_response_callback_with_handoff_function_call():
"""Test the response callback of the HandoffOrchestration with a handoff function call."""
agent_b = MockAgent()
agent_a = MockAgentWithHandoffFunctionCall(agent_b)
runtime = InProcessRuntime()
runtime.start()
responses: list[DefaultTypeAlias] = []
try:
orchestration = HandoffOrchestration(
members=[agent_a, agent_b],
handoffs={agent_a.name: {agent_b.name: "test"}},
agent_response_callback=lambda x: responses.append(x),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert len(responses) == 3
assert responses[0].name == agent_a.name
assert isinstance(responses[0].items[0], FunctionCallContent)
assert responses[1].name == agent_a.name
assert isinstance(responses[1].items[0], FunctionResultContent)
assert responses[2].name == agent_b.name
async def test_streaming_response_callback_with_handoff_function_call():
"""Test the streaming sresponse callback of the HandoffOrchestration with a handoff function call."""
agent_b = MockAgent()
agent_a = MockAgentWithHandoffFunctionCall(agent_b)
runtime = InProcessRuntime()
runtime.start()
responses: dict[str, list[StreamingChatMessageContent]] = {}
try:
orchestration = HandoffOrchestration(
members=[agent_a, agent_b],
handoffs={agent_a.name: {agent_b.name: "test"}},
streaming_agent_response_callback=lambda x, _: responses.setdefault(x.name, []).append(x),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert len(responses[agent_a.name]) == 2
assert len(responses[agent_b.name]) == 2 # 2 chunks
assert responses[agent_a.name][0].name == agent_a.name
assert isinstance(responses[agent_a.name][0].items[0], FunctionCallContent)
assert responses[agent_a.name][1].name == agent_a.name
assert isinstance(responses[agent_a.name][1].items[0], FunctionResultContent)
assert responses[agent_b.name][0].name == agent_b.name
assert all([isinstance(response, StreamingChatMessageContent) for response in responses[agent_b.name]])
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke_with_human_response_function():
"""Test the invoke method of the HandoffOrchestration with a human response function."""
complete_task_agent_instance = MockAgentWithCompleteTaskFunctionCall()
normal_agent_instance = MockAgent()
call_sequence = iter([normal_agent_instance.invoke_stream, complete_task_agent_instance.invoke_stream])
user_input_count = 0
def human_response_function() -> ChatMessageContent:
# Simulate user input
nonlocal user_input_count
user_input_count += 1
return ChatMessageContent(role=AuthorRole.USER, content="user_input")
with (
patch.object(MockAgent, "invoke_stream") as mock_invoke_stream,
):
mock_invoke_stream.side_effect = lambda *args, **kwargs: next(call_sequence)(*args, **kwargs)
agent_a = MockAgent(name="agent_a")
agent_b = MockAgent(name="agent_b")
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = HandoffOrchestration(
members=[agent_a, agent_b],
handoffs={agent_a.name: {agent_b.name: "test"}},
human_response_function=human_response_function,
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert user_input_count == 1
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke_with_handoff_function_call():
"""Test the invoke method of the HandoffOrchestration with a handoff function call."""
agent_b = MockAgent()
agent_a = MockAgentWithHandoffFunctionCall(agent_b)
with (
patch.object(
HandoffAgentActor, "_handoff_to_agent", wraps=HandoffAgentActor._handoff_to_agent, autospec=True
) as mock_handoff_to_agent,
):
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = HandoffOrchestration(
members=[agent_a, agent_b],
handoffs={agent_a.name: {agent_b.name: "test"}},
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert mock_handoff_to_agent.call_count == 1
assert mock_handoff_to_agent.call_args_list[0][0][1] == agent_b.name
# The kernel in the agent should not be modified
assert len(agent_a.kernel.plugins) == 0
assert len(agent_b.kernel.plugins) == 0
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke_cancel_before_completion():
"""Test the invoke method of the HandoffOrchestration with cancellation before completion."""
with (
patch.object(MockAgent, "invoke_stream", wraps=MockAgent.invoke_stream, autospec=True) as mock_invoke_stream,
):
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = HandoffOrchestration(
members=[agent_a, agent_b],
handoffs={agent_a.name: {agent_b.name: "test"}},
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
# Cancel before first agent completes
await asyncio.sleep(0.05)
orchestration_result.cancel()
finally:
await runtime.stop_when_idle()
assert mock_invoke_stream.call_count == 1
async def test_invoke_cancel_after_completion():
"""Test the invoke method of the HandoffOrchestration with cancellation after completion."""
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = HandoffOrchestration(
members=[agent_a, agent_b],
handoffs={agent_a.name: {agent_b.name: "test"}},
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
# Wait for the orchestration to complete
await orchestration_result.get(1.0)
with pytest.raises(RuntimeError, match="The invocation has already been completed."):
orchestration_result.cancel()
finally:
await runtime.stop_when_idle()
async def test_invoke_with_agent_raising_exception():
"""Test the invoke method of the HandoffOrchestration with an agent raising an exception."""
agent_a = MockAgentWithException()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = HandoffOrchestration(
members=[agent_a, agent_b],
handoffs={agent_a.name: {agent_b.name: "test"}},
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
with pytest.raises(RuntimeError, match="Mock agent exception"):
await orchestration_result.get(1.0)
assert orchestration_result.exception is not None
finally:
await runtime.stop_when_idle()
# endregion
@@ -0,0 +1,806 @@
# Copyright (c) Microsoft. All rights reserved.
import sys
from typing import Any, Literal
from unittest.mock import AsyncMock, patch
import pytest
from pydantic import BaseModel
from semantic_kernel.agents.orchestration.magentic import (
MagenticContext,
MagenticOrchestration,
ProgressLedger,
ProgressLedgerItem,
StandardMagenticManager,
)
from semantic_kernel.agents.orchestration.orchestration_base import DefaultTypeAlias, OrchestrationResult
from semantic_kernel.agents.orchestration.prompts._magentic_prompts import (
ORCHESTRATOR_FINAL_ANSWER_PROMPT,
ORCHESTRATOR_PROGRESS_LEDGER_PROMPT,
ORCHESTRATOR_TASK_LEDGER_FACTS_PROMPT,
ORCHESTRATOR_TASK_LEDGER_FACTS_UPDATE_PROMPT,
ORCHESTRATOR_TASK_LEDGER_FULL_PROMPT,
ORCHESTRATOR_TASK_LEDGER_PLAN_PROMPT,
ORCHESTRATOR_TASK_LEDGER_PLAN_UPDATE_PROMPT,
)
from semantic_kernel.agents.runtime.in_process.in_process_runtime import InProcessRuntime
from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from tests.unit.agents.orchestration.conftest import MockAgent, MockAgentWithException, MockRuntime
class MockChatCompletionService(ChatCompletionClientBase):
"""A mock chat completion service for testing purposes."""
pass
class MockPromptExecutionSettings(PromptExecutionSettings):
"""A mock prompt execution settings class for testing purposes."""
response_format: (
dict[Literal["type"], Literal["text", "json_object"]] | dict[str, Any] | type[BaseModel] | type | None
) = None
# region MagenticOrchestration
async def test_init_member_without_description_throws():
"""Test the prepare method of the MagenticOrchestration with a member without description."""
agent_a = MockAgent()
agent_b = MockAgent()
with pytest.raises(ValueError):
MagenticOrchestration(
members=[agent_a, agent_b],
manager=StandardMagenticManager(
chat_completion_service=MockChatCompletionService(ai_model_id="test"),
prompt_execution_settings=MockPromptExecutionSettings(),
),
)
async def test_prepare():
"""Test the prepare method of the MagenticOrchestration."""
agent_a = MockAgent(description="test agent")
agent_b = MockAgent(description="test agent")
runtime = MockRuntime()
package_path = "semantic_kernel.agents.orchestration.magentic"
with (
patch(f"{package_path}.MagenticOrchestration._start"),
patch(f"{package_path}.MagenticAgentActor.register") as mock_agent_actor_register,
patch(f"{package_path}.MagenticManagerActor.register") as mock_manager_actor_register,
patch.object(runtime, "add_subscription") as mock_add_subscription,
):
orchestration = MagenticOrchestration(
members=[agent_a, agent_b],
manager=StandardMagenticManager(
chat_completion_service=MockChatCompletionService(ai_model_id="test"),
prompt_execution_settings=MockPromptExecutionSettings(),
),
)
await orchestration.invoke(task="test_message", runtime=runtime)
assert mock_agent_actor_register.call_count == 2
assert mock_manager_actor_register.call_count == 1
assert mock_add_subscription.call_count == 3
ManagerProgressList = [
ProgressLedger(
is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"),
next_speaker=ProgressLedgerItem(answer="agent_a", reason="mock_reasoning"),
instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"),
),
ProgressLedger(
is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"),
next_speaker=ProgressLedgerItem(answer="agent_b", reason="mock_reasoning"),
instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"),
),
ProgressLedger(
is_request_satisfied=ProgressLedgerItem(answer=True, reason="mock_reasoning"),
is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"),
next_speaker=ProgressLedgerItem(answer="N/A", reason="mock_reasoning"),
instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"),
),
]
ManagerProgressListStalling = [
ProgressLedger(
is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"),
next_speaker=ProgressLedgerItem(answer="agent_a", reason="mock_reasoning"),
instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"),
),
ProgressLedger(
is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_in_loop=ProgressLedgerItem(answer=True, reason="mock_reasoning"), # is_in_loop=True
is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"),
next_speaker=ProgressLedgerItem(answer="agent_a", reason="mock_reasoning"),
instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"),
),
ProgressLedger(
is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_in_loop=ProgressLedgerItem(answer=True, reason="mock_reasoning"), # is_in_loop=True
is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"),
next_speaker=ProgressLedgerItem(answer="N/A", reason="mock_reasoning"),
instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"),
),
ProgressLedger(
is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"),
next_speaker=ProgressLedgerItem(answer="agent_b", reason="mock_reasoning"),
instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"),
),
ProgressLedger(
is_request_satisfied=ProgressLedgerItem(answer=True, reason="mock_reasoning"),
is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"),
next_speaker=ProgressLedgerItem(answer="N/A", reason="mock_reasoning"),
instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"),
),
]
ManagerProgressListUnknownSpeaker = [
ProgressLedger(
is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_progress_being_made=ProgressLedgerItem(answer=True, reason="mock_reasoning"),
next_speaker=ProgressLedgerItem(answer="unknown", reason="mock_reasoning"),
instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"),
),
]
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke():
"""Test the invoke method of the MagenticOrchestration."""
with (
patch.object(MockAgent, "invoke_stream", wraps=MockAgent.invoke_stream, autospec=True) as mock_invoke_stream,
patch.object(
MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock
) as mock_get_chat_message_content,
patch.object(
StandardMagenticManager, "create_progress_ledger", new_callable=AsyncMock, side_effect=ManagerProgressList
),
):
mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response")
chat_completion_service = MockChatCompletionService(ai_model_id="test")
prompt_execution_settings = MockPromptExecutionSettings()
manager = StandardMagenticManager(
chat_completion_service=chat_completion_service,
prompt_execution_settings=prompt_execution_settings,
)
agent_a = MockAgent(name="agent_a", description="test agent")
agent_b = MockAgent(name="agent_b", description="test agent")
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = MagenticOrchestration(members=[agent_a, agent_b], manager=manager)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
result = await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert isinstance(orchestration_result, OrchestrationResult)
assert isinstance(result, ChatMessageContent)
assert result.role == AuthorRole.ASSISTANT
assert result.content == "mock_response"
assert mock_invoke_stream.call_count == 2
assert mock_get_chat_message_content.call_count == 3
async def test_invoke_with_list_error():
"""Test the invoke method of the MagenticOrchestration with a list of messages which raises an error."""
chat_completion_service = MockChatCompletionService(ai_model_id="test")
prompt_execution_settings = MockPromptExecutionSettings()
manager = StandardMagenticManager(
chat_completion_service=chat_completion_service,
prompt_execution_settings=prompt_execution_settings,
)
agent_a = MockAgent(name="agent_a", description="test agent")
agent_b = MockAgent(name="agent_b", description="test agent")
messages = [
ChatMessageContent(role=AuthorRole.USER, content="test_message_1"),
ChatMessageContent(role=AuthorRole.USER, content="test_message_2"),
]
runtime = MockRuntime()
package_path = "semantic_kernel.agents.orchestration.magentic"
with (
patch(f"{package_path}.MagenticAgentActor.register"),
patch(f"{package_path}.MagenticManagerActor.register"),
patch.object(runtime, "add_subscription"),
pytest.raises(ValueError),
):
orchestration = MagenticOrchestration(members=[agent_a, agent_b], manager=manager)
orchestration_result = await orchestration.invoke(task=messages, runtime=runtime)
await orchestration_result.get(1.0)
async def test_invoke_with_agent_raising_exception():
"""Test the invoke method of the MagenticOrchestration with a list of messages which raises an error."""
with (
patch.object(
MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock
) as mock_get_chat_message_content,
patch.object(
StandardMagenticManager, "create_progress_ledger", new_callable=AsyncMock, side_effect=ManagerProgressList
),
):
mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response")
chat_completion_service = MockChatCompletionService(ai_model_id="test")
prompt_execution_settings = MockPromptExecutionSettings()
manager = StandardMagenticManager(
chat_completion_service=chat_completion_service,
prompt_execution_settings=prompt_execution_settings,
)
agent_a = MockAgentWithException(name="agent_a", description="test agent")
agent_b = MockAgent(name="agent_b", description="test agent")
runtime = InProcessRuntime()
runtime.start()
orchestration = MagenticOrchestration(members=[agent_a, agent_b], manager=manager)
try:
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
with pytest.raises(RuntimeError, match="Mock agent exception"):
await orchestration_result.get(1.0)
assert orchestration_result.exception is not None
finally:
await runtime.stop_when_idle()
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke_with_response_callback():
"""Test the invoke method of the MagenticOrchestration with a response callback."""
runtime = InProcessRuntime()
runtime.start()
responses: list[DefaultTypeAlias] = []
with (
patch.object(
MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock
) as mock_get_chat_message_content,
patch.object(
StandardMagenticManager, "create_progress_ledger", new_callable=AsyncMock, side_effect=ManagerProgressList
),
):
mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response")
agent_a = MockAgent(name="agent_a", description="test agent")
agent_b = MockAgent(name="agent_b", description="test agent")
try:
orchestration = MagenticOrchestration(
members=[agent_a, agent_b],
manager=StandardMagenticManager(
chat_completion_service=MockChatCompletionService(ai_model_id="test"),
prompt_execution_settings=MockPromptExecutionSettings(),
),
agent_response_callback=lambda x: responses.append(x),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert len(responses) == 2
assert all(isinstance(item, ChatMessageContent) for item in responses)
assert all(item.content == "mock_response" for item in responses)
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke_with_streaming_response_callback():
"""Test the invoke method of the MagenticOrchestration with a streaming response callback."""
runtime = InProcessRuntime()
runtime.start()
responses: dict[str, list[StreamingChatMessageContent]] = {}
with (
patch.object(
MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock
) as mock_get_chat_message_content,
patch.object(
StandardMagenticManager, "create_progress_ledger", new_callable=AsyncMock, side_effect=ManagerProgressList
),
):
mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response")
agent_a = MockAgent(name="agent_a", description="test agent")
agent_b = MockAgent(name="agent_b", description="test agent")
try:
orchestration = MagenticOrchestration(
members=[agent_a, agent_b],
manager=StandardMagenticManager(
chat_completion_service=MockChatCompletionService(ai_model_id="test"),
prompt_execution_settings=MockPromptExecutionSettings(),
),
streaming_agent_response_callback=lambda x, _: responses.setdefault(x.name, []).append(x),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert len(responses[agent_a.name]) == 2
assert len(responses[agent_b.name]) == 2
assert all(isinstance(item, StreamingChatMessageContent) for item in responses[agent_a.name])
assert all(isinstance(item, StreamingChatMessageContent) for item in responses[agent_b.name])
agent_a_response = sum(responses[agent_a.name][1:], responses[agent_a.name][0])
assert agent_a_response.content == "mock_response"
agent_b_response = sum(responses[agent_b.name][1:], responses[agent_b.name][0])
assert agent_b_response.content == "mock_response"
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke_with_max_stall_count_exceeded():
""" "Test the invoke method of the MagenticOrchestration with max stall count exceeded."""
runtime = InProcessRuntime()
runtime.start()
with (
patch.object(MockAgent, "invoke_stream", wraps=MockAgent.invoke_stream, autospec=True) as mock_invoke_stream,
patch.object(
MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock
) as mock_get_chat_message_content,
patch.object(
StandardMagenticManager,
"create_progress_ledger",
new_callable=AsyncMock,
side_effect=ManagerProgressListStalling,
),
):
mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response")
agent_a = MockAgent(name="agent_a", description="test agent")
agent_b = MockAgent(name="agent_b", description="test agent")
try:
orchestration = MagenticOrchestration(
members=[agent_a, agent_b],
manager=StandardMagenticManager(
chat_completion_service=MockChatCompletionService(ai_model_id="test"),
prompt_execution_settings=MockPromptExecutionSettings(),
max_stall_count=1,
),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
assert mock_invoke_stream.call_count == 3
# Exceeding max stall count will trigger replanning, which will recreate the facts and plan,
# resulting in two additional calls to get_chat_message_content compared to the `test_invoke` test.
assert mock_get_chat_message_content.call_count == 5
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke_with_max_round_count_exceeded():
""" "Test the invoke method of the MagenticOrchestration with max round count exceeded."""
runtime = InProcessRuntime()
runtime.start()
with (
patch.object(MockAgent, "invoke_stream", wraps=MockAgent.invoke_stream, autospec=True) as mock_invoke_stream,
patch.object(
MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock
) as mock_get_chat_message_content,
patch.object(
StandardMagenticManager,
"create_progress_ledger",
new_callable=AsyncMock,
side_effect=ManagerProgressListStalling,
),
):
mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response")
agent_a = MockAgent(name="agent_a", description="test agent")
agent_b = MockAgent(name="agent_b", description="test agent")
try:
orchestration = MagenticOrchestration(
members=[agent_a, agent_b],
manager=StandardMagenticManager(
chat_completion_service=MockChatCompletionService(ai_model_id="test"),
prompt_execution_settings=MockPromptExecutionSettings(),
max_round_count=1,
),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
result = await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
# Partial result will be returned when max round count is exceeded.
assert result.content == mock_get_chat_message_content.return_value.content
assert mock_invoke_stream.call_count == 1
# Planning will be called once, so the facts and plan will be created once.
assert mock_get_chat_message_content.call_count == 2
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke_with_max_reset_count_exceeded():
""" "Test the invoke method of the MagenticOrchestration with max reset count exceeded."""
runtime = InProcessRuntime()
runtime.start()
with (
patch.object(MockAgent, "invoke_stream", wraps=MockAgent.invoke_stream, autospec=True) as mock_invoke_stream,
patch.object(
MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock
) as mock_get_chat_message_content,
patch.object(
StandardMagenticManager,
"create_progress_ledger",
new_callable=AsyncMock,
side_effect=ManagerProgressListStalling,
),
):
mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response")
agent_a = MockAgent(name="agent_a", description="test agent")
agent_b = MockAgent(name="agent_b", description="test agent")
try:
orchestration = MagenticOrchestration(
members=[agent_a, agent_b],
manager=StandardMagenticManager(
chat_completion_service=MockChatCompletionService(ai_model_id="test"),
prompt_execution_settings=MockPromptExecutionSettings(),
max_stall_count=0, # No stall allowed
max_reset_count=0, # No reset allowed
),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
result = await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
# Partial result will be returned when max reset count is exceeded. The test emits content based on the prompt
# so check that the content is not None and not an exact match to a mock response.
assert result.content is not None
assert mock_invoke_stream.call_count == 1
# Planning and replanning will be each called once, so the facts and plan will be created twice.
assert mock_get_chat_message_content.call_count == 4
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke_with_unknown_speaker():
"""Test the invoke method of the MagenticOrchestration with an unknown speaker."""
runtime = InProcessRuntime()
runtime.start()
with (
patch.object(
MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock
) as mock_get_chat_message_content,
patch.object(
StandardMagenticManager,
"create_progress_ledger",
new_callable=AsyncMock,
side_effect=ManagerProgressListUnknownSpeaker,
),
pytest.raises(ValueError),
):
mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response")
agent_a = MockAgent(name="agent_a", description="test agent")
agent_b = MockAgent(name="agent_b", description="test agent")
try:
orchestration = MagenticOrchestration(
members=[agent_a, agent_b],
manager=StandardMagenticManager(
chat_completion_service=MockChatCompletionService(ai_model_id="test"),
prompt_execution_settings=MockPromptExecutionSettings(),
),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
finally:
await runtime.stop_when_idle()
# endregion MagenticOrchestration
# region StandardMagenticManager
def test_standard_magentic_manager_init():
"""Test the initialization of the StandardMagenticManager."""
chat_completion_service = MockChatCompletionService(ai_model_id="test")
prompt_execution_settings = MockPromptExecutionSettings()
manager = StandardMagenticManager(
chat_completion_service=chat_completion_service,
prompt_execution_settings=prompt_execution_settings,
)
assert manager.max_stall_count > 0
assert manager.max_reset_count is None
assert manager.max_round_count is None
assert (
manager.task_ledger_facts_prompt is not None
and manager.task_ledger_facts_prompt == ORCHESTRATOR_TASK_LEDGER_FACTS_PROMPT
)
assert (
manager.task_ledger_plan_prompt is not None
and manager.task_ledger_plan_prompt == ORCHESTRATOR_TASK_LEDGER_PLAN_PROMPT
)
assert (
manager.task_ledger_full_prompt is not None
and manager.task_ledger_full_prompt == ORCHESTRATOR_TASK_LEDGER_FULL_PROMPT
)
assert (
manager.task_ledger_facts_update_prompt is not None
and manager.task_ledger_facts_update_prompt == ORCHESTRATOR_TASK_LEDGER_FACTS_UPDATE_PROMPT
)
assert (
manager.task_ledger_plan_update_prompt is not None
and manager.task_ledger_plan_update_prompt == ORCHESTRATOR_TASK_LEDGER_PLAN_UPDATE_PROMPT
)
assert (
manager.progress_ledger_prompt is not None
and manager.progress_ledger_prompt == ORCHESTRATOR_PROGRESS_LEDGER_PROMPT
)
assert manager.final_answer_prompt is not None and manager.final_answer_prompt == ORCHESTRATOR_FINAL_ANSWER_PROMPT
def test_standard_magentic_manager_init_with_custom_prompts():
"""Test the initialization of the StandardMagenticManager with custom prompts."""
chat_completion_service = MockChatCompletionService(ai_model_id="test")
prompt_execution_settings = MockPromptExecutionSettings()
manager = StandardMagenticManager(
chat_completion_service=chat_completion_service,
prompt_execution_settings=prompt_execution_settings,
task_ledger_facts_prompt="custom_task_ledger_facts_prompt",
task_ledger_plan_prompt="custom_task_ledger_plan_prompt",
task_ledger_full_prompt="custom_task_ledger_full_prompt",
task_ledger_facts_update_prompt="custom_task_ledger_facts_update_prompt",
task_ledger_plan_update_prompt="custom_task_ledger_plan_update_prompt",
progress_ledger_prompt="custom_progress_ledger_prompt",
final_answer_prompt="custom_final_answer_prompt",
)
assert manager.task_ledger_facts_prompt == "custom_task_ledger_facts_prompt"
assert manager.task_ledger_plan_prompt == "custom_task_ledger_plan_prompt"
assert manager.task_ledger_full_prompt == "custom_task_ledger_full_prompt"
assert manager.task_ledger_facts_update_prompt == "custom_task_ledger_facts_update_prompt"
assert manager.task_ledger_plan_update_prompt == "custom_task_ledger_plan_update_prompt"
assert manager.progress_ledger_prompt == "custom_progress_ledger_prompt"
assert manager.final_answer_prompt == "custom_final_answer_prompt"
def test_standard_magentic_manager_init_with_invalid_prompt_execution_settings():
"""Test the initialization of the StandardMagenticManager with invalid prompt execution settings."""
chat_completion_service = MockChatCompletionService(ai_model_id="test")
prompt_execution_settings = PromptExecutionSettings()
with pytest.raises(ValueError):
StandardMagenticManager(
chat_completion_service=chat_completion_service,
prompt_execution_settings=prompt_execution_settings,
)
def test_standard_magentic_manager_init_without_prompt_execution_settings():
"""Test the initialization of the StandardMagenticManager without prompt execution settings."""
# The default prompt execution settings of the mock chat completion service
# does not support structured output.
chat_completion_service = MockChatCompletionService(ai_model_id="test")
with pytest.raises(ValueError):
StandardMagenticManager(chat_completion_service=chat_completion_service)
async def test_standard_magentic_manager_plan():
"""Test the plan method of the StandardMagenticManager."""
with patch.object(
MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock
) as mock_get_chat_message_content:
mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response")
chat_completion_service = MockChatCompletionService(ai_model_id="test")
prompt_execution_settings = MockPromptExecutionSettings()
manager = StandardMagenticManager(
chat_completion_service=chat_completion_service,
prompt_execution_settings=prompt_execution_settings,
task_ledger_facts_prompt="custom_task_ledger_facts_prompt",
task_ledger_plan_prompt="custom_task_ledger_plan_prompt {{$team}}",
)
magentic_context = MagenticContext(
chat_history=ChatHistory(),
task=ChatMessageContent(role="user", content="test_message"),
participant_descriptions={"agent_a": "test_agent_a", "agent_b": "test_agent_b"},
)
task_ledger = await manager.plan(magentic_context.model_copy(deep=True))
assert isinstance(task_ledger, ChatMessageContent)
assert task_ledger.content.count("mock_response") == 2
assert "test_message" in task_ledger.content
assert "{'agent_a': 'test_agent_a', 'agent_b': 'test_agent_b'}" in task_ledger.content
assert mock_get_chat_message_content.call_count == 2
assert (
mock_get_chat_message_content.call_args_list[0][0][0].messages[0].content
== "custom_task_ledger_facts_prompt"
)
assert (
mock_get_chat_message_content.call_args_list[1][0][0].messages[2].content
== "custom_task_ledger_plan_prompt {'agent_a': 'test_agent_a', 'agent_b': 'test_agent_b'}"
)
async def test_standard_magentic_manager_replan():
"""Test the replan method of the StandardMagenticManager."""
with patch.object(
MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock
) as mock_get_chat_message_content:
mock_get_chat_message_content.return_value = ChatMessageContent(role="assistant", content="mock_response")
chat_completion_service = MockChatCompletionService(ai_model_id="test")
prompt_execution_settings = MockPromptExecutionSettings()
manager = StandardMagenticManager(
chat_completion_service=chat_completion_service,
prompt_execution_settings=prompt_execution_settings,
task_ledger_facts_update_prompt="custom_task_ledger_facts_prompt {{$old_facts}}",
task_ledger_plan_update_prompt="custom_task_ledger_plan_prompt {{$team}}",
)
magentic_context = MagenticContext(
chat_history=ChatHistory(),
task=ChatMessageContent(role="user", content="test_message"),
participant_descriptions={"agent_a": "test_agent_a", "agent_b": "test_agent_b"},
)
# Need to plan before replanning
_ = await manager.plan(magentic_context.model_copy(deep=True))
task_ledger = await manager.replan(magentic_context.model_copy(deep=True))
assert isinstance(task_ledger, ChatMessageContent)
assert task_ledger.content.count("mock_response") == 2
assert "test_message" in task_ledger.content
assert "{'agent_a': 'test_agent_a', 'agent_b': 'test_agent_b'}" in task_ledger.content
assert mock_get_chat_message_content.call_count == 4
assert (
mock_get_chat_message_content.call_args_list[2][0][0].messages[0].content
== "custom_task_ledger_facts_prompt mock_response"
)
assert (
mock_get_chat_message_content.call_args_list[3][0][0].messages[2].content
== "custom_task_ledger_plan_prompt {'agent_a': 'test_agent_a', 'agent_b': 'test_agent_b'}"
)
async def test_standard_magentic_manager_replan_without_plan():
"""Test the replan method of the StandardMagenticManager."""
chat_completion_service = MockChatCompletionService(ai_model_id="test")
prompt_execution_settings = MockPromptExecutionSettings()
manager = StandardMagenticManager(
chat_completion_service=chat_completion_service,
prompt_execution_settings=prompt_execution_settings,
)
magentic_context = MagenticContext(
chat_history=ChatHistory(),
task=ChatMessageContent(role="user", content="test_message"),
participant_descriptions={"agent_a": "test_agent_a", "agent_b": "test_agent_b"},
)
with pytest.raises(RuntimeError):
_ = await manager.replan(magentic_context.model_copy(deep=True))
async def test_standard_magentic_manager_create_progress_ledger():
"""Test the create_progress_ledger method of the StandardMagenticManager."""
mock_progress_ledger = ProgressLedger(
is_request_satisfied=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_in_loop=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
is_progress_being_made=ProgressLedgerItem(answer=False, reason="mock_reasoning"),
next_speaker=ProgressLedgerItem(answer="agent_a", reason="mock_reasoning"),
instruction_or_question=ProgressLedgerItem(answer="mock_instruction", reason="mock_reasoning"),
)
with patch.object(
MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock
) as mock_get_chat_message_content:
mock_get_chat_message_content.return_value = ChatMessageContent(
role="assistant", content=mock_progress_ledger.model_dump_json()
)
chat_completion_service = MockChatCompletionService(ai_model_id="test")
prompt_execution_settings = MockPromptExecutionSettings()
manager = StandardMagenticManager(
chat_completion_service=chat_completion_service,
prompt_execution_settings=prompt_execution_settings,
)
magentic_context = MagenticContext(
chat_history=ChatHistory(),
task=ChatMessageContent(role="user", content="test_message"),
participant_descriptions={"agent_a": "test_agent_a", "agent_b": "test_agent_b"},
)
progress_ledger = await manager.create_progress_ledger(magentic_context.model_copy(deep=True))
assert isinstance(progress_ledger, ProgressLedger)
assert progress_ledger == mock_progress_ledger
assert (
"{'agent_a': 'test_agent_a', 'agent_b': 'test_agent_b'}"
in mock_get_chat_message_content.call_args_list[0][0][0].messages[0].content
)
assert "agent_a, agent_b" in mock_get_chat_message_content.call_args_list[0][0][0].messages[0].content
assert (
magentic_context.task.content in mock_get_chat_message_content.call_args_list[0][0][0].messages[0].content
)
assert mock_get_chat_message_content.call_args_list[0][0][1].extension_data["response_format"] == ProgressLedger
# endregion MagenticManager
@@ -0,0 +1,422 @@
# Copyright (c) Microsoft. All rights reserved.
import asyncio
import json
import sys
from collections.abc import AsyncIterable, Awaitable, Callable
from dataclasses import dataclass
from unittest.mock import ANY, patch
import pytest
from semantic_kernel.agents.agent import Agent, AgentResponseItem, AgentThread
from semantic_kernel.agents.orchestration.orchestration_base import DefaultTypeAlias, OrchestrationBase, TIn, TOut
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.kernel_pydantic import KernelBaseModel
from tests.unit.agents.orchestration.conftest import MockRuntime
if sys.version_info >= (3, 12):
from typing import override # pragma: no cover
else:
from typing_extensions import override # pragma: no cover
class MockAgent(Agent):
"""A mock agent for testing purposes."""
@override
async def get_response(
self,
*,
messages: str | ChatMessageContent | list[str | ChatMessageContent] | None = None,
thread: AgentThread | None = None,
**kwargs,
) -> AgentResponseItem[ChatMessageContent]:
pass
@override
async def invoke(
self,
*,
messages: str | ChatMessageContent | list[str | ChatMessageContent] | None = None,
thread: AgentThread | None = None,
on_intermediate_message: Callable[[ChatMessageContent], Awaitable[None]] | None = None,
**kwargs,
) -> AgentResponseItem[ChatMessageContent]:
pass
@override
async def invoke_stream(
self,
*,
messages: str | ChatMessageContent | list[str | ChatMessageContent] | None = None,
thread: AgentThread | None = None,
on_intermediate_message: Callable[[ChatMessageContent], Awaitable[None]] | None = None,
**kwargs,
) -> AsyncIterable[AgentResponseItem[StreamingChatMessageContent]]:
pass
class MockOrchestration(OrchestrationBase[TIn, TOut]):
"""A mock orchestration base for testing purposes."""
async def _start(self, task, runtime, internal_topic_type, collection_agent_type):
pass
async def _prepare(self, runtime, internal_topic_type, exception_callback, result_callback):
pass
def test_orchestration_init():
"""Test the initialization of the MockOrchestration."""
agent_a = MockAgent()
agent_b = MockAgent()
agent_c = MockAgent()
orchestration = MockOrchestration(
members=[agent_a, agent_b, agent_c],
name="test_orchestration",
description="Test Orchestration",
)
assert orchestration.name == "test_orchestration"
assert orchestration.description == "Test Orchestration"
assert len(orchestration._members) == 3
assert orchestration._input_transform is not None
assert orchestration._output_transform is not None
assert orchestration._agent_response_callback is None
def test_orchestration_init_with_default_values():
"""Test the initialization of the MockOrchestration with default values."""
agent_a = MockAgent()
agent_b = MockAgent()
orchestration = MockOrchestration(members=[agent_a, agent_b])
assert orchestration.name
assert orchestration.description
assert len(orchestration._members) == 2
assert orchestration._input_transform is not None
assert orchestration._output_transform is not None
assert orchestration._agent_response_callback is None
def test_orchestration_init_with_no_members():
"""Test the initialization of the OrchestrationBase with no members."""
with pytest.raises(ValueError):
_ = MockOrchestration(members=[])
def test_orchestration_set_types():
"""Test the set_types method of OrchestrationBase."""
agent_a = MockAgent()
agent_b = MockAgent()
# Test with default types
orchestration_a = MockOrchestration(members=[agent_a, agent_b])
orchestration_a._set_types()
assert orchestration_a.t_in is DefaultTypeAlias
assert orchestration_a.t_out is DefaultTypeAlias
# Test with a custom input type and default output type
orchestration_c = MockOrchestration[int](members=[agent_a, agent_b])
orchestration_c._set_types()
assert orchestration_c.t_in is int
assert orchestration_c.t_out is DefaultTypeAlias
# Test with a custom input type and custom output type
orchestration_b = MockOrchestration[str, int](members=[agent_a, agent_b])
orchestration_b._set_types()
assert orchestration_b.t_in is str
assert orchestration_b.t_out is int
# Test with an incorrect number of types
with pytest.raises(TypeError):
orchestration_d = MockOrchestration[str, str, str](members=[agent_a, agent_b])
orchestration_d._set_types()
async def test_orchestration_invoke_with_str():
"""Test the invoke method of OrchestrationBase with a string input."""
orchestration = MockOrchestration(members=[MockAgent(), MockAgent()])
with patch.object(orchestration, "_start") as mock_start:
await orchestration.invoke("Test message", MockRuntime())
mock_start.assert_called_once_with(
ChatMessageContent(role=AuthorRole.USER, content="Test message"), ANY, ANY, ANY
)
async def test_orchestration_invoke_with_chat_message_content():
"""Test the invoke method of OrchestrationBase with a ChatMessageContent input."""
orchestration = MockOrchestration(members=[MockAgent(), MockAgent()])
chat_message_content = ChatMessageContent(role=AuthorRole.USER, content="Test message")
with patch.object(orchestration, "_start") as mock_start:
await orchestration.invoke(chat_message_content, MockRuntime())
mock_start.assert_called_once_with(chat_message_content, ANY, ANY, ANY)
chat_message_content_list = [
ChatMessageContent(role=AuthorRole.USER, content="Test message 1"),
ChatMessageContent(role=AuthorRole.USER, content="Test message 2"),
]
with patch.object(orchestration, "_start") as mock_start:
await orchestration.invoke(chat_message_content_list, MockRuntime())
mock_start.assert_called_once_with(chat_message_content_list, ANY, ANY, ANY)
async def test_orchestration_invoke_with_custom_type():
"""Test the invoke method of OrchestrationBase with a custom type."""
@dataclass
class CustomType:
value: str
number: int
orchestration = MockOrchestration[CustomType, TOut](members=[MockAgent(), MockAgent()])
custom_type_instance = CustomType(value="Test message", number=42)
with patch.object(orchestration, "_start") as mock_start:
await orchestration.invoke(custom_type_instance, MockRuntime())
mock_start.assert_called_once_with(
ChatMessageContent(role=AuthorRole.USER, content=json.dumps(custom_type_instance.__dict__)), ANY, ANY, ANY
)
async def test_orchestration_invoke_with_custom_type_async_input_transform():
"""Test the invoke method of OrchestrationBase with a custom type and async input transform."""
@dataclass
class CustomType:
value: str
number: int
async def async_input_transform(input_data: CustomType) -> ChatMessageContent:
await asyncio.sleep(0.1) # Simulate async processing
return ChatMessageContent(role=AuthorRole.USER, content="Test message")
orchestration = MockOrchestration[CustomType, TOut](
members=[MockAgent(), MockAgent()],
input_transform=async_input_transform,
)
custom_type_instance = CustomType(value="Test message", number=42)
with patch.object(orchestration, "_start") as mock_start:
await orchestration.invoke(custom_type_instance, MockRuntime())
mock_start.assert_called_once_with(
ChatMessageContent(role=AuthorRole.USER, content="Test message"), ANY, ANY, ANY
)
def test_default_input_transform_default_type_alias():
"""Test the default_input_transform method of OrchestrationBase."""
orchestration = MockOrchestration(members=[MockAgent(), MockAgent()])
orchestration._set_types()
chat_message_content = ChatMessageContent(role=AuthorRole.USER, content="Test message")
transformed_input = orchestration._default_input_transform(chat_message_content)
assert isinstance(transformed_input, ChatMessageContent)
chat_message_content_list = [
ChatMessageContent(role=AuthorRole.USER, content="Test message 1"),
ChatMessageContent(role=AuthorRole.USER, content="Test message 2"),
]
transformed_input_list = orchestration._default_input_transform(chat_message_content_list)
assert isinstance(transformed_input_list, list) and all(
isinstance(item, ChatMessageContent) for item in transformed_input_list
)
def test_default_input_transform_custom_type():
"""Test the default_input_transform method of OrchestrationBase with a custom type."""
@dataclass
class CustomType:
value: str
number: int
orchestration_a = MockOrchestration[CustomType, TOut](members=[MockAgent(), MockAgent()])
orchestration_a._set_types()
custom_type_instance = CustomType(value="Test message", number=42)
transformed_input_a = orchestration_a._default_input_transform(custom_type_instance)
assert isinstance(transformed_input_a, ChatMessageContent)
assert CustomType(**json.loads(transformed_input_a.content)) == custom_type_instance
assert transformed_input_a.role == AuthorRole.USER
class CustomModel(KernelBaseModel):
value: str
number: int
orchestration_b = MockOrchestration[CustomModel, TOut](members=[MockAgent(), MockAgent()])
orchestration_b._set_types()
custom_model_instance = CustomModel(value="Test message", number=42)
transformed_input_b = orchestration_b._default_input_transform(custom_model_instance)
assert isinstance(transformed_input_b, ChatMessageContent)
assert CustomModel.model_validate_json(transformed_input_b.content) == custom_model_instance
assert transformed_input_b.role == AuthorRole.USER
def test_default_input_transform_custom_type_error():
"""Test the default_input_transform method of OrchestrationBase with an incorrect type."""
@dataclass
class CustomType:
value: str
number: int
class CustomModel(KernelBaseModel):
value: str
number: int
orchestration = MockOrchestration[CustomModel, TOut](members=[MockAgent(), MockAgent()])
orchestration._set_types()
with pytest.raises(TypeError):
custom_type_instance = CustomType(value="Test message", number=42)
orchestration._default_input_transform(custom_type_instance)
def test_default_output_transform_default_type_alias():
"""Test the default_output_transform method of OrchestrationBase."""
orchestration = MockOrchestration(members=[MockAgent(), MockAgent()])
orchestration._set_types()
chat_message_content = ChatMessageContent(role=AuthorRole.USER, content="Test message")
transformed_output = orchestration._default_output_transform(chat_message_content)
assert isinstance(transformed_output, ChatMessageContent)
chat_message_content_list = [
ChatMessageContent(role=AuthorRole.USER, content="Test message 1"),
ChatMessageContent(role=AuthorRole.USER, content="Test message 2"),
]
transformed_output_list = orchestration._default_output_transform(chat_message_content_list)
assert isinstance(transformed_output_list, list) and all(
isinstance(item, ChatMessageContent) for item in transformed_output_list
)
with pytest.raises(TypeError, match="Invalid output message type"):
orchestration._default_output_transform("Invalid type")
def test_default_output_transform_custom_type():
"""Test the default_output_transform method of OrchestrationBase with a custom type."""
@dataclass
class CustomType:
value: str
number: int
orchestration_a = MockOrchestration[TIn, CustomType](members=[MockAgent(), MockAgent()])
orchestration_a._set_types()
custom_type_instance = CustomType(value="Test message", number=42)
chat_message_content = ChatMessageContent(role=AuthorRole.USER, content=json.dumps(custom_type_instance.__dict__))
transformed_output_a = orchestration_a._default_output_transform(chat_message_content)
assert isinstance(transformed_output_a, CustomType)
assert transformed_output_a == custom_type_instance
class CustomModel(KernelBaseModel):
value: str
number: int
orchestration_b = MockOrchestration[TIn, CustomModel](members=[MockAgent(), MockAgent()])
orchestration_b._set_types()
custom_model_instance = CustomModel(value="Test message", number=42)
chat_message_content = ChatMessageContent(role=AuthorRole.USER, content=custom_model_instance.model_dump_json())
transformed_output_b = orchestration_b._default_output_transform(chat_message_content)
assert isinstance(transformed_output_b, CustomModel)
assert transformed_output_b == custom_model_instance
def test_default_output_transform_custom_type_error():
"""Test the default_output_transform method of OrchestrationBase with an incorrect type."""
@dataclass
class CustomType:
value: str
number: int
class CustomModel(KernelBaseModel):
value: str
number: int
orchestration = MockOrchestration[TIn, CustomModel](members=[MockAgent(), MockAgent()])
orchestration._set_types()
with pytest.raises(TypeError, match="Unable to transform output message"):
custom_type_instance = CustomType(value="Test message", number=42)
orchestration._default_output_transform(custom_type_instance)
async def test_invoke_with_timeout_error():
"""Test the invoke method of the MockOrchestration with a timeout error."""
agent_a = MockAgent()
agent_b = MockAgent()
orchestration = MockOrchestration(members=[agent_a, agent_b])
# The orchestration_result will never be set by the MockOrchestration
orchestration_result = await orchestration.invoke(
task="test_message",
runtime=MockRuntime(),
)
with pytest.raises(asyncio.TimeoutError):
await orchestration_result.get(timeout=0.1)
async def test_invoke_cancel_before_completion():
"""Test the invoke method of the MockOrchestration with cancellation before completion."""
agent_a = MockAgent()
agent_b = MockAgent()
orchestration = MockOrchestration(members=[agent_a, agent_b])
# The orchestration_result will never be set by the MockOrchestration
orchestration_result = await orchestration.invoke(
task="test_message",
runtime=MockRuntime(),
)
# Cancel the orchestration before completion
orchestration_result.cancel()
with pytest.raises(RuntimeError, match="The invocation was canceled before it could complete."):
await orchestration_result.get()
async def test_invoke_with_double_cancel():
"""Test the invoke method of the MockOrchestration with double cancel."""
agent_a = MockAgent()
agent_b = MockAgent()
orchestration = MockOrchestration(members=[agent_a, agent_b])
# The orchestration_result will never be set by the MockOrchestration
orchestration_result = await orchestration.invoke(
task="test_message",
runtime=MockRuntime(),
)
orchestration_result.cancel()
# Cancelling again should raise an error
with pytest.raises(RuntimeError, match="The invocation has already been canceled."):
orchestration_result.cancel()
@@ -0,0 +1,173 @@
# Copyright (c) Microsoft. All rights reserved.
import sys
from collections.abc import Callable
from typing import Any, Literal
from unittest.mock import AsyncMock, patch
import pytest
from pydantic import BaseModel
from semantic_kernel.agents.orchestration.tools import structured_outputs_transform
from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.kernel_pydantic import KernelBaseModel
if sys.version_info >= (3, 12):
from typing import override # pragma: no cover
else:
from typing_extensions import override # pragma: no cover
class MockPromptExecutionSettings(PromptExecutionSettings):
"""A mock prompt execution settings class for testing purposes."""
response_format: (
dict[Literal["type"], Literal["text", "json_object"]] | dict[str, Any] | type[BaseModel] | type | None
) = None
class MockChatCompletionService(ChatCompletionClientBase):
"""A mock chat completion service for testing purposes."""
@override
def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]:
"""Get the request settings class."""
return MockPromptExecutionSettings
class MockModel(KernelBaseModel):
name: str
age: int
def test_structured_outputs_transform():
"""Test the structured_outputs_transform function."""
service = MockChatCompletionService(ai_model_id="test_model")
prompt_execution_settings = PromptExecutionSettings()
transform = structured_outputs_transform(
target_structure=MockModel,
service=service,
prompt_execution_settings=prompt_execution_settings,
)
assert isinstance(transform, Callable)
def test_structured_outputs_transform_original_settings_not_changed():
"""Test the structured_outputs_transform function with original settings not changed."""
service = MockChatCompletionService(ai_model_id="test_model")
prompt_execution_settings = PromptExecutionSettings(
extension_data={"test_key": "test_value"},
)
_ = structured_outputs_transform(
target_structure=MockModel,
service=service,
prompt_execution_settings=prompt_execution_settings,
)
assert not hasattr(prompt_execution_settings, "response_format")
assert prompt_execution_settings.extension_data["test_key"] == "test_value"
def test_structured_outputs_transform_unsupported_service():
"""Test the structured_outputs_transform function with unsupported service."""
with (
patch.object(
MockChatCompletionService, "get_prompt_execution_settings_class"
) as mock_get_prompt_execution_settings_class,
pytest.raises(ValueError),
):
mock_get_prompt_execution_settings_class.return_value = PromptExecutionSettings
service = MockChatCompletionService(ai_model_id="test_model")
prompt_execution_settings = PromptExecutionSettings()
_ = structured_outputs_transform(MockModel, service, prompt_execution_settings)
async def test_structured_outputs_transform_invoke():
"""Test the structured_outputs_transform function and invoke the transform."""
mock_model = MockModel(name="John Doe", age=30)
with (
patch.object(
MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock
) as mock_get_chat_message_content,
):
mock_get_chat_message_content.return_value = ChatMessageContent(
role="assistant", content=mock_model.model_dump_json()
)
service = MockChatCompletionService(ai_model_id="test_model")
prompt_execution_settings = PromptExecutionSettings()
transform = structured_outputs_transform(
target_structure=MockModel,
service=service,
prompt_execution_settings=prompt_execution_settings,
)
result = await transform(ChatMessageContent(role="user", content="name is John Doe and age is 30"))
assert isinstance(result, MockModel)
assert result == mock_model
mock_get_chat_message_content.assert_called_once()
assert len(mock_get_chat_message_content.call_args[0][0].messages) == 2
async def test_structured_outputs_transform_invoke_with_messages():
"""Test the structured_outputs_transform function and invoke the transform with messages."""
mock_model = MockModel(name="John Doe", age=30)
with (
patch.object(
MockChatCompletionService, "get_chat_message_content", new_callable=AsyncMock
) as mock_get_chat_message_content,
):
mock_get_chat_message_content.return_value = ChatMessageContent(
role="assistant", content=mock_model.model_dump_json()
)
service = MockChatCompletionService(ai_model_id="test_model")
prompt_execution_settings = PromptExecutionSettings()
transform = structured_outputs_transform(
target_structure=MockModel,
service=service,
prompt_execution_settings=prompt_execution_settings,
)
result = await transform([
ChatMessageContent(role="user", content="name is John Doe"),
ChatMessageContent(role="user", content="age is 30"),
])
assert isinstance(result, MockModel)
assert result == mock_model
mock_get_chat_message_content.assert_called_once()
assert len(mock_get_chat_message_content.call_args[0][0].messages) == 3
async def test_structured_outputs_transform_invoke_unsupported_type():
"""Test the structured_outputs_transform function and invoke the transform with messages of unsupported type."""
service = MockChatCompletionService(ai_model_id="test_model")
prompt_execution_settings = PromptExecutionSettings()
transform = structured_outputs_transform(
target_structure=MockModel,
service=service,
prompt_execution_settings=prompt_execution_settings,
)
with pytest.raises(ValueError):
_ = await transform("unsupported type")
@@ -0,0 +1,202 @@
# Copyright (c) Microsoft. All rights reserved.
import asyncio
import sys
from unittest.mock import patch
import pytest
from semantic_kernel.agents.orchestration.orchestration_base import DefaultTypeAlias, OrchestrationResult
from semantic_kernel.agents.orchestration.sequential import SequentialOrchestration
from semantic_kernel.agents.runtime.in_process.in_process_runtime import InProcessRuntime
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from tests.unit.agents.orchestration.conftest import MockAgent, MockAgentWithException, MockRuntime
async def test_prepare():
"""Test the prepare method of the SequentialOrchestration."""
agent_a = MockAgent()
agent_b = MockAgent()
runtime = MockRuntime()
package_path = "semantic_kernel.agents.orchestration.sequential"
with (
patch(f"{package_path}.SequentialOrchestration._start"),
patch(f"{package_path}.SequentialAgentActor.register") as mock_agent_actor_register,
patch(f"{package_path}.CollectionActor.register") as mock_collection_actor_register,
patch.object(runtime, "add_subscription") as mock_add_subscription,
):
orchestration = SequentialOrchestration(members=[agent_a, agent_b])
await orchestration.invoke(task="test_message", runtime=runtime)
assert mock_agent_actor_register.call_count == 2
assert mock_collection_actor_register.call_count == 1
assert mock_add_subscription.call_count == 0
async def test_invoke():
"""Test the invoke method of the SequentialOrchestration."""
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = SequentialOrchestration(members=[agent_a, agent_b])
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
result = await orchestration_result.get(1.0)
assert isinstance(orchestration_result, OrchestrationResult)
assert isinstance(result, ChatMessageContent)
finally:
await runtime.stop_when_idle()
async def test_invoke_with_response_callback():
"""Test the invoke method of the SequentialOrchestration with a response callback."""
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
responses: list[DefaultTypeAlias] = []
try:
orchestration = SequentialOrchestration(
members=[agent_a, agent_b],
agent_response_callback=lambda x: responses.append(x),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
assert len(responses) == 2
assert all(isinstance(item, ChatMessageContent) for item in responses)
assert all(item.content == "mock_response" for item in responses)
finally:
await runtime.stop_when_idle()
async def test_invoke_with_streaming_response_callback():
"""Test the invoke method of the SequentialOrchestration with a streaming response callback."""
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
responses: dict[str, list[StreamingChatMessageContent]] = {}
try:
orchestration = SequentialOrchestration(
members=[agent_a, agent_b],
streaming_agent_response_callback=lambda x, _: responses.setdefault(x.name, []).append(x),
)
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
await orchestration_result.get(1.0)
assert len(responses[agent_a.name]) == 2
assert len(responses[agent_b.name]) == 2
assert all(isinstance(item, StreamingChatMessageContent) for item in responses[agent_a.name])
assert all(isinstance(item, StreamingChatMessageContent) for item in responses[agent_b.name])
agent_a_response = sum(responses[agent_a.name][1:], responses[agent_a.name][0])
assert agent_a_response.content == "mock_response"
agent_b_response = sum(responses[agent_b.name][1:], responses[agent_b.name][0])
assert agent_b_response.content == "mock_response"
finally:
await runtime.stop_when_idle()
@pytest.mark.skipif(
sys.version_info < (3, 11),
reason="Python 3.10 doesn't bound the original function provided to the wraps argument of the patch object.",
)
async def test_invoke_cancel_before_completion():
"""Test the invoke method of the SequentialOrchestration with cancellation before completion."""
with (
patch.object(MockAgent, "invoke_stream", wraps=MockAgent.invoke_stream, autospec=True) as mock_invoke_stream,
):
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = SequentialOrchestration(members=[agent_a, agent_b])
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
# Cancel while the first agent is processing
await asyncio.sleep(0.05)
orchestration_result.cancel()
finally:
await runtime.stop_when_idle()
assert mock_invoke_stream.call_count == 1
async def test_invoke_cancel_after_completion():
"""Test the invoke method of the SequentialOrchestration with cancellation after completion."""
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = SequentialOrchestration(members=[agent_a, agent_b])
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
# Wait for the orchestration to complete
await orchestration_result.get(1.0)
with pytest.raises(RuntimeError, match="The invocation has already been completed."):
orchestration_result.cancel()
finally:
await runtime.stop_when_idle()
async def test_invoke_with_double_get_result():
"""Test the invoke method of the SequentialOrchestration with double get result."""
agent_a = MockAgent()
agent_b = MockAgent()
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = SequentialOrchestration(members=[agent_a, agent_b])
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
# Get result before completion
with pytest.raises(asyncio.TimeoutError):
await orchestration_result.get(0.1)
# The invocation should still be in progress and getting the result again should not raise an error
result = await orchestration_result.get(1.0)
assert isinstance(result, ChatMessageContent)
assert result.content == "mock_response"
finally:
await runtime.stop_when_idle()
async def test_invoke_with_agent_raising_exception():
"""Test the invoke method of the SequentialOrchestration with an agent raising an exception."""
agent_a = MockAgent()
agent_b = MockAgentWithException()
runtime = InProcessRuntime()
runtime.start()
try:
orchestration = SequentialOrchestration(members=[agent_a, agent_b])
orchestration_result = await orchestration.invoke(task="test_message", runtime=runtime)
with pytest.raises(RuntimeError, match="Mock agent exception"):
await orchestration_result.get(1.0)
assert orchestration_result.exception is not None
finally:
await runtime.stop_when_idle()
@@ -0,0 +1,142 @@
# Copyright (c) Microsoft. All rights reserved.
from dataclasses import dataclass
import pytest
from pydantic import BaseModel
from semantic_kernel.agents.runtime.core.serialization import (
JSON_DATA_CONTENT_TYPE,
DataclassJsonMessageSerializer,
MessageSerializer,
SerializationRegistry,
try_get_known_serializers_for_type,
)
class PydanticMessage(BaseModel):
message: str
class NestingPydanticMessage(BaseModel):
message: str
nested: PydanticMessage
@dataclass
class DataclassMessage:
message: str
@dataclass
class NestingDataclassMessage:
message: str
nested: DataclassMessage
@dataclass
class NestingPydanticDataclassMessage:
message: str
nested: PydanticMessage
def test_pydantic() -> None:
serde = SerializationRegistry()
serde.add_serializer(try_get_known_serializers_for_type(PydanticMessage))
message = PydanticMessage(message="hello")
name = serde.type_name(message)
json = serde.serialize(message, type_name=name, data_content_type=JSON_DATA_CONTENT_TYPE)
assert name == "PydanticMessage"
assert json == b'{"message":"hello"}'
deserialized = serde.deserialize(json, type_name=name, data_content_type=JSON_DATA_CONTENT_TYPE)
assert deserialized == message
def test_nested_pydantic() -> None:
serde = SerializationRegistry()
serde.add_serializer(try_get_known_serializers_for_type(NestingPydanticMessage))
message = NestingPydanticMessage(message="hello", nested=PydanticMessage(message="world"))
name = serde.type_name(message)
json = serde.serialize(message, type_name=name, data_content_type=JSON_DATA_CONTENT_TYPE)
assert json == b'{"message":"hello","nested":{"message":"world"}}'
deserialized = serde.deserialize(json, type_name=name, data_content_type=JSON_DATA_CONTENT_TYPE)
assert deserialized == message
def test_dataclass() -> None:
serde = SerializationRegistry()
serde.add_serializer(try_get_known_serializers_for_type(DataclassMessage))
message = DataclassMessage(message="hello")
name = serde.type_name(message)
json = serde.serialize(message, type_name=name, data_content_type=JSON_DATA_CONTENT_TYPE)
assert json == b'{"message": "hello"}'
deserialized = serde.deserialize(json, type_name=name, data_content_type=JSON_DATA_CONTENT_TYPE)
assert deserialized == message
def test_nesting_dataclass_dataclass() -> None:
serde = SerializationRegistry()
with pytest.raises(ValueError):
serde.add_serializer(try_get_known_serializers_for_type(NestingDataclassMessage))
@dataclass
class DataclassNestedUnionSyntaxOldMessage:
message: str | int
@dataclass
class DataclassNestedUnionSyntaxNewMessage:
message: str | int
@pytest.mark.parametrize("cls", [DataclassNestedUnionSyntaxOldMessage, DataclassNestedUnionSyntaxNewMessage])
def test_nesting_union_old_syntax_dataclass(
cls: type[DataclassNestedUnionSyntaxOldMessage | DataclassNestedUnionSyntaxNewMessage],
) -> None:
with pytest.raises(ValueError):
_serializer = DataclassJsonMessageSerializer(cls)
def test_nesting_dataclass_pydantic() -> None:
serde = SerializationRegistry()
with pytest.raises(ValueError):
serde.add_serializer(try_get_known_serializers_for_type(NestingPydanticDataclassMessage))
def test_invalid_type() -> None:
serde = SerializationRegistry()
try:
serde.add_serializer(try_get_known_serializers_for_type(str))
except ValueError as e:
assert str(e) == "Unsupported type <class 'str'>"
def test_custom_type() -> None:
serde = SerializationRegistry()
class CustomStringTypeSerializer(MessageSerializer[str]):
@property
def data_content_type(self) -> str:
return "str"
@property
def type_name(self) -> str:
return "custom_str"
def deserialize(self, payload: bytes) -> str:
message = payload.decode("utf-8")
return message[1:-1]
def serialize(self, message: str) -> bytes:
return f'"{message}"'.encode()
serde.add_serializer(CustomStringTypeSerializer())
message = "hello"
json = serde.serialize(message, type_name="custom_str", data_content_type="str")
assert json == b'"hello"'
deserialized = serde.deserialize(json, type_name="custom_str", data_content_type="str")
assert deserialized == message
@@ -0,0 +1,397 @@
# Copyright (c) Microsoft. All rights reserved.
import logging
from asyncio import Event
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any
import pytest
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExporter, SpanExportResult
from semantic_kernel.agents.runtime.core import CoreAgentId
from semantic_kernel.agents.runtime.core.agent_type import CoreAgentType
from semantic_kernel.agents.runtime.core.base_agent import BaseAgent
from semantic_kernel.agents.runtime.core.message_context import MessageContext
from semantic_kernel.agents.runtime.core.routed_agent import RoutedAgent, event, message_handler
from semantic_kernel.agents.runtime.core.serialization import try_get_known_serializers_for_type
from semantic_kernel.agents.runtime.core.topic import TopicId
from semantic_kernel.agents.runtime.in_process.agent_instantiation_context import AgentInstantiationContext
from semantic_kernel.agents.runtime.in_process.default_subscription import default_subscription, type_subscription
from semantic_kernel.agents.runtime.in_process.default_topic import DefaultTopicId
from semantic_kernel.agents.runtime.in_process.in_process_runtime import InProcessRuntime
from semantic_kernel.agents.runtime.in_process.type_subscription import TypeSubscription
@dataclass
class MessageType: ...
@dataclass
class CascadingMessageType:
round: int
@dataclass
class ContentMessage:
content: str
class LoopbackAgent(RoutedAgent):
def __init__(self) -> None:
super().__init__("A loop back agent.")
self.num_calls = 0
self.received_messages: list[Any] = []
self.event = Event()
@message_handler
async def on_new_message(
self, message: MessageType | ContentMessage, ctx: MessageContext
) -> MessageType | ContentMessage:
self.num_calls += 1
self.received_messages.append(message)
self.event.set()
return message
@default_subscription
class LoopbackAgentWithDefaultSubscription(LoopbackAgent): ...
@default_subscription
class CascadingAgent(RoutedAgent):
def __init__(self, max_rounds: int) -> None:
super().__init__("A cascading agent.")
self.num_calls = 0
self.max_rounds = max_rounds
@message_handler
async def on_new_message(self, message: CascadingMessageType, ctx: MessageContext) -> None:
self.num_calls += 1
if message.round == self.max_rounds:
return
await self.publish_message(CascadingMessageType(round=message.round + 1), topic_id=DefaultTopicId())
class NoopAgent(BaseAgent):
def __init__(self) -> None:
super().__init__("A no op agent")
async def on_message_impl(self, message: Any, ctx: MessageContext) -> Any:
raise NotImplementedError
class MyTestExporter(SpanExporter):
def __init__(self) -> None:
self.exported_spans: list[ReadableSpan] = []
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
self.exported_spans.extend(spans)
return SpanExportResult.SUCCESS
def shutdown(self) -> None:
pass
def clear(self) -> None:
"""Clears the list of exported spans."""
self.exported_spans.clear()
def get_exported_spans(self) -> list[ReadableSpan]:
"""Returns the list of exported spans."""
return self.exported_spans
def get_test_tracer_provider(exporter: MyTestExporter) -> TracerProvider:
tracer_provider = TracerProvider()
tracer_provider.add_span_processor(SimpleSpanProcessor(exporter))
return tracer_provider
test_exporter = MyTestExporter()
class FakeAgent(BaseAgent):
def __init__(self) -> None:
super().__init__("A fake agent")
async def on_message_impl(self, message: Any, ctx: MessageContext) -> Any:
raise NotImplementedError
@pytest.fixture
def tracer_provider() -> TracerProvider:
test_exporter.clear()
return get_test_tracer_provider(test_exporter)
@pytest.mark.asyncio
async def test_agent_type_register_factory() -> None:
runtime = InProcessRuntime()
def agent_factory() -> NoopAgent:
id = AgentInstantiationContext.current_agent_id()
assert id == CoreAgentId("name1", "default")
agent = NoopAgent()
assert agent.id == id
return agent
await runtime.register_factory(type=CoreAgentType("name1"), agent_factory=agent_factory, expected_class=NoopAgent)
with pytest.raises(ValueError):
# This should fail because the expected class does not match the actual class.
await runtime.register_factory(
type=CoreAgentType("name1"),
agent_factory=agent_factory, # type: ignore
expected_class=FakeAgent,
)
# Without expected_class, no error.
await runtime.register_factory(type=CoreAgentType("name2"), agent_factory=agent_factory)
@pytest.mark.asyncio
async def test_agent_type_must_be_unique() -> None:
runtime = InProcessRuntime()
def agent_factory() -> NoopAgent:
id = AgentInstantiationContext.current_agent_id()
assert id == CoreAgentId("name1", "default")
agent = NoopAgent()
assert agent.id == id
return agent
await NoopAgent.register(runtime, "name1", agent_factory)
with pytest.raises(ValueError):
await runtime.register_factory(
type=CoreAgentType("name1"), agent_factory=agent_factory, expected_class=NoopAgent
)
await runtime.register_factory(type=CoreAgentType("name2"), agent_factory=agent_factory, expected_class=NoopAgent)
@pytest.mark.asyncio
async def test_register_receives_publish(tracer_provider: TracerProvider) -> None:
runtime = InProcessRuntime(tracer_provider=tracer_provider)
runtime.add_message_serializer(try_get_known_serializers_for_type(MessageType))
await runtime.register_factory(
type=CoreAgentType("name"), agent_factory=lambda: LoopbackAgent(), expected_class=LoopbackAgent
)
await runtime.add_subscription(TypeSubscription("default", "name"))
runtime.start()
await runtime.publish_message(MessageType(), topic_id=TopicId("default", "default"))
await runtime.stop_when_idle()
# Agent in default namespace should have received the message
long_running_agent = await runtime.try_get_underlying_agent_instance(
CoreAgentId("name", "default"),
type=LoopbackAgent,
)
assert long_running_agent.num_calls == 1
# Agent in other namespace should not have received the message
other_long_running_agent: LoopbackAgent = await runtime.try_get_underlying_agent_instance(
CoreAgentId("name", key="other"), type=LoopbackAgent
)
assert other_long_running_agent.num_calls == 0
await runtime.close()
@pytest.mark.asyncio
async def test_register_receives_publish_with_construction(caplog: pytest.LogCaptureFixture) -> None:
runtime = InProcessRuntime()
runtime.add_message_serializer(try_get_known_serializers_for_type(MessageType))
async def agent_factory() -> LoopbackAgent:
raise ValueError("test")
await runtime.register_factory(
type=CoreAgentType("name"), agent_factory=agent_factory, expected_class=LoopbackAgent
)
await runtime.add_subscription(TypeSubscription("default", "name"))
with caplog.at_level(logging.ERROR):
runtime.start()
await runtime.publish_message(MessageType(), topic_id=TopicId("default", "default"))
await runtime.stop_when_idle()
# Check if logger has the exception.
assert any("Error constructing agent" in e.message for e in caplog.records)
await runtime.close()
@pytest.mark.asyncio
async def test_register_receives_publish_cascade() -> None:
num_agents = 5
num_initial_messages = 5
max_rounds = 5
total_num_calls_expected = 0
for i in range(0, max_rounds):
total_num_calls_expected += num_initial_messages * ((num_agents - 1) ** i)
runtime = InProcessRuntime()
# Register agents
for i in range(num_agents):
await CascadingAgent.register(runtime, f"name{i}", lambda: CascadingAgent(max_rounds))
runtime.start()
# Publish messages
for _ in range(num_initial_messages):
await runtime.publish_message(CascadingMessageType(round=1), DefaultTopicId())
# Process until idle.
await runtime.stop_when_idle()
# Check that each agent received the correct number of messages.
for i in range(num_agents):
agent = await runtime.try_get_underlying_agent_instance(CoreAgentId(f"name{i}", "default"), CascadingAgent)
assert agent.num_calls == total_num_calls_expected
await runtime.close()
@pytest.mark.asyncio
async def test_register_factory_explicit_name() -> None:
runtime = InProcessRuntime()
await LoopbackAgent.register(runtime, "name", LoopbackAgent)
await runtime.add_subscription(TypeSubscription("default", "name"))
runtime.start()
agent_id = CoreAgentId("name", key="default")
topic_id = TopicId("default", "default")
await runtime.publish_message(MessageType(), topic_id=topic_id)
await runtime.stop_when_idle()
# Agent in default namespace should have received the message
long_running_agent = await runtime.try_get_underlying_agent_instance(agent_id, type=LoopbackAgent)
assert long_running_agent.num_calls == 1
# Agent in other namespace should not have received the message
other_long_running_agent: LoopbackAgent = await runtime.try_get_underlying_agent_instance(
CoreAgentId("name", key="other"), type=LoopbackAgent
)
assert other_long_running_agent.num_calls == 0
await runtime.close()
@pytest.mark.asyncio
async def test_default_subscription() -> None:
runtime = InProcessRuntime()
runtime.start()
await LoopbackAgentWithDefaultSubscription.register(runtime, "name", LoopbackAgentWithDefaultSubscription)
agent_id = CoreAgentId("name", key="default")
await runtime.publish_message(MessageType(), topic_id=DefaultTopicId())
await runtime.stop_when_idle()
long_running_agent = await runtime.try_get_underlying_agent_instance(
agent_id, type=LoopbackAgentWithDefaultSubscription
)
assert long_running_agent.num_calls == 1
other_long_running_agent = await runtime.try_get_underlying_agent_instance(
CoreAgentId("name", key="other"), type=LoopbackAgentWithDefaultSubscription
)
assert other_long_running_agent.num_calls == 0
await runtime.close()
@pytest.mark.asyncio
async def test_type_subscription() -> None:
runtime = InProcessRuntime()
runtime.start()
@type_subscription(topic_type="Other")
class LoopbackAgentWithSubscription(LoopbackAgent): ...
await LoopbackAgentWithSubscription.register(runtime, "name", LoopbackAgentWithSubscription)
agent_id = CoreAgentId("name", key="default")
await runtime.publish_message(MessageType(), topic_id=TopicId("Other", "default"))
await runtime.stop_when_idle()
long_running_agent = await runtime.try_get_underlying_agent_instance(agent_id, type=LoopbackAgentWithSubscription)
assert long_running_agent.num_calls == 1
other_long_running_agent = await runtime.try_get_underlying_agent_instance(
CoreAgentId("name", key="other"), type=LoopbackAgentWithSubscription
)
assert other_long_running_agent.num_calls == 0
await runtime.close()
@pytest.mark.asyncio
async def test_default_subscription_publish_to_other_source() -> None:
runtime = InProcessRuntime()
runtime.start()
await LoopbackAgentWithDefaultSubscription.register(runtime, "name", LoopbackAgentWithDefaultSubscription)
agent_id = CoreAgentId("name", key="default")
await runtime.publish_message(MessageType(), topic_id=DefaultTopicId(source="other"))
await runtime.stop_when_idle()
long_running_agent = await runtime.try_get_underlying_agent_instance(
agent_id, type=LoopbackAgentWithDefaultSubscription
)
assert long_running_agent.num_calls == 0
other_long_running_agent = await runtime.try_get_underlying_agent_instance(
CoreAgentId("name", key="other"), type=LoopbackAgentWithDefaultSubscription
)
assert other_long_running_agent.num_calls == 1
await runtime.close()
@default_subscription
class FailingAgent(RoutedAgent):
def __init__(self) -> None:
super().__init__("A failing agent.")
@event
async def on_new_message_event(self, message: MessageType, ctx: MessageContext) -> None:
raise ValueError("Test exception")
@pytest.mark.asyncio
async def test_event_handler_exception_propagates() -> None:
runtime = InProcessRuntime(ignore_unhandled_exceptions=False)
await FailingAgent.register(runtime, "name", FailingAgent)
with pytest.raises(ValueError, match="Test exception"):
runtime.start()
await runtime.publish_message(MessageType(), topic_id=DefaultTopicId())
await runtime.stop_when_idle()
await runtime.close()
@pytest.mark.asyncio
async def test_event_handler_exception_multi_message() -> None:
runtime = InProcessRuntime(ignore_unhandled_exceptions=False)
await FailingAgent.register(runtime, "name", FailingAgent)
with pytest.raises(ValueError, match="Test exception"):
runtime.start()
await runtime.publish_message(MessageType(), topic_id=DefaultTopicId())
await runtime.publish_message(MessageType(), topic_id=DefaultTopicId())
await runtime.publish_message(MessageType(), topic_id=DefaultTopicId())
await runtime.stop_when_idle()
await runtime.close()
+419
View File
@@ -0,0 +1,419 @@
# Copyright (c) Microsoft. All rights reserved.
import sys
import uuid
from typing import ClassVar
from unittest.mock import AsyncMock
import pytest
from semantic_kernel.agents.agent import AGENT_TYPE_REGISTRY, AgentRegistry, DeclarativeSpecMixin, register_agent_type
from semantic_kernel.exceptions.agent_exceptions import AgentInitializationException
from semantic_kernel.kernel import Kernel
if sys.version_info >= (3, 12):
from typing import override # pragma: no cover
else:
from typing_extensions import override # pragma: no cover
from semantic_kernel.agents import Agent
from semantic_kernel.agents.channels.agent_channel import AgentChannel
from semantic_kernel.functions.kernel_arguments import KernelArguments
class MockChatHistory:
"""Minimal mock for ChatHistory to hold messages."""
def __init__(self, messages=None):
self.messages = messages if messages is not None else []
class MockChannel(AgentChannel):
"""Mock channel for testing get_channel_keys and create_channel."""
class MockAgent(Agent):
"""A mock agent for testing purposes."""
channel_type: ClassVar[type[AgentChannel]] = MockChannel
def __init__(self, name: str = "Test-Agent", description: str = "A test agent", id: str = None):
args = {
"name": name,
"description": description,
}
if id is not None:
args["id"] = id
super().__init__(**args)
async def create_channel(self) -> AgentChannel:
return AsyncMock(spec=AgentChannel)
@override
async def get_response(self, *args, **kwargs):
raise NotImplementedError
@override
async def invoke(self, *args, **kwargs):
raise NotImplementedError
@override
async def invoke_stream(self, *args, **kwargs):
raise NotImplementedError
class MockAgentWithoutChannelType(MockAgent):
channel_type = None
async def test_agent_initialization():
name = "TestAgent"
description = "A test agent"
id_value = str(uuid.uuid4())
agent = MockAgent(name=name, description=description, id=id_value)
assert agent.name == name
assert agent.description == description
assert agent.id == id_value
async def test_agent_default_id():
agent = MockAgent()
assert agent.id is not None
assert isinstance(uuid.UUID(agent.id), uuid.UUID)
def test_get_channel_keys():
agent = MockAgent()
keys = agent.get_channel_keys()
assert len(list(keys)) == 1, "Should return a single key"
async def test_create_channel():
agent = MockAgent()
channel = await agent.create_channel()
assert isinstance(channel, AgentChannel)
async def test_agent_equality():
id_value = str(uuid.uuid4())
agent1 = MockAgent(name="TestAgent", description="A test agent", id=id_value)
agent2 = MockAgent(name="TestAgent", description="A test agent", id=id_value)
assert agent1 == agent2
agent3 = MockAgent(name="TestAgent", description="A different description", id=id_value)
assert agent1 != agent3
agent4 = MockAgent(name="AnotherAgent", description="A test agent", id=id_value)
assert agent1 != agent4
async def test_agent_equality_different_type():
agent = MockAgent(name="TestAgent", description="A test agent", id=str(uuid.uuid4()))
non_agent = "Not an agent"
assert agent != non_agent
async def test_agent_hash():
id_value = str(uuid.uuid4())
agent1 = MockAgent(name="TestAgent", description="A test agent", id=id_value)
agent2 = MockAgent(name="TestAgent", description="A test agent", id=id_value)
assert hash(agent1) == hash(agent2)
agent3 = MockAgent(name="TestAgent", description="A different description", id=id_value)
assert hash(agent1) != hash(agent3)
def test_get_channel_keys_no_channel_type():
agent = MockAgentWithoutChannelType()
with pytest.raises(NotImplementedError):
list(agent.get_channel_keys())
def test_merge_arguments_both_none():
agent = MockAgent()
merged = agent._merge_arguments(None)
assert isinstance(merged, KernelArguments)
assert len(merged) == 0, "If both arguments are None, should return an empty KernelArguments object"
def test_merge_arguments_agent_none_override_not_none():
agent = MockAgent()
override = KernelArguments(settings={"key": "override"}, param1="val1")
merged = agent._merge_arguments(override)
assert merged is override, "If agent.arguments is None, just return override_args"
def test_merge_arguments_override_none_agent_not_none():
agent = MockAgent()
agent.arguments = KernelArguments(settings={"key": "base"}, param1="baseVal")
merged = agent._merge_arguments(None)
assert merged is agent.arguments, "If override_args is None, should return the agent's arguments"
def test_merge_arguments_both_not_none():
agent = MockAgent()
agent.arguments = KernelArguments(settings={"key1": "val1", "common": "base"}, param1="baseVal")
override = KernelArguments(settings={"key2": "override_val", "common": "override"}, param2="override_param")
merged = agent._merge_arguments(override)
assert merged.execution_settings["key1"] == "val1", "Should retain original setting from agent"
assert merged.execution_settings["key2"] == "override_val", "Should include new setting from override"
assert merged.execution_settings["common"] == "override", "Override should take precedence"
assert merged["param1"] == "baseVal", "Should retain base param from agent"
assert merged["param2"] == "override_param", "Should include param from override"
# region Declarative Spec tests
class DummyPlugin:
def __init__(self):
self.functions = {"dummy_function": lambda: "result"}
def get(self, name):
return self.functions.get(name)
class DummyAgentSettings:
azure_ai_search_connection_id = "test-conn-id"
azure_ai_search_index_name = "test-index"
class DummyKernel:
def __init__(self):
self.plugins = {}
def add_plugin(self, plugin):
name = plugin.__class__.__name__
self.plugins[name] = plugin
async def test_resolve_placeholders_with_short_and_long_keys():
class DummyDeclarativeSpec:
@classmethod
def resolve_placeholders(cls, yaml_str, settings=None, extras=None):
import re
pattern = re.compile(r"\$\{([^}]+)\}")
field_mapping = {
"AzureAISearchConnectionId": "conn-123",
"AzureAI:AzureAISearchConnectionId": "conn-123-override",
}
if extras:
field_mapping.update(extras)
def replacer(match):
full_key = match.group(1)
section, _, key = full_key.partition(":")
if section != "AzureAI":
return match.group(0)
return str(field_mapping.get(key) or field_mapping.get(full_key) or match.group(0))
return pattern.sub(replacer, yaml_str)
spec = "connection: ${AzureAI:AzureAISearchConnectionId}"
resolved = DummyDeclarativeSpec.resolve_placeholders(spec)
assert resolved == "connection: conn-123"
async def test_validate_tools_succeeds_with_valid_plugin():
class DummyDeclarativeSpec:
@classmethod
def _validate_tools(cls, tools_list, kernel):
for tool in tools_list:
tool_id = tool.get("id")
if not tool_id or tool.get("type") != "function":
continue
plugin_name, function_name = tool_id.split(".")
plugin = kernel.plugins.get(plugin_name)
if not plugin:
raise ValueError(f"Plugin {plugin_name} missing")
if function_name not in plugin.functions:
raise ValueError(f"Function {function_name} missing in plugin {plugin_name}")
kernel = DummyKernel()
plugin = DummyPlugin()
kernel.add_plugin(plugin)
DummyDeclarativeSpec._validate_tools([{"id": "DummyPlugin.dummy_function", "type": "function"}], kernel)
async def test_validate_tools_raises_on_missing_plugin():
class DummyDeclarativeSpec:
@classmethod
def _validate_tools(cls, tools_list, kernel):
for tool in tools_list:
tool_id = tool.get("id")
if not tool_id or tool.get("type") != "function":
continue
plugin_name, function_name = tool_id.split(".")
plugin = kernel.plugins.get(plugin_name)
if not plugin:
raise ValueError(f"Plugin {plugin_name} missing")
if function_name not in plugin.functions:
raise ValueError(f"Function {function_name} missing in plugin {plugin_name}")
kernel = DummyKernel()
with pytest.raises(ValueError, match="Plugin DummyPlugin missing"):
DummyDeclarativeSpec._validate_tools([{"id": "DummyPlugin.dummy_function", "type": "function"}], kernel)
def test_normalize_spec_fields_creates_kernel_and_extracts_fields():
data = {
"name": "TestAgent",
"description": "An agent",
"instructions": "Use this.",
"model": {"options": {"temperature": 0.7}},
}
fields, kernel = DeclarativeSpecMixin._normalize_spec_fields(data)
assert isinstance(kernel, Kernel)
assert fields["name"] == "TestAgent"
assert isinstance(fields["arguments"], KernelArguments)
def test_normalize_spec_fields_adds_plugins_to_kernel():
plugin = DummyPlugin()
data = {"name": "PluginAgent"}
_, kernel = DeclarativeSpecMixin._normalize_spec_fields(data, plugins=[plugin])
assert "DummyPlugin" in kernel.plugins
def test_normalize_spec_fields_parses_prompt_template_and_overwrites_instructions():
data = {"name": "T", "prompt_template": {"template": "new instructions", "template_format": "semantic-kernel"}}
fields, _ = DeclarativeSpecMixin._normalize_spec_fields(data)
assert fields["instructions"] == "new instructions"
def test_validate_tools_success(custom_plugin_class):
kernel = Kernel()
kernel.add_plugin(custom_plugin_class())
tools_list = [{"id": "CustomPlugin.getLightStatus", "type": "function"}]
DeclarativeSpecMixin._validate_tools(tools_list, kernel)
def test_validate_tools_fails_on_invalid_format():
kernel = Kernel()
with pytest.raises(AgentInitializationException, match="format"):
DeclarativeSpecMixin._validate_tools([{"id": "badformat", "type": "function"}], kernel)
def test_validate_tools_fails_on_missing_plugin():
kernel = Kernel()
with pytest.raises(AgentInitializationException, match="not found in kernel"):
DeclarativeSpecMixin._validate_tools([{"id": "MissingPlugin.foo", "type": "function"}], kernel)
def test_validate_tools_fails_on_missing_function():
plugin = DummyPlugin()
kernel = Kernel()
kernel.add_plugin(plugin)
with pytest.raises(AgentInitializationException, match="not found in plugin"):
DeclarativeSpecMixin._validate_tools([{"id": "DummyPlugin.bar", "type": "function"}], kernel)
@register_agent_type("test_agent")
class TestAgent(DeclarativeSpecMixin, Agent):
@classmethod
def resolve_placeholders(cls, yaml_str, settings=None, extras=None):
return yaml_str
@classmethod
async def _from_dict(cls, data, kernel, **kwargs):
return cls(
name=data.get("name"),
description=data.get("description"),
instructions=data.get("instructions"),
kernel=kernel,
)
async def get_response(self, messages, instructions_override=None):
return "Test response"
async def invoke(self, messages, **kwargs):
return "invoke result"
async def invoke_stream(self, messages, **kwargs):
yield "streamed result"
async def test_register_type_and_create_from_yaml_success():
yaml_str = """
type: test_agent
name: TestAgent
"""
agent = await AgentRegistry.create_from_yaml(yaml_str)
assert agent.__class__.__name__ == "TestAgent"
async def test_create_from_yaml_missing_type():
yaml_str = """
name: InvalidAgent
"""
with pytest.raises(AgentInitializationException, match="Missing 'type'"):
await AgentRegistry.create_from_yaml(yaml_str)
async def test_create_from_yaml_unregistered_type():
yaml_str = """
type: nonexistent_agent
"""
# Ensure unregistered
AGENT_TYPE_REGISTRY.pop("nonexistent_agent", None)
with pytest.raises(AgentInitializationException, match="not registered"):
await AgentRegistry.create_from_yaml(yaml_str)
async def test_create_from_dict_success(test_agent_cls):
data = {"type": "test_agent", "name": "FromDictAgent"}
agent: TestAgent = await AgentRegistry.create_from_dict(data)
assert agent.name == "FromDictAgent"
assert type(agent).__name__ == "TestAgent"
async def test_create_from_dict_missing_type():
data = {"name": "NoType"}
with pytest.raises(AgentInitializationException, match="Missing 'type'"):
await AgentRegistry.create_from_dict(data)
async def test_create_from_dict_type_not_supported():
AGENT_TYPE_REGISTRY.pop("unknown", None)
data = {"type": "unknown"}
with pytest.raises(AgentInitializationException, match="not supported"):
await AgentRegistry.create_from_dict(data)
async def test_create_from_file_reads_and_creates(tmp_path, test_agent_cls):
file_path = tmp_path / "spec.yaml"
file_path.write_text("type: test_agent\nname: FileAgent\n", encoding="utf-8")
agent: TestAgent = await AgentRegistry.create_from_file(str(file_path))
assert agent.name == "FileAgent"
assert type(agent).__name__ == "TestAgent"
async def test_create_from_file_raises_on_bad_path():
with pytest.raises(AgentInitializationException, match="Failed to read agent spec file"):
await AgentRegistry.create_from_file("/nonexistent/path/spec.yaml")
# endregion
@@ -0,0 +1,59 @@
# Copyright (c) Microsoft. All rights reserved.
from collections.abc import AsyncIterable
from unittest.mock import AsyncMock
from semantic_kernel.agents import Agent
from semantic_kernel.agents.channels.agent_channel import AgentChannel
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
class MockAgentChannel(AgentChannel):
async def receive(self, history: list[ChatMessageContent]) -> None:
pass
async def invoke(self, agent: "Agent") -> AsyncIterable[ChatMessageContent]:
yield ChatMessageContent(role=AuthorRole.SYSTEM, content="test message")
async def get_history(self) -> AsyncIterable[ChatMessageContent]:
yield ChatMessageContent(role=AuthorRole.SYSTEM, content="test history message")
async def test_receive():
mock_channel = AsyncMock(spec=MockAgentChannel)
history = [
ChatMessageContent(role=AuthorRole.SYSTEM, content="test message 1"),
ChatMessageContent(role=AuthorRole.USER, content="test message 2"),
]
await mock_channel.receive(history)
mock_channel.receive.assert_called_once_with(history)
async def test_invoke():
mock_channel = AsyncMock(spec=MockAgentChannel)
agent = AsyncMock()
async def async_generator():
yield ChatMessageContent(role=AuthorRole.SYSTEM, content="test message")
mock_channel.invoke.return_value = async_generator()
async for message in mock_channel.invoke(agent):
assert message.content == "test message"
mock_channel.invoke.assert_called_once_with(agent)
async def test_get_history():
mock_channel = AsyncMock(spec=MockAgentChannel)
async def async_generator():
yield ChatMessageContent(role=AuthorRole.SYSTEM, content="test history message")
mock_channel.get_history.return_value = async_generator()
async for message in mock_channel.get_history():
assert message.content == "test history message"
mock_channel.get_history.assert_called_once()
@@ -0,0 +1,245 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest import mock
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from semantic_kernel.agents.channels.agent_channel import AgentChannel
from semantic_kernel.agents.group_chat.agent_chat import AgentChat
from semantic_kernel.agents.group_chat.broadcast_queue import ChannelReference
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions.agent_exceptions import AgentChatException
@pytest.fixture
def agent_chat():
return AgentChat()
@pytest.fixture
def agent():
mock_agent = MagicMock()
mock_agent.name = "TestAgent"
return mock_agent
@pytest.fixture
def chat_message():
mock_chat_message = MagicMock(spec=ChatMessageContent)
mock_chat_message.role = "user"
return mock_chat_message
async def test_set_activity_or_throw_when_inactive(agent_chat):
agent_chat._is_active = False
agent_chat.set_activity_or_throw()
assert agent_chat.is_active
async def test_set_activity_or_throw_when_active(agent_chat):
agent_chat._is_active = True
with pytest.raises(Exception, match="Unable to proceed while another agent is active."):
agent_chat.set_activity_or_throw()
async def test_clear_activity_signal(agent_chat):
agent_chat._is_active = True
agent_chat.clear_activity_signal()
assert not agent_chat.is_active
async def test_get_messages_in_descending_order(agent_chat, chat_message):
agent_chat.history.messages = [chat_message, chat_message, chat_message]
messages = []
async for message in agent_chat.get_messages_in_descending_order():
messages.append(message)
assert len(messages) == 3
async def test_get_chat_messages_without_agent(agent_chat, chat_message):
agent_chat.history.messages = [chat_message]
with patch(
"semantic_kernel.agents.group_chat.agent_chat.AgentChat.get_messages_in_descending_order",
return_value=AsyncMock(),
) as mock_get_messages:
async for _ in agent_chat.get_chat_messages():
pass
mock_get_messages.assert_called_once()
async def test_get_chat_messages_with_agent(agent_chat, agent, chat_message):
agent_chat.channel_map[agent] = "test_channel"
mock_channel = mock.MagicMock(spec=AgentChannel)
agent_chat.agent_channels["test_channel"] = mock_channel
with (
patch("semantic_kernel.agents.group_chat.agent_chat.AgentChat._get_agent_hash", return_value="test_channel"),
patch("semantic_kernel.agents.group_chat.agent_chat.AgentChat._synchronize_channel", return_value=mock_channel),
patch.object(mock_channel, "get_history", return_value=AsyncMock()),
):
async for _ in agent_chat.get_chat_messages(agent):
pass
async def test_add_chat_message(agent_chat, chat_message):
with patch(
"semantic_kernel.agents.group_chat.agent_chat.AgentChat.add_chat_messages",
return_value=AsyncMock(),
) as mock_add_chat_messages:
await agent_chat.add_chat_message(chat_message)
mock_add_chat_messages.assert_called_once_with([chat_message])
async def test_add_chat_messages(agent_chat, chat_message):
with patch("semantic_kernel.agents.group_chat.broadcast_queue.BroadcastQueue.enqueue", return_value=AsyncMock()):
await agent_chat.add_chat_messages([chat_message])
assert chat_message in agent_chat.history.messages
async def test_invoke_agent(agent_chat, agent, chat_message):
mock_channel = mock.MagicMock(spec=AgentChannel)
async def mock_invoke(*args, **kwargs):
yield True, chat_message
mock_channel.invoke.side_effect = mock_invoke
with (
patch(
"semantic_kernel.agents.group_chat.agent_chat.AgentChat._get_or_create_channel", return_value=mock_channel
),
patch(
"semantic_kernel.agents.group_chat.broadcast_queue.BroadcastQueue.enqueue",
return_value=AsyncMock(),
),
):
async for _ in agent_chat.invoke_agent(agent):
pass
mock_channel.invoke.assert_called_once_with(agent)
await agent_chat.reset()
async def test_invoke_streaming_agent(agent_chat, agent, chat_message):
mock_channel = mock.MagicMock(spec=AgentChannel)
async def mock_invoke(*args, **kwargs):
yield chat_message
mock_channel.invoke_stream.side_effect = mock_invoke
with (
patch(
"semantic_kernel.agents.group_chat.agent_chat.AgentChat._get_or_create_channel", return_value=mock_channel
),
patch(
"semantic_kernel.agents.group_chat.broadcast_queue.BroadcastQueue.enqueue",
return_value=AsyncMock(),
),
):
async for _ in agent_chat.invoke_agent_stream(agent):
pass
mock_channel.invoke_stream.assert_called_once_with(agent, [])
await agent_chat.reset()
async def test_synchronize_channel_with_existing_channel(agent_chat):
mock_channel = MagicMock(spec=AgentChannel)
channel_key = "test_channel_key"
agent_chat.agent_channels[channel_key] = mock_channel
with patch(
"semantic_kernel.agents.group_chat.broadcast_queue.BroadcastQueue.ensure_synchronized", return_value=AsyncMock()
) as mock_ensure_synchronized:
result = await agent_chat._synchronize_channel(channel_key)
assert result == mock_channel
mock_ensure_synchronized.assert_called_once_with(ChannelReference(channel=mock_channel, hash=channel_key))
async def test_synchronize_channel_with_nonexistent_channel(agent_chat):
channel_key = "test_channel_key"
result = await agent_chat._synchronize_channel(channel_key)
assert result is None
def test_get_agent_hash_with_existing_hash(agent_chat, agent):
expected_hash = "existing_hash"
agent_chat.channel_map[agent] = expected_hash
result = agent_chat._get_agent_hash(agent)
assert result == expected_hash
def test_get_agent_hash_generates_new_hash(agent_chat, agent):
expected_hash = "new_hash"
agent.get_channel_keys = MagicMock(return_value=["key1", "key2"])
with patch(
"semantic_kernel.agents.group_chat.agent_chat.KeyEncoder.generate_hash", return_value=expected_hash
) as mock_generate_hash:
result = agent_chat._get_agent_hash(agent)
assert result == expected_hash
mock_generate_hash.assert_called_once_with(["key1", "key2"])
assert agent_chat.channel_map[agent] == expected_hash
async def test_add_chat_messages_throws_exception_for_system_role(agent_chat):
system_message = MagicMock(spec=ChatMessageContent)
system_message.role = AuthorRole.SYSTEM
with pytest.raises(AgentChatException, match="System messages cannot be added to the chat history."):
await agent_chat.add_chat_messages([system_message])
async def test_get_or_create_channel_creates_new_channel(agent_chat, agent):
agent_chat.history.messages = [MagicMock(spec=ChatMessageContent)]
channel_key = "test_channel_key"
mock_channel = AsyncMock(spec=AgentChannel)
with (
patch(
"semantic_kernel.agents.group_chat.agent_chat.AgentChat._get_agent_hash", return_value=channel_key
) as mock_get_agent_hash,
patch(
"semantic_kernel.agents.group_chat.agent_chat.AgentChat._synchronize_channel", return_value=None
) as mock_synchronize_channel,
):
agent.create_channel = AsyncMock(return_value=mock_channel)
with patch.object(mock_channel, "receive", return_value=AsyncMock()) as mock_receive:
result = await agent_chat._get_or_create_channel(agent)
assert result == mock_channel
mock_get_agent_hash.assert_called_once_with(agent)
mock_synchronize_channel.assert_called_once_with(channel_key)
agent.create_channel.assert_called_once()
mock_receive.assert_called_once_with(agent_chat.history.messages)
assert agent_chat.agent_channels[channel_key] == mock_channel
async def test_get_or_create_channel_reuses_existing_channel(agent_chat, agent):
channel_key = "test_channel_key"
mock_channel = MagicMock(spec=AgentChannel)
with (
patch(
"semantic_kernel.agents.group_chat.agent_chat.AgentChat._get_agent_hash", return_value=channel_key
) as mock_get_agent_hash,
patch(
"semantic_kernel.agents.group_chat.agent_chat.AgentChat._synchronize_channel", return_value=mock_channel
) as mock_synchronize_channel,
):
result = await agent_chat._get_or_create_channel(agent)
assert result == mock_channel
mock_get_agent_hash.assert_called_once_with(agent)
mock_synchronize_channel.assert_called_once_with(channel_key)
agent.create_channel.assert_not_called()
@@ -0,0 +1,27 @@
# Copyright (c) Microsoft. All rights reserved.
import base64
from hashlib import sha256
import pytest
from semantic_kernel.agents.group_chat.agent_chat_utils import KeyEncoder
from semantic_kernel.exceptions.agent_exceptions import AgentExecutionException
def test_generate_hash_valid_keys():
keys = ["key1", "key2", "key3"]
expected_joined_keys = ":".join(keys).encode("utf-8")
expected_hash = sha256(expected_joined_keys).digest()
expected_base64 = base64.b64encode(expected_hash).decode("utf-8")
result = KeyEncoder.generate_hash(keys)
assert result == expected_base64
def test_generate_hash_empty_keys():
with pytest.raises(
AgentExecutionException, match="Channel Keys must not be empty. Unable to generate channel hash."
):
KeyEncoder.generate_hash([])
@@ -0,0 +1,341 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest import mock
from unittest.mock import AsyncMock, MagicMock
import pytest
from semantic_kernel.agents.agent import Agent
from semantic_kernel.agents.group_chat.agent_chat import AgentChat
from semantic_kernel.agents.group_chat.agent_group_chat import AgentGroupChat
from semantic_kernel.agents.strategies.selection.selection_strategy import SelectionStrategy
from semantic_kernel.agents.strategies.selection.sequential_selection_strategy import SequentialSelectionStrategy
from semantic_kernel.agents.strategies.termination.default_termination_strategy import DefaultTerminationStrategy
from semantic_kernel.agents.strategies.termination.termination_strategy import TerminationStrategy
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions.agent_exceptions import AgentChatException
@pytest.fixture
def agents():
"""Fixture that provides a list of mock agents."""
return [MagicMock(spec=Agent, id=f"agent-{i}") for i in range(3)]
@pytest.fixture
def termination_strategy():
"""Fixture that provides a mock termination strategy."""
return AsyncMock(spec=TerminationStrategy)
@pytest.fixture
def selection_strategy():
"""Fixture that provides a mock selection strategy."""
return AsyncMock(spec=SelectionStrategy)
# region Non-Streaming
def test_agent_group_chat_initialization(agents, termination_strategy, selection_strategy):
group_chat = AgentGroupChat(
agents=agents, termination_strategy=termination_strategy, selection_strategy=selection_strategy
)
assert group_chat.agents == agents
assert group_chat.agent_ids == {agent.id for agent in agents}
assert group_chat.termination_strategy == termination_strategy
assert group_chat.selection_strategy == selection_strategy
def test_agent_group_chat_initialization_defaults():
group_chat = AgentGroupChat()
assert group_chat.agents == []
assert group_chat.agent_ids == set()
assert isinstance(group_chat.termination_strategy, DefaultTerminationStrategy)
assert isinstance(group_chat.selection_strategy, SequentialSelectionStrategy)
def test_add_agent(agents):
group_chat = AgentGroupChat()
group_chat.add_agent(agents[0])
assert agents[0] in group_chat.agents
assert agents[0].id in group_chat.agent_ids
def test_add_duplicate_agent(agents):
group_chat = AgentGroupChat(agents=[agents[0]])
group_chat.add_agent(agents[0])
assert len(group_chat.agents) == 1
assert len(group_chat.agent_ids) == 1
async def test_invoke_single_turn(agents, termination_strategy):
group_chat = AgentGroupChat(termination_strategy=termination_strategy)
async def mock_invoke(agent, is_joining=True):
yield MagicMock(role=AuthorRole.ASSISTANT)
with mock.patch.object(AgentGroupChat, "invoke", side_effect=mock_invoke):
termination_strategy.should_terminate.return_value = False
async for message in group_chat.invoke_single_turn(agents[0]):
assert message.role == AuthorRole.ASSISTANT
termination_strategy.should_terminate.assert_awaited_once()
async def test_invoke_single_turn_sets_complete(agents, termination_strategy):
group_chat = AgentGroupChat(termination_strategy=termination_strategy)
async def mock_invoke(agent, is_joining=True):
yield MagicMock(role=AuthorRole.ASSISTANT)
with mock.patch.object(AgentGroupChat, "invoke", side_effect=mock_invoke):
termination_strategy.should_terminate.return_value = True
async for _ in group_chat.invoke_single_turn(agents[0]):
pass
assert group_chat.is_complete is True
termination_strategy.should_terminate.assert_awaited_once()
async def test_invoke_with_agent_joining(agents, termination_strategy):
for agent in agents:
agent.name = f"Agent {agent.id}"
agent.id = f"agent-{agent.id}"
group_chat = AgentGroupChat(termination_strategy=termination_strategy)
with (
mock.patch.object(AgentGroupChat, "add_agent", autospec=True) as mock_add_agent,
mock.patch.object(AgentChat, "invoke_agent", autospec=True) as mock_invoke_agent,
):
async def mock_invoke_gen(*args, **kwargs):
yield MagicMock(role=AuthorRole.ASSISTANT)
mock_invoke_agent.side_effect = mock_invoke_gen
async for _ in group_chat.invoke(agents[0], is_joining=True):
pass
mock_add_agent.assert_called_once_with(group_chat, agents[0])
async def test_invoke_with_complete_chat(agents, termination_strategy):
termination_strategy.automatic_reset = False
group_chat = AgentGroupChat(agents=agents, termination_strategy=termination_strategy)
group_chat.is_complete = True
with pytest.raises(AgentChatException, match="Chat is already complete"):
async for _ in group_chat.invoke():
pass
async def test_invoke_agent_with_none_defined_errors(agents):
group_chat = AgentGroupChat()
with pytest.raises(AgentChatException, match="No agents are available"):
async for _ in group_chat.invoke():
pass
async def test_invoke_selection_strategy_error(agents, selection_strategy):
group_chat = AgentGroupChat(agents=agents, selection_strategy=selection_strategy)
selection_strategy.next.side_effect = Exception("Selection failed")
with pytest.raises(AgentChatException, match="Failed to select agent"):
async for _ in group_chat.invoke():
pass
async def test_invoke_iterations(agents, termination_strategy, selection_strategy):
for agent in agents:
agent.name = f"Agent {agent.id}"
agent.id = f"agent-{agent.id}"
termination_strategy.maximum_iterations = 2
group_chat = AgentGroupChat(
agents=agents, termination_strategy=termination_strategy, selection_strategy=selection_strategy
)
selection_strategy.next.side_effect = lambda agents, history: agents[0]
async def mock_invoke_agent(*args, **kwargs):
yield MagicMock(role=AuthorRole.ASSISTANT)
with mock.patch.object(AgentChat, "invoke_agent", side_effect=mock_invoke_agent):
termination_strategy.should_terminate.return_value = False
iteration_count = 0
async for _ in group_chat.invoke():
iteration_count += 1
assert iteration_count == 2
async def test_invoke_is_complete_then_reset(agents, termination_strategy, selection_strategy):
for agent in agents:
agent.name = f"Agent {agent.id}"
agent.id = f"agent-{agent.id}"
termination_strategy.maximum_iterations = 2
termination_strategy.automatic_reset = True
group_chat = AgentGroupChat(
agents=agents, termination_strategy=termination_strategy, selection_strategy=selection_strategy
)
group_chat.is_complete = True
selection_strategy.next.side_effect = lambda agents, history: agents[0]
async def mock_invoke_agent(*args, **kwargs):
yield MagicMock(role=AuthorRole.ASSISTANT)
with mock.patch.object(AgentChat, "invoke_agent", side_effect=mock_invoke_agent):
termination_strategy.should_terminate.return_value = False
iteration_count = 0
async for _ in group_chat.invoke():
iteration_count += 1
assert iteration_count == 2
# endregion
# region Streaming
async def test_invoke_streaming_single_turn(agents, termination_strategy):
group_chat = AgentGroupChat(termination_strategy=termination_strategy)
async def mock_invoke(agent, is_joining=True):
yield MagicMock(role=AuthorRole.ASSISTANT)
with mock.patch.object(AgentGroupChat, "invoke_stream", side_effect=mock_invoke):
termination_strategy.should_terminate.return_value = False
async for message in group_chat.invoke_stream_single_turn(agents[0]):
assert message.role == AuthorRole.ASSISTANT
termination_strategy.should_terminate.assert_awaited_once()
async def test_invoke_stream_with_agent_joining(agents, termination_strategy):
for agent in agents:
agent.name = f"Agent {agent.id}"
agent.id = f"agent-{agent.id}"
group_chat = AgentGroupChat(termination_strategy=termination_strategy)
with (
mock.patch.object(AgentGroupChat, "add_agent", autospec=True) as mock_add_agent,
mock.patch.object(AgentChat, "invoke_agent_stream", autospec=True) as mock_invoke_agent,
):
async def mock_invoke_gen(*args, **kwargs):
yield MagicMock(role=AuthorRole.ASSISTANT)
mock_invoke_agent.side_effect = mock_invoke_gen
async for _ in group_chat.invoke_stream(agents[0], is_joining=True):
pass
mock_add_agent.assert_called_once_with(group_chat, agents[0])
async def test_invoke_stream_with_complete_chat(agents, termination_strategy):
termination_strategy.automatic_reset = False
group_chat = AgentGroupChat(agents=agents, termination_strategy=termination_strategy)
group_chat.is_complete = True
with pytest.raises(AgentChatException, match="Chat is already complete"):
async for _ in group_chat.invoke_stream():
pass
async def test_invoke_stream_selection_strategy_error(agents, selection_strategy):
group_chat = AgentGroupChat(agents=agents, selection_strategy=selection_strategy)
selection_strategy.next.side_effect = Exception("Selection failed")
with pytest.raises(AgentChatException, match="Failed to select agent"):
async for _ in group_chat.invoke_stream():
pass
async def test_invoke_stream_iterations(agents, termination_strategy, selection_strategy):
for agent in agents:
agent.name = f"Agent {agent.id}"
agent.id = f"agent-{agent.id}"
termination_strategy.maximum_iterations = 2
group_chat = AgentGroupChat(
agents=agents, termination_strategy=termination_strategy, selection_strategy=selection_strategy
)
selection_strategy.next.side_effect = lambda agents, history: agents[0]
async def mock_invoke_agent(*args, **kwargs):
yield MagicMock(role=AuthorRole.ASSISTANT)
with mock.patch.object(AgentChat, "invoke_agent_stream", side_effect=mock_invoke_agent):
termination_strategy.should_terminate.return_value = False
iteration_count = 0
async for _ in group_chat.invoke_stream():
iteration_count += 1
assert iteration_count == 2
async def test_invoke_stream_is_complete_then_reset(agents, termination_strategy, selection_strategy):
for agent in agents:
agent.name = f"Agent {agent.id}"
agent.id = f"agent-{agent.id}"
termination_strategy.maximum_iterations = 2
termination_strategy.automatic_reset = True
group_chat = AgentGroupChat(
agents=agents, termination_strategy=termination_strategy, selection_strategy=selection_strategy
)
group_chat.is_complete = True
selection_strategy.next.side_effect = lambda agents, history: agents[0]
async def mock_invoke_agent(*args, **kwargs):
yield MagicMock(role=AuthorRole.ASSISTANT)
with mock.patch.object(AgentChat, "invoke_agent_stream", side_effect=mock_invoke_agent):
termination_strategy.should_terminate.return_value = False
iteration_count = 0
async for _ in group_chat.invoke_stream():
iteration_count += 1
assert iteration_count == 2
async def test_invoke_streaming_agent_with_none_defined_errors(agents):
group_chat = AgentGroupChat()
with pytest.raises(AgentChatException, match="No agents are available"):
async for _ in group_chat.invoke_stream():
pass
# endregion
@@ -0,0 +1,172 @@
# Copyright (c) Microsoft. All rights reserved.
import asyncio
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from semantic_kernel.agents.channels.agent_channel import AgentChannel
from semantic_kernel.agents.group_chat.broadcast_queue import BroadcastQueue, ChannelReference, QueueReference
from semantic_kernel.contents.chat_message_content import ChatMessageContent
@pytest.fixture
def channel_ref():
"""Fixture that provides a mock ChannelReference."""
mock_channel = AsyncMock(spec=AgentChannel)
return ChannelReference(channel=mock_channel, hash="test-hash")
@pytest.fixture
def message():
"""Fixture that provides a mock ChatMessageContent."""
return MagicMock(spec=ChatMessageContent)
# region QueueReference Tests
def test_queue_reference_is_empty_true():
queue_ref = QueueReference()
assert queue_ref.is_empty is True
def test_queue_reference_is_empty_false():
queue_ref = QueueReference()
queue_ref.queue.append(MagicMock())
assert queue_ref.is_empty is False
# endregion
# region BroadcastQueue Tests
async def test_enqueue_new_channel(channel_ref, message):
broadcast_queue = BroadcastQueue()
await broadcast_queue.enqueue([channel_ref], [message])
assert channel_ref.hash in broadcast_queue.queues
queue_ref = broadcast_queue.queues[channel_ref.hash]
assert queue_ref.queue[0] == [message]
assert queue_ref.receive_task is not None
assert not queue_ref.receive_task.done()
async def test_enqueue_existing_channel(channel_ref, message):
broadcast_queue = BroadcastQueue()
await broadcast_queue.enqueue([channel_ref], [message])
await broadcast_queue.enqueue([channel_ref], [message])
queue_ref = broadcast_queue.queues[channel_ref.hash]
assert len(queue_ref.queue) == 2
assert queue_ref.queue[1] == [message]
assert queue_ref.receive_task is not None
assert not queue_ref.receive_task.done()
async def test_ensure_synchronized_channel_empty(channel_ref):
broadcast_queue = BroadcastQueue()
await broadcast_queue.ensure_synchronized(channel_ref)
assert channel_ref.hash not in broadcast_queue.queues
async def test_ensure_synchronized_with_messages(channel_ref, message):
broadcast_queue = BroadcastQueue()
await broadcast_queue.enqueue([channel_ref], [message])
await broadcast_queue.ensure_synchronized(channel_ref)
queue_ref = broadcast_queue.queues[channel_ref.hash]
assert queue_ref.is_empty is True
async def test_ensure_synchronized_with_failure(channel_ref, message):
broadcast_queue = BroadcastQueue()
await broadcast_queue.enqueue([channel_ref], [message])
queue_ref = broadcast_queue.queues[channel_ref.hash]
queue_ref.receive_failure = Exception("Simulated failure")
with pytest.raises(Exception, match="Unexpected failure broadcasting to channel"):
await broadcast_queue.ensure_synchronized(channel_ref)
assert queue_ref.receive_failure is None
async def test_ensure_synchronized_creates_new_task(channel_ref, message):
broadcast_queue = BroadcastQueue()
await broadcast_queue.enqueue([channel_ref], [message])
queue_ref = broadcast_queue.queues[channel_ref.hash]
queue_ref.receive_task = None
with patch(
"semantic_kernel.agents.group_chat.broadcast_queue.BroadcastQueue.receive", new_callable=AsyncMock
) as mock_receive:
mock_receive.return_value = await asyncio.sleep(0.1)
await broadcast_queue.ensure_synchronized(channel_ref)
assert queue_ref.receive_task is None
async def test_receive_processes_queue(channel_ref, message):
broadcast_queue = BroadcastQueue()
await broadcast_queue.enqueue([channel_ref], [message])
queue_ref = broadcast_queue.queues[channel_ref.hash]
await broadcast_queue.receive(channel_ref, queue_ref)
assert queue_ref.is_empty is True
assert channel_ref.channel.receive.await_count >= 1
channel_ref.channel.receive.assert_any_await([message])
async def test_receive_handles_failure(channel_ref, message):
broadcast_queue = BroadcastQueue()
await broadcast_queue.enqueue([channel_ref], [message])
channel_ref.channel.receive.side_effect = Exception("Simulated failure")
queue_ref = broadcast_queue.queues[channel_ref.hash]
await broadcast_queue.receive(channel_ref, queue_ref)
assert queue_ref.receive_failure is not None
assert str(queue_ref.receive_failure) == "Simulated failure"
async def test_receive_breaks_when_queue_is_empty(channel_ref, message):
broadcast_queue = BroadcastQueue()
await broadcast_queue.enqueue([channel_ref], [message])
queue_ref = broadcast_queue.queues[channel_ref.hash]
assert not queue_ref.is_empty
channel_ref.channel.receive = AsyncMock()
queue_ref.queue.clear()
await broadcast_queue.receive(channel_ref, queue_ref)
channel_ref.channel.receive.assert_not_awaited()
assert queue_ref.is_empty
# endregion
@@ -0,0 +1,99 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, MagicMock
from semantic_kernel.agents.strategies.termination.aggregator_termination_strategy import (
AggregateTerminationCondition,
AggregatorTerminationStrategy,
)
from semantic_kernel.agents.strategies.termination.termination_strategy import TerminationStrategy
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from tests.unit.agents.test_agent import MockAgent
async def test_aggregate_termination_condition_all_true():
agent = MockAgent(id="test-agent-id")
history = [MagicMock(spec=ChatMessageContent)]
# Mocking two strategies that return True
strategy1 = AsyncMock(spec=TerminationStrategy)
strategy1.should_terminate.return_value = True
strategy2 = AsyncMock(spec=TerminationStrategy)
strategy2.should_terminate.return_value = True
strategy = AggregatorTerminationStrategy(
strategies=[strategy1, strategy2], condition=AggregateTerminationCondition.ALL
)
result = await strategy.should_terminate_async(agent, history)
assert result is True
strategy1.should_terminate.assert_awaited_once()
strategy2.should_terminate.assert_awaited_once()
async def test_aggregate_termination_condition_all_false():
agent = MockAgent(id="test-agent-id")
history = [MagicMock(spec=ChatMessageContent)]
# Mocking two strategies, one returns True, the other False
strategy1 = AsyncMock(spec=TerminationStrategy)
strategy1.should_terminate.return_value = True
strategy2 = AsyncMock(spec=TerminationStrategy)
strategy2.should_terminate.return_value = False
strategy = AggregatorTerminationStrategy(
strategies=[strategy1, strategy2], condition=AggregateTerminationCondition.ALL
)
result = await strategy.should_terminate_async(agent, history)
assert result is False
strategy1.should_terminate.assert_awaited_once()
strategy2.should_terminate.assert_awaited_once()
async def test_aggregate_termination_condition_any_true():
agent = MockAgent(id="test-agent-id")
history = [MagicMock(spec=ChatMessageContent)]
# Mocking two strategies, one returns False, the other True
strategy1 = AsyncMock(spec=TerminationStrategy)
strategy1.should_terminate.return_value = False
strategy2 = AsyncMock(spec=TerminationStrategy)
strategy2.should_terminate.return_value = True
strategy = AggregatorTerminationStrategy(
strategies=[strategy1, strategy2], condition=AggregateTerminationCondition.ANY
)
result = await strategy.should_terminate_async(agent, history)
assert result is True
strategy1.should_terminate.assert_awaited_once()
strategy2.should_terminate.assert_awaited_once()
async def test_aggregate_termination_condition_any_false():
agent = MockAgent(id="test-agent-id")
history = [MagicMock(spec=ChatMessageContent)]
# Mocking two strategies that return False
strategy1 = AsyncMock(spec=TerminationStrategy)
strategy1.should_terminate.return_value = False
strategy2 = AsyncMock(spec=TerminationStrategy)
strategy2.should_terminate.return_value = False
strategy = AggregatorTerminationStrategy(
strategies=[strategy1, strategy2], condition=AggregateTerminationCondition.ANY
)
result = await strategy.should_terminate_async(agent, history)
assert result is False
strategy1.should_terminate.assert_awaited_once()
strategy2.should_terminate.assert_awaited_once()
@@ -0,0 +1,10 @@
# Copyright (c) Microsoft. All rights reserved.
from semantic_kernel.agents.strategies.termination.default_termination_strategy import DefaultTerminationStrategy
async def test_should_agent_terminate_():
strategy = DefaultTerminationStrategy(maximum_iterations=2)
result = await strategy.should_agent_terminate(None, [])
assert not result
@@ -0,0 +1,106 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, MagicMock
import pytest
from semantic_kernel.agents.strategies.selection.kernel_function_selection_strategy import (
KernelFunctionSelectionStrategy,
)
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.exceptions.agent_exceptions import AgentExecutionException
from semantic_kernel.functions.kernel_function import KernelFunction
from semantic_kernel.kernel import Kernel
from tests.unit.agents.test_agent import MockAgent
@pytest.fixture
def agents():
"""Fixture that provides a list of mock agents."""
return [MockAgent(id=f"agent-{i}", name=f"Agent_{i}") for i in range(3)]
async def test_kernel_function_selection_next_success(agents):
history = [MagicMock(spec=ChatMessageContent)]
mock_function = AsyncMock(spec=KernelFunction)
mock_function.invoke.return_value = MagicMock(value="Agent_1")
mock_kernel = MagicMock(spec=Kernel)
strategy = KernelFunctionSelectionStrategy(
function=mock_function, kernel=mock_kernel, result_parser=lambda result: result.value
)
selected_agent = await strategy.next(agents, history)
assert selected_agent.name == "Agent_1"
mock_function.invoke.assert_awaited_once()
async def test_kernel_function_selection_next_agent_not_found(agents):
history = [MagicMock(spec=ChatMessageContent)]
mock_function = AsyncMock(spec=KernelFunction)
mock_function.invoke.return_value = MagicMock(value="Nonexistent-Agent")
mock_kernel = MagicMock(spec=Kernel)
strategy = KernelFunctionSelectionStrategy(
function=mock_function, kernel=mock_kernel, result_parser=lambda result: result.value
)
with pytest.raises(AgentExecutionException) as excinfo:
await strategy.next(agents, history)
assert "Strategy unable to select next agent" in str(excinfo.value)
mock_function.invoke.assert_awaited_once()
async def test_kernel_function_selection_next_result_is_none(agents):
history = [MagicMock(spec=ChatMessageContent)]
mock_function = AsyncMock(spec=KernelFunction)
mock_function.invoke.return_value = None
mock_kernel = MagicMock(spec=Kernel)
strategy = KernelFunctionSelectionStrategy(
function=mock_function, kernel=mock_kernel, result_parser=lambda result: result.value if result else None
)
with pytest.raises(AgentExecutionException) as excinfo:
await strategy.next(agents, history)
assert "Strategy unable to determine next agent" in str(excinfo.value)
mock_function.invoke.assert_awaited_once()
async def test_kernel_function_selection_next_exception_during_invoke(agents):
history = [MagicMock(spec=ChatMessageContent)]
mock_function = AsyncMock(spec=KernelFunction)
mock_function.invoke.side_effect = Exception("Test exception")
mock_kernel = MagicMock(spec=Kernel)
strategy = KernelFunctionSelectionStrategy(
function=mock_function, kernel=mock_kernel, result_parser=lambda result: result.value
)
with pytest.raises(AgentExecutionException) as excinfo:
await strategy.next(agents, history)
assert "Strategy failed to execute function" in str(excinfo.value)
mock_function.invoke.assert_awaited_once()
async def test_kernel_function_selection_result_parser_is_async(agents):
history = [MagicMock(spec=ChatMessageContent)]
mock_function = AsyncMock(spec=KernelFunction)
mock_function.invoke.return_value = MagicMock(value="Agent_2")
mock_kernel = MagicMock(spec=Kernel)
async def async_result_parser(result):
return result.value
strategy = KernelFunctionSelectionStrategy(
function=mock_function, kernel=mock_kernel, result_parser=async_result_parser
)
selected_agent = await strategy.next(agents, history)
assert selected_agent.name == "Agent_2"
mock_function.invoke.assert_awaited_once()
@@ -0,0 +1,114 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, MagicMock, patch
from semantic_kernel.agents.strategies import KernelFunctionTerminationStrategy
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.functions.kernel_arguments import KernelArguments
from semantic_kernel.functions.kernel_function import KernelFunction
from semantic_kernel.kernel import Kernel
from tests.unit.agents.test_agent import MockAgent
async def test_should_agent_terminate_with_result_true():
agent = MockAgent(id="test-agent-id")
history = [MagicMock(spec=ChatMessageContent)]
mock_function = AsyncMock(spec=KernelFunction)
mock_function.invoke.return_value = MagicMock(value=True)
mock_kernel = MagicMock(spec=Kernel)
strategy = KernelFunctionTerminationStrategy(
agents=[agent], function=mock_function, kernel=mock_kernel, result_parser=lambda result: result.value
)
result = await strategy.should_agent_terminate(agent, history)
assert result is True
mock_function.invoke.assert_awaited_once()
async def test_should_agent_terminate_with_result_false():
agent = MockAgent(id="test-agent-id")
history = [MagicMock(spec=ChatMessageContent)]
mock_function = AsyncMock(spec=KernelFunction)
mock_function.invoke.return_value = MagicMock(value=False)
mock_kernel = MagicMock(spec=Kernel)
strategy = KernelFunctionTerminationStrategy(
agents=[agent], function=mock_function, kernel=mock_kernel, result_parser=lambda result: result.value
)
result = await strategy.should_agent_terminate(agent, history)
assert result is False
mock_function.invoke.assert_awaited_once()
async def test_should_agent_terminate_with_none_result():
agent = MockAgent(id="test-agent-id")
history = [MagicMock(spec=ChatMessageContent)]
mock_function = AsyncMock(spec=KernelFunction)
mock_function.invoke.return_value = None
mock_kernel = MagicMock(spec=Kernel)
strategy = KernelFunctionTerminationStrategy(
agents=[agent],
function=mock_function,
kernel=mock_kernel,
result_parser=lambda result: result.value if result else False,
)
result = await strategy.should_agent_terminate(agent, history)
assert result is False
mock_function.invoke.assert_awaited_once()
async def test_should_agent_terminate_custom_arguments():
agent = MockAgent(id="test-agent-id")
history = [MagicMock(spec=ChatMessageContent)]
mock_function = AsyncMock(spec=KernelFunction)
mock_function.invoke.return_value = MagicMock(value=True)
mock_kernel = MagicMock(spec=Kernel)
custom_args = KernelArguments(execution_settings={"some_setting": MagicMock(model_dump=lambda: {"key": "value"})})
strategy = KernelFunctionTerminationStrategy(
agents=[agent],
function=mock_function,
kernel=mock_kernel,
arguments=custom_args,
result_parser=lambda result: result.value,
)
with patch.object(KernelArguments, "__init__", return_value=None) as mock_init:
result = await strategy.should_agent_terminate(agent, history)
mock_init.assert_called_once()
assert result is True
mock_function.invoke.assert_awaited_once()
async def test_should_agent_terminate_result_parser_awaitable():
agent = MockAgent(id="test-agent-id")
history = [MagicMock(spec=ChatMessageContent)]
mock_function = AsyncMock(spec=KernelFunction)
mock_function.invoke.return_value = MagicMock(value=True)
mock_kernel = MagicMock(spec=Kernel)
async def mock_result_parser(result):
return result.value
strategy = KernelFunctionTerminationStrategy(
agents=[agent], function=mock_function, kernel=mock_kernel, result_parser=mock_result_parser
)
result = await strategy.should_agent_terminate(agent, history)
assert result is True
mock_function.invoke.assert_awaited_once()
@@ -0,0 +1,108 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import MagicMock
import pytest
from semantic_kernel.agents.agent import Agent
from semantic_kernel.agents.strategies.selection.sequential_selection_strategy import SequentialSelectionStrategy
from semantic_kernel.exceptions.agent_exceptions import AgentExecutionException
from tests.unit.agents.test_agent import MockAgent
@pytest.fixture
def agents():
"""Fixture that provides a list of mock agents."""
return [MockAgent(id=f"agent-{i}") for i in range(3)]
async def test_sequential_selection_next(agents):
strategy = SequentialSelectionStrategy()
# Test the sequence of selections
selected_agent_1 = await strategy.next(agents, [])
selected_agent_2 = await strategy.next(agents, [])
selected_agent_3 = await strategy.next(agents, [])
assert selected_agent_1.id == "agent-0"
assert selected_agent_2.id == "agent-1"
assert selected_agent_3.id == "agent-2"
async def test_sequential_selection_wraps_around(agents):
strategy = SequentialSelectionStrategy()
for _ in range(3):
await strategy.next(agents, [])
selected_agent = await strategy.next(agents, [])
assert selected_agent.id == "agent-0"
async def test_sequential_selection_reset(agents):
strategy = SequentialSelectionStrategy()
# Move the index to the middle of the list
await strategy.next(agents, [])
await strategy.next(agents, [])
strategy.reset()
selected_agent = await strategy.next(agents, [])
assert selected_agent.id == "agent-0"
async def test_sequential_selection_exceeds_length(agents):
strategy = SequentialSelectionStrategy()
strategy._index = len(agents)
selected_agent = await strategy.next(agents, [])
assert selected_agent.id == "agent-0"
assert strategy._index == 0
selected_agent = await strategy.next(agents, [])
assert selected_agent.id == "agent-1"
assert strategy._index == 1
async def test_sequential_selection_empty_agents():
strategy = SequentialSelectionStrategy()
with pytest.raises(AgentExecutionException) as excinfo:
await strategy.next([], [])
assert "Agent Failure - No agents present to select." in str(excinfo.value)
async def test_sequential_selection_avoid_selecting_same_agent_twice():
# Arrange
agent_0 = MagicMock(spec=Agent)
agent_0.id = "agent-0"
agent_0.name = "Agent0"
agent_0.plugins = []
agent_1 = MagicMock(spec=Agent)
agent_1.id = "agent-1"
agent_1.name = "Agent1"
agent_1.plugins = []
agents = [agent_0, agent_1]
strategy = SequentialSelectionStrategy()
# Simulate that we've already selected an agent once:
strategy.has_selected = True
# Set the initial agent to the first agent
strategy.initial_agent = agent_0
# Ensure the internal index is set to -1
strategy._index = -1
# Act
selected_agent = await strategy.next(agents, [])
# Assert
# According to the condition, we should skip selecting agent_0 again
assert selected_agent.id == "agent-1"
assert strategy._index == 1
@@ -0,0 +1,63 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import MagicMock
import pytest
from semantic_kernel.agents import Agent
from semantic_kernel.agents.strategies.termination.termination_strategy import TerminationStrategy
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from tests.unit.agents.test_agent import MockAgent
class TerminationStrategyTest(TerminationStrategy):
"""A test implementation of TerminationStrategy for testing purposes."""
async def should_agent_terminate(self, agent: "Agent", history: list[ChatMessageContent]) -> bool:
"""Simple test implementation that always returns True."""
return True
async def test_should_terminate_with_matching_agent():
agent = MockAgent(id="test-agent-id")
strategy = TerminationStrategyTest(agents=[agent])
# Assuming history is a list of ChatMessageContent; can be mocked or made minimal
history = [MagicMock(spec=ChatMessageContent)]
result = await strategy.should_terminate(agent, history)
assert result is True
async def test_should_terminate_with_non_matching_agent():
agent = MockAgent(id="test-agent-id")
non_matching_agent = MockAgent(id="non-matching-agent-id")
strategy = TerminationStrategyTest(agents=[non_matching_agent])
# Assuming history is a list of ChatMessageContent; can be mocked or made minimal
history = [MagicMock(spec=ChatMessageContent)]
result = await strategy.should_terminate(agent, history)
assert result is False
async def test_should_terminate_no_agents_in_strategy():
agent = MockAgent(id="test-agent-id")
strategy = TerminationStrategyTest()
# Assuming history is a list of ChatMessageContent; can be mocked or made minimal
history = [MagicMock(spec=ChatMessageContent)]
result = await strategy.should_terminate(agent, history)
assert result is True
async def test_should_agent_terminate_not_implemented():
agent = MockAgent(id="test-agent-id")
strategy = TerminationStrategy(agents=[agent])
# Assuming history is a list of ChatMessageContent; can be mocked or made minimal
history = [MagicMock(spec=ChatMessageContent)]
with pytest.raises(NotImplementedError):
await strategy.should_agent_terminate(agent, history)
@@ -0,0 +1,50 @@
# Copyright (c) Microsoft. All rights reserved.
from datetime import timedelta
from semantic_kernel.agents.open_ai.run_polling_options import RunPollingOptions
def test_get_polling_interval_below_threshold():
options = RunPollingOptions()
iteration_count = 1
expected_interval = timedelta(milliseconds=250)
assert options.get_polling_interval(iteration_count) == expected_interval
def test_get_polling_interval_at_threshold():
options = RunPollingOptions()
iteration_count = 2
expected_interval = timedelta(milliseconds=250)
assert options.get_polling_interval(iteration_count) == expected_interval
def test_get_polling_interval_above_threshold():
options = RunPollingOptions()
iteration_count = 3
expected_interval = timedelta(seconds=1)
assert options.get_polling_interval(iteration_count) == expected_interval
def test_get_polling_interval_custom_threshold():
options = RunPollingOptions(run_polling_backoff_threshold=5)
iteration_count = 4
expected_interval = timedelta(milliseconds=250)
assert options.get_polling_interval(iteration_count) == expected_interval
iteration_count = 6
expected_interval = timedelta(seconds=1)
assert options.get_polling_interval(iteration_count) == expected_interval
def test_get_polling_interval_custom_intervals():
options = RunPollingOptions(
run_polling_interval=timedelta(milliseconds=500), run_polling_backoff=timedelta(seconds=2)
)
iteration_count = 1
expected_interval = timedelta(milliseconds=500)
assert options.get_polling_interval(iteration_count) == expected_interval
iteration_count = 3
expected_interval = timedelta(seconds=2)
assert options.get_polling_interval(iteration_count) == expected_interval
@@ -0,0 +1,400 @@
# Copyright (c) Microsoft. All rights reserved.
from collections.abc import AsyncGenerator
from unittest.mock import AsyncMock, MagicMock
import pytest
from anthropic import AsyncAnthropic
from anthropic.lib.streaming import TextEvent
from anthropic.lib.streaming._types import InputJsonEvent
from anthropic.types import (
ContentBlockStopEvent,
InputJSONDelta,
Message,
MessageDeltaUsage,
MessageStopEvent,
RawContentBlockDeltaEvent,
RawContentBlockStartEvent,
RawMessageDeltaEvent,
RawMessageStartEvent,
TextBlock,
TextDelta,
ToolUseBlock,
Usage,
)
from anthropic.types.raw_message_delta_event import Delta
from semantic_kernel.connectors.ai.anthropic.prompt_execution_settings.anthropic_prompt_execution_settings import (
AnthropicChatPromptExecutionSettings,
)
from semantic_kernel.contents.chat_message_content import (
ChatMessageContent,
FunctionCallContent,
FunctionResultContent,
TextContent,
)
from semantic_kernel.contents.const import ContentTypes
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent, StreamingTextContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.contents.utils.finish_reason import FinishReason
@pytest.fixture
def mock_tool_calls_message() -> ChatMessageContent:
return ChatMessageContent(
ai_model_id="claude-3-opus-20240229",
metadata={},
content_type="message",
role=AuthorRole.ASSISTANT,
name=None,
items=[
TextContent(
inner_content=None,
ai_model_id=None,
metadata={},
content_type="text",
text="<thinking></thinking>",
encoding=None,
),
FunctionCallContent(
inner_content=None,
ai_model_id=None,
metadata={},
content_type=ContentTypes.FUNCTION_CALL_CONTENT,
id="test_function_call_content",
index=1,
name="math-Add",
function_name="Add",
plugin_name="math",
arguments={"input": 3, "amount": 3},
),
],
encoding=None,
finish_reason=FinishReason.TOOL_CALLS,
)
@pytest.fixture
def mock_parallel_tool_calls_message() -> ChatMessageContent:
return ChatMessageContent(
ai_model_id="claude-3-opus-20240229",
metadata={},
content_type="message",
role=AuthorRole.ASSISTANT,
name=None,
items=[
TextContent(
inner_content=None,
ai_model_id=None,
metadata={},
content_type="text",
text="<thinking></thinking>",
encoding=None,
),
FunctionCallContent(
inner_content=None,
ai_model_id=None,
metadata={},
content_type=ContentTypes.FUNCTION_CALL_CONTENT,
id="test_function_call_content_1",
index=1,
name="math-Add",
function_name="Add",
plugin_name="math",
arguments={"input": 3, "amount": 3},
),
FunctionCallContent(
inner_content=None,
ai_model_id=None,
metadata={},
content_type=ContentTypes.FUNCTION_CALL_CONTENT,
id="test_function_call_content_2",
index=1,
name="math-Subtract",
function_name="Subtract",
plugin_name="math",
arguments={"input": 6, "amount": 3},
),
],
encoding=None,
finish_reason=FinishReason.TOOL_CALLS,
)
@pytest.fixture
def mock_streaming_tool_calls_message() -> list:
stream_events = [
RawMessageStartEvent(
message=Message(
id="test_message_id",
content=[],
model="claude-3-opus-20240229",
role="assistant",
stop_reason=None,
stop_sequence=None,
type="message",
usage=Usage(input_tokens=1720, output_tokens=2),
),
type="message_start",
),
RawContentBlockStartEvent(content_block=TextBlock(text="", type="text"), index=0, type="content_block_start"),
RawContentBlockDeltaEvent(
delta=TextDelta(text="<thinking>", type="text_delta"), index=0, type="content_block_delta"
),
TextEvent(type="text", text="<thinking>", snapshot="<thinking>"),
RawContentBlockDeltaEvent(
delta=TextDelta(text="</thinking>", type="text_delta"), index=0, type="content_block_delta"
),
TextEvent(type="text", text="</thinking>", snapshot="<thinking></thinking>"),
ContentBlockStopEvent(
index=0, type="content_block_stop", content_block=TextBlock(text="<thinking></thinking>", type="text")
),
RawContentBlockStartEvent(
content_block=ToolUseBlock(id="test_tool_use_message_id", input={}, name="math-Add", type="tool_use"),
index=1,
type="content_block_start",
),
RawContentBlockDeltaEvent(
delta=InputJSONDelta(partial_json='{"input": 3, "amount": 3}', type="input_json_delta"),
index=1,
type="content_block_delta",
),
InputJsonEvent(type="input_json", partial_json='{"input": 3, "amount": 3}', snapshot={"input": 3, "amount": 3}),
ContentBlockStopEvent(
index=1,
type="content_block_stop",
content_block=ToolUseBlock(
id="test_tool_use_block_id", input={"input": 3, "amount": 3}, name="math-Add", type="tool_use"
),
),
RawMessageDeltaEvent(
delta=Delta(stop_reason="tool_use", stop_sequence=None),
type="message_delta",
usage=MessageDeltaUsage(output_tokens=159),
),
MessageStopEvent(
type="message_stop",
message=Message(
id="test_message_id",
content=[
TextBlock(text="<thinking></thinking>", type="text"),
ToolUseBlock(
id="test_tool_use_block_id", input={"input": 3, "amount": 3}, name="math-Add", type="tool_use"
),
],
model="claude-3-opus-20240229",
role="assistant",
stop_reason="tool_use",
stop_sequence=None,
type="message",
usage=Usage(input_tokens=100, output_tokens=100),
),
),
]
async def async_generator():
for event in stream_events:
yield event
stream_mock = AsyncMock()
stream_mock.__aenter__.return_value = async_generator()
return stream_mock
@pytest.fixture
def mock_tool_call_result_message() -> ChatMessageContent:
return ChatMessageContent(
inner_content=None,
ai_model_id=None,
metadata={},
content_type="message",
role=AuthorRole.TOOL,
name=None,
items=[
FunctionResultContent(
id="test_function_call_content",
result=6,
)
],
encoding=None,
finish_reason=FinishReason.TOOL_CALLS,
)
@pytest.fixture
def mock_parallel_tool_call_result_message() -> ChatMessageContent:
return ChatMessageContent(
inner_content=None,
ai_model_id=None,
metadata={},
content_type="message",
role=AuthorRole.TOOL,
name=None,
items=[
FunctionResultContent(
id="test_function_call_content_1",
result=6,
),
FunctionResultContent(
id="test_function_call_content_2",
result=3,
),
],
encoding=None,
finish_reason=FinishReason.TOOL_CALLS,
)
@pytest.fixture
def mock_streaming_chat_message_content() -> StreamingChatMessageContent:
return StreamingChatMessageContent(
choice_index=0,
ai_model_id="claude-3-opus-20240229",
metadata={},
role=AuthorRole.ASSISTANT,
name=None,
items=[
StreamingTextContent(
inner_content=None,
ai_model_id=None,
metadata={},
content_type="text",
text="<thinking></thinking>",
encoding=None,
choice_index=0,
),
FunctionCallContent(
inner_content=None,
ai_model_id=None,
metadata={},
content_type=ContentTypes.FUNCTION_CALL_CONTENT,
id="tool_id",
index=0,
name="math-Add",
function_name="Add",
plugin_name="math",
arguments='{"input": 3, "amount": 3}',
),
],
encoding=None,
finish_reason=FinishReason.TOOL_CALLS,
)
@pytest.fixture
def mock_settings() -> AnthropicChatPromptExecutionSettings:
return AnthropicChatPromptExecutionSettings()
@pytest.fixture
def mock_chat_message_response() -> Message:
return Message(
id="test_message_id",
content=[TextBlock(text="Hello, how are you?", type="text")],
model="claude-3-opus-20240229",
role="assistant",
stop_reason="end_turn",
stop_sequence=None,
type="message",
usage=Usage(input_tokens=10, output_tokens=10),
)
@pytest.fixture
def mock_streaming_message_response() -> AsyncGenerator:
raw_message_start_event = RawMessageStartEvent(
message=Message(
id="test_message_id",
content=[],
model="claude-3-opus-20240229",
role="assistant",
stop_reason=None,
stop_sequence=None,
type="message",
usage=Usage(input_tokens=41, output_tokens=3),
),
type="message_start",
)
raw_content_block_start_event = RawContentBlockStartEvent(
content_block=TextBlock(text="", type="text"),
index=0,
type="content_block_start",
)
raw_content_block_delta_event = RawContentBlockDeltaEvent(
delta=TextDelta(text="Hello! It", type="text_delta"),
index=0,
type="content_block_delta",
)
text_event = TextEvent(
type="text",
text="Hello! It",
snapshot="Hello! It",
)
content_block_stop_event = ContentBlockStopEvent(
index=0,
type="content_block_stop",
content_block=TextBlock(text="Hello! It's nice to meet you.", type="text"),
)
raw_message_delta_event = RawMessageDeltaEvent(
delta=Delta(stop_reason="end_turn", stop_sequence=None),
type="message_delta",
usage=MessageDeltaUsage(output_tokens=84),
)
message_stop_event = MessageStopEvent(
type="message_stop",
message=Message(
id="test_message_stop_id",
content=[TextBlock(text="Hello! It's nice to meet you.", type="text")],
model="claude-3-opus-20240229",
role="assistant",
stop_reason="end_turn",
stop_sequence=None,
type="message",
usage=Usage(input_tokens=41, output_tokens=84),
),
)
# Combine all mock events into a list
stream_events = [
raw_message_start_event,
raw_content_block_start_event,
raw_content_block_delta_event,
text_event,
content_block_stop_event,
raw_message_delta_event,
message_stop_event,
]
async def async_generator():
for event in stream_events:
yield event
# Create an AsyncMock for the stream
stream_mock = AsyncMock()
stream_mock.__aenter__.return_value = async_generator()
return stream_mock
@pytest.fixture
def mock_anthropic_client_completion(mock_chat_message_response: Message) -> AsyncAnthropic:
client = MagicMock(spec=AsyncAnthropic)
messages_mock = MagicMock()
messages_mock.create = AsyncMock(return_value=mock_chat_message_response)
client.messages = messages_mock
return client
@pytest.fixture
def mock_anthropic_client_completion_stream(mock_streaming_message_response: AsyncGenerator) -> AsyncAnthropic:
client = MagicMock(spec=AsyncAnthropic)
messages_mock = MagicMock()
messages_mock.stream.return_value = mock_streaming_message_response
client.messages = messages_mock
return client
@@ -0,0 +1,549 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from anthropic import AsyncAnthropic
from anthropic.types import Message
from semantic_kernel.connectors.ai.anthropic.prompt_execution_settings.anthropic_prompt_execution_settings import (
AnthropicChatPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.anthropic.services.anthropic_chat_completion import AnthropicChatCompletion
from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import (
OpenAIChatPromptExecutionSettings,
)
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.contents.chat_message_content import ChatMessageContent, FunctionCallContent, TextContent
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions import (
ServiceInitializationError,
ServiceInvalidExecutionSettingsError,
ServiceResponseException,
)
from semantic_kernel.exceptions.service_exceptions import ServiceInvalidRequestError
from semantic_kernel.functions.kernel_arguments import KernelArguments
from semantic_kernel.functions.kernel_function_decorator import kernel_function
from semantic_kernel.kernel import Kernel
async def test_complete_chat_contents(
kernel: Kernel,
mock_settings: AnthropicChatPromptExecutionSettings,
mock_chat_message_response: Message,
):
client = MagicMock(spec=AsyncAnthropic)
messages_mock = MagicMock()
messages_mock.create = AsyncMock(return_value=mock_chat_message_response)
client.messages = messages_mock
chat_history = ChatHistory()
chat_history.add_user_message("test_user_message")
arguments = KernelArguments()
chat_completion_base = AnthropicChatCompletion(
ai_model_id="test_model_id", service_id="test", api_key="", async_client=client
)
content: list[ChatMessageContent] = await chat_completion_base.get_chat_message_contents(
chat_history=chat_history, settings=mock_settings, kernel=kernel, arguments=arguments
)
assert len(content) > 0
assert content[0].content != ""
assert content[0].role == AuthorRole.ASSISTANT
mock_message_text_content = ChatMessageContent(role=AuthorRole.ASSISTANT, items=[TextContent(text="test")])
mock_message_function_call = ChatMessageContent(
role=AuthorRole.ASSISTANT,
items=[
FunctionCallContent(
name="test",
arguments={"key": "test"},
)
],
)
@pytest.mark.parametrize(
"function_choice_behavior,model_responses,expected_result",
[
pytest.param(
FunctionChoiceBehavior.Auto(),
[[mock_message_function_call], [mock_message_text_content]],
TextContent,
id="auto",
),
pytest.param(
FunctionChoiceBehavior.Auto(auto_invoke=False),
[[mock_message_function_call]],
FunctionCallContent,
id="auto_none_invoke",
),
pytest.param(
FunctionChoiceBehavior.Required(auto_invoke=False),
[[mock_message_function_call]],
FunctionCallContent,
id="required_none_invoke",
),
],
)
async def test_complete_chat_contents_function_call_behavior_tool_call(
kernel: Kernel,
mock_settings: AnthropicChatPromptExecutionSettings,
function_choice_behavior: FunctionChoiceBehavior,
model_responses,
expected_result,
):
kernel.add_function("test", kernel_function(lambda key: "test", name="test"))
mock_settings.function_choice_behavior = function_choice_behavior
arguments = KernelArguments()
chat_completion_base = AnthropicChatCompletion(ai_model_id="test_model_id", service_id="test", api_key="")
with (
patch.object(chat_completion_base, "_inner_get_chat_message_contents", side_effect=model_responses),
):
response: list[ChatMessageContent] = await chat_completion_base.get_chat_message_contents(
chat_history=ChatHistory(system_message="Test"), settings=mock_settings, kernel=kernel, arguments=arguments
)
assert all(isinstance(content, expected_result) for content in response[0].items)
async def test_complete_chat_contents_function_call_behavior_without_kernel(
mock_settings: AnthropicChatPromptExecutionSettings,
mock_anthropic_client_completion: AsyncAnthropic,
):
chat_history = MagicMock()
chat_completion_base = AnthropicChatCompletion(
ai_model_id="test_model_id", service_id="test", api_key="", async_client=mock_anthropic_client_completion
)
mock_settings.function_choice_behavior = FunctionChoiceBehavior.Auto()
with pytest.raises(ServiceInvalidExecutionSettingsError):
await chat_completion_base.get_chat_message_contents(chat_history=chat_history, settings=mock_settings)
async def test_complete_chat_stream_contents(
kernel: Kernel,
mock_settings: AnthropicChatPromptExecutionSettings,
mock_streaming_message_response,
):
client = MagicMock(spec=AsyncAnthropic)
messages_mock = MagicMock()
messages_mock.stream.return_value = mock_streaming_message_response
client.messages = messages_mock
chat_history = ChatHistory()
chat_history.add_user_message("test_user_message")
arguments = KernelArguments()
chat_completion_base = AnthropicChatCompletion(
ai_model_id="test_model_id",
service_id="test",
api_key="",
async_client=client,
)
async for content in chat_completion_base.get_streaming_chat_message_contents(
chat_history, mock_settings, kernel=kernel, arguments=arguments
):
assert content is not None
mock_message_function_call = StreamingChatMessageContent(
role=AuthorRole.ASSISTANT, items=[FunctionCallContent(name="test")], choice_index="0"
)
mock_message_text_content = StreamingChatMessageContent(
role=AuthorRole.ASSISTANT, items=[TextContent(text="test")], choice_index="0"
)
@pytest.mark.parametrize(
"function_choice_behavior,model_responses,expected_result",
[
pytest.param(
FunctionChoiceBehavior.Auto(),
[[mock_message_function_call], [mock_message_text_content]],
TextContent,
id="auto",
),
pytest.param(
FunctionChoiceBehavior.Auto(auto_invoke=False),
[[mock_message_function_call]],
FunctionCallContent,
id="auto_none_invoke",
),
pytest.param(
FunctionChoiceBehavior.Required(auto_invoke=False),
[[mock_message_function_call]],
FunctionCallContent,
id="required_none_invoke",
),
pytest.param(FunctionChoiceBehavior.NoneInvoke(), [[mock_message_text_content]], TextContent, id="none"),
],
)
async def test_complete_chat_contents_streaming_function_call_behavior_tool_call(
kernel: Kernel,
mock_settings: AnthropicChatPromptExecutionSettings,
function_choice_behavior: FunctionChoiceBehavior,
model_responses,
expected_result,
):
mock_settings.function_choice_behavior = function_choice_behavior
# Mock sequence of model responses
generator_mocks = []
for mock_message in model_responses:
generator_mock = MagicMock()
generator_mock.__aiter__.return_value = [mock_message]
generator_mocks.append(generator_mock)
arguments = KernelArguments()
chat_completion_base = AnthropicChatCompletion(ai_model_id="test_model_id", service_id="test", api_key="")
with patch.object(chat_completion_base, "_inner_get_streaming_chat_message_contents", side_effect=generator_mocks):
messages = []
async for chunk in chat_completion_base.get_streaming_chat_message_contents(
chat_history=ChatHistory(system_message="Test"), settings=mock_settings, kernel=kernel, arguments=arguments
):
messages.append(chunk)
response = messages[-1]
assert all(isinstance(content, expected_result) for content in response[0].items)
async def test_anthropic_sdk_exception(kernel: Kernel, mock_settings: AnthropicChatPromptExecutionSettings):
client = MagicMock(spec=AsyncAnthropic)
messages_mock = MagicMock()
messages_mock.create.side_effect = Exception("Test Exception")
client.messages = messages_mock
chat_history = MagicMock()
arguments = KernelArguments()
chat_completion_base = AnthropicChatCompletion(
ai_model_id="test_model_id", service_id="test", api_key="", async_client=client
)
with pytest.raises(ServiceResponseException):
await chat_completion_base.get_chat_message_contents(
chat_history=chat_history, settings=mock_settings, kernel=kernel, arguments=arguments
)
async def test_anthropic_sdk_exception_streaming(kernel: Kernel, mock_settings: AnthropicChatPromptExecutionSettings):
client = MagicMock(spec=AsyncAnthropic)
messages_mock = MagicMock()
messages_mock.stream.side_effect = Exception("Test Exception")
client.messages = messages_mock
chat_history = MagicMock()
arguments = KernelArguments()
chat_completion_base = AnthropicChatCompletion(
ai_model_id="test_model_id", service_id="test", api_key="", async_client=client
)
with pytest.raises(ServiceResponseException):
async for content in chat_completion_base.get_streaming_chat_message_contents(
chat_history, mock_settings, kernel=kernel, arguments=arguments
):
assert content is not None
def test_anthropic_chat_completion_init(anthropic_unit_test_env) -> None:
# Test successful initialization
anthropic_chat_completion = AnthropicChatCompletion()
assert anthropic_chat_completion.ai_model_id == anthropic_unit_test_env["ANTHROPIC_CHAT_MODEL_ID"]
assert isinstance(anthropic_chat_completion, ChatCompletionClientBase)
@pytest.mark.parametrize("exclude_list", [["ANTHROPIC_API_KEY"]], indirect=True)
def test_anthropic_chat_completion_init_with_empty_api_key(anthropic_unit_test_env) -> None:
ai_model_id = "test_model_id"
with pytest.raises(ServiceInitializationError):
AnthropicChatCompletion(
ai_model_id=ai_model_id,
env_file_path="test.env",
)
@pytest.mark.parametrize("exclude_list", [["ANTHROPIC_CHAT_MODEL_ID"]], indirect=True)
def test_anthropic_chat_completion_init_with_empty_model_id(anthropic_unit_test_env) -> None:
with pytest.raises(ServiceInitializationError):
AnthropicChatCompletion(
env_file_path="test.env",
)
def test_prompt_execution_settings_class(anthropic_unit_test_env):
anthropic_chat_completion = AnthropicChatCompletion()
prompt_execution_settings = anthropic_chat_completion.get_prompt_execution_settings_class()
assert prompt_execution_settings == AnthropicChatPromptExecutionSettings
async def test_with_different_execution_settings(kernel: Kernel, mock_anthropic_client_completion: MagicMock):
chat_history = MagicMock()
settings = OpenAIChatPromptExecutionSettings(temperature=0.2)
arguments = KernelArguments()
chat_completion_base = AnthropicChatCompletion(
ai_model_id="test_model_id", service_id="test", api_key="", async_client=mock_anthropic_client_completion
)
await chat_completion_base.get_chat_message_contents(
chat_history=chat_history, settings=settings, kernel=kernel, arguments=arguments
)
assert mock_anthropic_client_completion.messages.create.call_args.kwargs["temperature"] == 0.2
async def test_with_different_execution_settings_stream(
kernel: Kernel, mock_anthropic_client_completion_stream: MagicMock
):
chat_history = MagicMock()
settings = OpenAIChatPromptExecutionSettings(temperature=0.2, seed=2)
arguments = KernelArguments()
chat_completion_base = AnthropicChatCompletion(
ai_model_id="test_model_id",
service_id="test",
api_key="",
async_client=mock_anthropic_client_completion_stream,
)
async for chunk in chat_completion_base.get_streaming_chat_message_contents(
chat_history, settings, kernel=kernel, arguments=arguments
):
assert chunk is not None
assert mock_anthropic_client_completion_stream.messages.stream.call_args.kwargs["temperature"] == 0.2
async def test_prepare_chat_history_for_request_with_system_message(mock_anthropic_client_completion_stream: MagicMock):
chat_history = ChatHistory()
chat_history.add_system_message("System message")
chat_history.add_user_message("User message")
chat_history.add_assistant_message("Assistant message")
chat_history.add_system_message("Another system message")
chat_completion_base = AnthropicChatCompletion(
ai_model_id="test_model_id",
service_id="test",
api_key="",
async_client=mock_anthropic_client_completion_stream,
)
remaining_messages, system_message_content = chat_completion_base._prepare_chat_history_for_request(
chat_history, role_key="role", content_key="content"
)
assert system_message_content == "System message"
assert remaining_messages == [
{"role": AuthorRole.USER, "content": "User message"},
{"role": AuthorRole.ASSISTANT, "content": [{"type": "text", "text": "Assistant message"}]},
]
assert not any(msg["role"] == AuthorRole.SYSTEM for msg in remaining_messages)
async def test_prepare_chat_history_for_request_with_tool_message(
mock_anthropic_client_completion_stream: MagicMock,
mock_tool_calls_message: ChatMessageContent,
mock_tool_call_result_message: ChatMessageContent,
):
chat_history = ChatHistory()
chat_history.add_user_message("What is 3+3?")
chat_history.add_message(mock_tool_calls_message)
chat_history.add_message(mock_tool_call_result_message)
chat_completion_client = AnthropicChatCompletion(
ai_model_id="test_model_id",
service_id="test",
api_key="",
async_client=mock_anthropic_client_completion_stream,
)
remaining_messages, system_message_content = chat_completion_client._prepare_chat_history_for_request(
chat_history, role_key="role", content_key="content"
)
assert system_message_content is None
assert remaining_messages == [
{"role": AuthorRole.USER, "content": "What is 3+3?"},
{
"role": AuthorRole.ASSISTANT,
"content": [
{"type": "text", "text": mock_tool_calls_message.items[0].text},
{
"type": "tool_use",
"id": mock_tool_calls_message.items[1].id,
"name": mock_tool_calls_message.items[1].name,
"input": mock_tool_calls_message.items[1].arguments,
},
],
},
{
"role": AuthorRole.USER,
"content": [
{
"type": "tool_result",
"tool_use_id": mock_tool_call_result_message.items[0].id,
"content": str(mock_tool_call_result_message.items[0].result),
}
],
},
]
async def test_prepare_chat_history_for_request_with_parallel_tool_message(
mock_anthropic_client_completion_stream: MagicMock,
mock_parallel_tool_calls_message: ChatMessageContent,
mock_parallel_tool_call_result_message: ChatMessageContent,
):
chat_history = ChatHistory()
chat_history.add_user_message("What is 3+3?")
chat_history.add_message(mock_parallel_tool_calls_message)
chat_history.add_message(mock_parallel_tool_call_result_message)
chat_completion_client = AnthropicChatCompletion(
ai_model_id="test_model_id",
service_id="test",
api_key="",
async_client=mock_anthropic_client_completion_stream,
)
remaining_messages, system_message_content = chat_completion_client._prepare_chat_history_for_request(
chat_history, role_key="role", content_key="content"
)
assert system_message_content is None
assert remaining_messages == [
{"role": AuthorRole.USER, "content": "What is 3+3?"},
{
"role": AuthorRole.ASSISTANT,
"content": [
{"type": "text", "text": mock_parallel_tool_calls_message.items[0].text},
*[
{
"type": "tool_use",
"id": function_call_content.id,
"name": function_call_content.name,
"input": function_call_content.arguments,
}
for function_call_content in mock_parallel_tool_calls_message.items[1:]
],
],
},
{
"role": AuthorRole.USER,
"content": [
{
"type": "tool_result",
"tool_use_id": function_result_content.id,
"content": str(function_result_content.result),
}
for function_result_content in mock_parallel_tool_call_result_message.items
],
},
]
async def test_prepare_chat_history_for_request_with_tool_message_right_after_user_message(
mock_anthropic_client_completion_stream: MagicMock,
mock_tool_call_result_message: ChatMessageContent,
):
chat_history = ChatHistory()
chat_history.add_user_message("What is 3+3?")
chat_history.add_message(mock_tool_call_result_message)
chat_completion_client = AnthropicChatCompletion(
ai_model_id="test_model_id",
service_id="test",
api_key="",
async_client=mock_anthropic_client_completion_stream,
)
with pytest.raises(ServiceInvalidRequestError, match="Tool message found after a user or system message."):
chat_completion_client._prepare_chat_history_for_request(chat_history, role_key="role", content_key="content")
async def test_prepare_chat_history_for_request_with_tool_message_as_the_first_message(
mock_anthropic_client_completion_stream: MagicMock,
mock_tool_call_result_message: ChatMessageContent,
):
chat_history = ChatHistory()
chat_history.add_message(mock_tool_call_result_message)
chat_completion_client = AnthropicChatCompletion(
ai_model_id="test_model_id",
service_id="test",
api_key="",
async_client=mock_anthropic_client_completion_stream,
)
with pytest.raises(ServiceInvalidRequestError, match="Tool message found without a preceding message."):
chat_completion_client._prepare_chat_history_for_request(chat_history, role_key="role", content_key="content")
async def test_send_chat_stream_request_tool_calls(
mock_streaming_tool_calls_message: MagicMock,
mock_streaming_chat_message_content: StreamingChatMessageContent,
):
chat_history = ChatHistory()
chat_history.add_user_message("What is 3+3?")
chat_history.add_message(mock_streaming_chat_message_content)
settings = AnthropicChatPromptExecutionSettings(
temperature=0.2,
max_tokens=100,
top_p=1.0,
frequency_penalty=0.0,
presence_penalty=0.0,
chat_history=chat_history,
)
client = MagicMock(spec=AsyncAnthropic)
messages_mock = MagicMock()
messages_mock.stream.return_value = mock_streaming_tool_calls_message
client.messages = messages_mock
chat_completion = AnthropicChatCompletion(
ai_model_id="test_model_id",
service_id="test",
api_key="",
async_client=client,
)
response = chat_completion._send_chat_stream_request(settings)
async for message in response:
assert message is not None
def test_client_base_url(mock_anthropic_client_completion: MagicMock):
chat_completion_base = AnthropicChatCompletion(
ai_model_id="test_model_id", service_id="test", api_key="", async_client=mock_anthropic_client_completion
)
assert chat_completion_base.service_url() is not None
def test_chat_completion_reset_settings(
mock_anthropic_client_completion: MagicMock,
):
chat_completion = AnthropicChatCompletion(
ai_model_id="test_model_id", service_id="test", api_key="", async_client=mock_anthropic_client_completion
)
settings = AnthropicChatPromptExecutionSettings(tools=[{"name": "test"}], tool_choice={"type": "any"})
chat_completion._reset_function_choice_settings(settings)
assert settings.tools is None
assert settings.tool_choice is None
@@ -0,0 +1,129 @@
# Copyright (c) Microsoft. All rights reserved.
import pytest
from semantic_kernel.connectors.ai.anthropic.prompt_execution_settings.anthropic_prompt_execution_settings import (
AnthropicChatPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
from semantic_kernel.exceptions import ServiceInvalidExecutionSettingsError
def test_default_anthropic_chat_prompt_execution_settings():
settings = AnthropicChatPromptExecutionSettings()
assert settings.temperature is None
assert settings.top_p is None
assert settings.max_tokens == 1024
assert settings.messages is None
def test_custom_anthropic_chat_prompt_execution_settings():
settings = AnthropicChatPromptExecutionSettings(
temperature=0.5,
top_p=0.5,
max_tokens=128,
messages=[{"role": "system", "content": "Hello"}],
)
assert settings.temperature == 0.5
assert settings.top_p == 0.5
assert settings.max_tokens == 128
assert settings.messages == [{"role": "system", "content": "Hello"}]
def test_anthropic_chat_prompt_execution_settings_from_default_completion_config():
settings = PromptExecutionSettings(service_id="test_service")
chat_settings = AnthropicChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.service_id == "test_service"
assert chat_settings.temperature is None
assert chat_settings.top_p is None
assert chat_settings.max_tokens == 1024
def test_anthropic_chat_prompt_execution_settings_from_openai_prompt_execution_settings():
chat_settings = AnthropicChatPromptExecutionSettings(service_id="test_service", temperature=1.0)
new_settings = AnthropicChatPromptExecutionSettings(service_id="test_2", temperature=0.0)
chat_settings.update_from_prompt_execution_settings(new_settings)
assert chat_settings.service_id == "test_2"
assert chat_settings.temperature == 0.0
def test_anthropic_chat_prompt_execution_settings_from_custom_completion_config():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"temperature": 0.5,
"top_p": 0.5,
"max_tokens": 128,
"messages": [{"role": "system", "content": "Hello"}],
},
)
chat_settings = AnthropicChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.temperature == 0.5
assert chat_settings.top_p == 0.5
assert chat_settings.max_tokens == 128
def test_openai_chat_prompt_execution_settings_from_custom_completion_config_with_none():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"temperature": 0.5,
"top_p": 0.5,
"max_tokens": 128,
"messages": [{"role": "system", "content": "Hello"}],
},
)
chat_settings = AnthropicChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.temperature == 0.5
assert chat_settings.top_p == 0.5
assert chat_settings.max_tokens == 128
def test_openai_chat_prompt_execution_settings_from_custom_completion_config_with_functions():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"temperature": 0.5,
"top_p": 0.5,
"max_tokens": 128,
"tools": [{}],
"messages": [{"role": "system", "content": "Hello"}],
},
)
chat_settings = AnthropicChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.temperature == 0.5
assert chat_settings.top_p == 0.5
assert chat_settings.max_tokens == 128
def test_create_options():
settings = AnthropicChatPromptExecutionSettings(
service_id="test_service",
extension_data={
"temperature": 0.5,
"top_p": 0.5,
"max_tokens": 128,
"tools": [{}],
"messages": [{"role": "system", "content": "Hello"}],
},
)
options = settings.prepare_settings_dict()
assert options["temperature"] == 0.5
assert options["top_p"] == 0.5
assert options["max_tokens"] == 128
def test_tool_choice_none():
with pytest.raises(ServiceInvalidExecutionSettingsError, match="Tool choice 'none' is not supported by Anthropic."):
AnthropicChatPromptExecutionSettings(
service_id="test_service",
extension_data={
"temperature": 0.5,
"top_p": 0.5,
"max_tokens": 128,
"tool_choice": {"type": "none"},
"messages": [{"role": "system", "content": "Hello"}],
},
function_choice_behavior=FunctionChoiceBehavior.NoneInvoke(),
)
@@ -0,0 +1,278 @@
# Copyright (c) Microsoft. All rights reserved.
import datetime
from collections.abc import AsyncGenerator, AsyncIterator
from unittest.mock import MagicMock
import pytest
from azure.ai.inference.aio import ChatCompletionsClient, EmbeddingsClient
from azure.ai.inference.models import (
ChatChoice,
ChatCompletions,
ChatCompletionsToolCall,
ChatResponseMessage,
CompletionsUsage,
FunctionCall,
StreamingChatChoiceUpdate,
StreamingChatCompletionsUpdate,
StreamingChatResponseToolCallUpdate,
)
from azure.core.credentials import AzureKeyCredential
from semantic_kernel.connectors.ai.azure_ai_inference import (
AzureAIInferenceChatCompletion,
AzureAIInferenceTextEmbedding,
)
@pytest.fixture()
def model_id() -> str:
return "test_model_id"
@pytest.fixture()
def service_id() -> str:
return "test_service_id"
@pytest.fixture()
def azure_ai_inference_unit_test_env(monkeypatch, exclude_list, override_env_param_dict):
"""Fixture to set environment variables for Azure AI Inference Unit Tests."""
if exclude_list is None:
exclude_list = []
if override_env_param_dict is None:
override_env_param_dict = {}
env_vars = {
"AZURE_AI_INFERENCE_API_KEY": "test-api-key",
"AZURE_AI_INFERENCE_ENDPOINT": "https://test-endpoint.com",
}
env_vars.update(override_env_param_dict)
for key, value in env_vars.items():
if key not in exclude_list:
monkeypatch.setenv(key, value)
else:
monkeypatch.delenv(key, raising=False)
return env_vars
@pytest.fixture()
def model_diagnostics_test_env(monkeypatch, exclude_list, override_env_param_dict):
"""Fixture to set environment variables for Azure AI Inference Unit Tests."""
if exclude_list is None:
exclude_list = []
if override_env_param_dict is None:
override_env_param_dict = {}
env_vars = {
"SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS": "true",
"SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE": "true",
}
env_vars.update(override_env_param_dict)
for key, value in env_vars.items():
if key not in exclude_list:
monkeypatch.setenv(key, value)
else:
monkeypatch.delenv(key, raising=False)
return env_vars
@pytest.fixture()
def disabled_model_diagnostics_test_env(monkeypatch):
"""Fixture to disable diagnostics for tests that use mocking.
This is needed because AIInferenceInstrumentor's instrument/uninstrument
cycle interferes with class-level mocking of ChatCompletionsClient.complete.
"""
monkeypatch.setenv("SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS", "false")
monkeypatch.setenv("SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE", "false")
@pytest.fixture(scope="function")
def azure_ai_inference_client(azure_ai_inference_unit_test_env, request) -> ChatCompletionsClient | EmbeddingsClient:
"""Fixture to create Azure AI Inference client for unit tests."""
endpoint = azure_ai_inference_unit_test_env["AZURE_AI_INFERENCE_ENDPOINT"]
api_key = azure_ai_inference_unit_test_env["AZURE_AI_INFERENCE_API_KEY"]
credential = AzureKeyCredential(api_key)
if request.param == AzureAIInferenceChatCompletion.__name__:
return ChatCompletionsClient(endpoint=endpoint, credential=credential)
if request.param == AzureAIInferenceTextEmbedding.__name__:
return EmbeddingsClient(endpoint=endpoint, credential=credential)
raise ValueError(f"Service {request.param} not supported.")
@pytest.fixture(scope="function")
def azure_ai_inference_service(azure_ai_inference_unit_test_env, model_id, request):
"""Fixture to create Azure AI Inference service for unit tests.
This is required because the Azure AI Inference services require a client to be created,
and the client will be talking to the endpoint at creation time.
"""
endpoint = azure_ai_inference_unit_test_env["AZURE_AI_INFERENCE_ENDPOINT"]
api_key = azure_ai_inference_unit_test_env["AZURE_AI_INFERENCE_API_KEY"]
if request.param == AzureAIInferenceChatCompletion.__name__:
return AzureAIInferenceChatCompletion(model_id, api_key=api_key, endpoint=endpoint)
if request.param == AzureAIInferenceTextEmbedding.__name__:
return AzureAIInferenceTextEmbedding(model_id, api_key=api_key, endpoint=endpoint)
raise ValueError(f"Service {request.param} not supported.")
@pytest.fixture()
def mock_azure_ai_inference_chat_completion_response(model_id) -> ChatCompletions:
return ChatCompletions(
id="test_id",
created=datetime.datetime.now(),
model=model_id,
usage=CompletionsUsage(
completion_tokens=0,
prompt_tokens=0,
total_tokens=0,
),
choices=[
ChatChoice(
index=0,
finish_reason="stop",
message=ChatResponseMessage(
role="assistant",
content="Hello",
),
)
],
)
@pytest.fixture()
def mock_azure_ai_inference_chat_completion_response_with_tool_call(model_id) -> ChatCompletions:
return ChatCompletions(
id="test_id",
created=datetime.datetime.now(),
model=model_id,
usage=CompletionsUsage(
completion_tokens=0,
prompt_tokens=0,
total_tokens=0,
),
choices=[
ChatChoice(
index=0,
finish_reason="tool_calls",
message=ChatResponseMessage(
role="assistant",
tool_calls=[
ChatCompletionsToolCall(
id="test_id",
function=FunctionCall(
name="getLightStatus",
arguments='{"arg1": "test_value"}',
),
),
],
),
)
],
)
@pytest.fixture()
def mock_azure_ai_inference_streaming_chat_completion_response(model_id) -> AsyncIterator:
streaming_chat_response = MagicMock(spec=AsyncGenerator)
streaming_chat_response.__aiter__.return_value = [
StreamingChatCompletionsUpdate(
id="test_id",
created=datetime.datetime.now(),
model=model_id,
usage=CompletionsUsage(
completion_tokens=0,
prompt_tokens=0,
total_tokens=0,
),
choices=[
# Empty choice
],
),
StreamingChatCompletionsUpdate(
id="test_id",
created=datetime.datetime.now(),
model=model_id,
usage=CompletionsUsage(
completion_tokens=0,
prompt_tokens=0,
total_tokens=0,
),
choices=[
StreamingChatChoiceUpdate(
index=0,
finish_reason="stop",
delta=ChatResponseMessage(
role="assistant",
content="Hello",
),
)
],
),
]
return streaming_chat_response
@pytest.fixture()
def mock_azure_ai_inference_streaming_chat_completion_response_with_tool_call(model_id) -> AsyncIterator:
streaming_chat_response = MagicMock(spec=AsyncGenerator)
streaming_chat_response.__aiter__.return_value = [
StreamingChatCompletionsUpdate(
id="test_id",
created=datetime.datetime.now(),
model=model_id,
usage=CompletionsUsage(
completion_tokens=0,
prompt_tokens=0,
total_tokens=0,
),
choices=[
# Empty choice
],
),
StreamingChatCompletionsUpdate(
id="test_id",
created=datetime.datetime.now(),
model=model_id,
usage=CompletionsUsage(
completion_tokens=0,
prompt_tokens=0,
total_tokens=0,
),
choices=[
StreamingChatChoiceUpdate(
index=0,
finish_reason="tool_calls",
delta=ChatResponseMessage(
role="assistant",
tool_calls=[
StreamingChatResponseToolCallUpdate(
id="test_id",
function=FunctionCall(
name="getLightStatus",
arguments='{"arg1": "test_value"}',
),
),
],
),
)
],
),
]
return streaming_chat_response
@@ -0,0 +1,710 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, patch
import pytest
from azure.ai.inference.aio import ChatCompletionsClient
from azure.ai.inference.models import JsonSchemaFormat, UserMessage
from azure.core.credentials import AzureKeyCredential
from pydantic import BaseModel, Field
from semantic_kernel.connectors.ai.azure_ai_inference import (
AzureAIInferenceChatCompletion,
AzureAIInferenceChatPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.azure_ai_inference.azure_ai_inference_settings import AzureAIInferenceSettings
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.contents.utils.finish_reason import FinishReason
from semantic_kernel.exceptions.service_exceptions import (
ServiceInitializationError,
ServiceInvalidExecutionSettingsError,
)
from semantic_kernel.functions.kernel_arguments import KernelArguments
from semantic_kernel.kernel import Kernel
from semantic_kernel.utils.telemetry.user_agent import SEMANTIC_KERNEL_USER_AGENT
# region init
def test_azure_ai_inference_chat_completion_init(azure_ai_inference_unit_test_env, model_id) -> None:
"""Test initialization of AzureAIInferenceChatCompletion"""
azure_ai_inference = AzureAIInferenceChatCompletion(model_id, instruction_role="developer")
assert azure_ai_inference.ai_model_id == model_id
assert azure_ai_inference.service_id == model_id
assert isinstance(azure_ai_inference.client, ChatCompletionsClient)
assert azure_ai_inference.instruction_role == "developer"
@patch("azure.ai.inference.aio.ChatCompletionsClient.__init__", return_value=None)
def test_azure_ai_inference_chat_completion_client_init(
mock_client, azure_ai_inference_unit_test_env, model_id
) -> None:
"""Test initialization of the Azure AI Inference client"""
endpoint = azure_ai_inference_unit_test_env["AZURE_AI_INFERENCE_ENDPOINT"]
api_key = azure_ai_inference_unit_test_env["AZURE_AI_INFERENCE_API_KEY"]
settings = AzureAIInferenceSettings(endpoint=endpoint, api_key=api_key)
_ = AzureAIInferenceChatCompletion(model_id)
assert mock_client.call_count == 1
assert isinstance(mock_client.call_args.kwargs["endpoint"], str)
assert mock_client.call_args.kwargs["endpoint"] == str(settings.endpoint)
assert isinstance(mock_client.call_args.kwargs["credential"], AzureKeyCredential)
assert mock_client.call_args.kwargs["credential"].key == settings.api_key.get_secret_value()
assert mock_client.call_args.kwargs["user_agent"] == SEMANTIC_KERNEL_USER_AGENT
def test_azure_ai_inference_chat_completion_init_with_service_id(
azure_ai_inference_unit_test_env, model_id, service_id
) -> None:
"""Test initialization of AzureAIInferenceChatCompletion with service_id"""
azure_ai_inference = AzureAIInferenceChatCompletion(model_id, service_id=service_id)
assert azure_ai_inference.ai_model_id == model_id
assert azure_ai_inference.service_id == service_id
assert isinstance(azure_ai_inference.client, ChatCompletionsClient)
def test_azure_ai_inference_chat_completion_init_with_api_version(azure_ai_inference_unit_test_env, model_id) -> None:
"""Test initialization of AzureAIInferenceChatCompletion with api_version"""
azure_ai_inference = AzureAIInferenceChatCompletion(model_id, api_version="2024-02-15-test")
assert azure_ai_inference.ai_model_id == model_id
assert isinstance(azure_ai_inference.client, ChatCompletionsClient)
assert azure_ai_inference.client._config.api_version == "2024-02-15-test"
@pytest.mark.parametrize(
"azure_ai_inference_client",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
def test_azure_ai_inference_chat_completion_init_with_custom_client(azure_ai_inference_client, model_id) -> None:
"""Test initialization of AzureAIInferenceChatCompletion with custom client"""
client = azure_ai_inference_client
azure_ai_inference = AzureAIInferenceChatCompletion(model_id, client=client)
assert azure_ai_inference.ai_model_id == model_id
assert azure_ai_inference.service_id == model_id
assert azure_ai_inference.client == client
@pytest.mark.parametrize("exclude_list", [["AZURE_AI_INFERENCE_ENDPOINT"]], indirect=True)
def test_azure_ai_inference_chat_completion_init_with_empty_endpoint(
azure_ai_inference_unit_test_env, model_id
) -> None:
"""Test initialization of AzureAIInferenceChatCompletion with empty endpoint"""
with pytest.raises(ServiceInitializationError):
AzureAIInferenceChatCompletion(model_id, env_file_path="fake_path")
def test_prompt_execution_settings_class(azure_ai_inference_unit_test_env, model_id) -> None:
azure_ai_inference = AzureAIInferenceChatCompletion(model_id)
assert azure_ai_inference.get_prompt_execution_settings_class() == AzureAIInferenceChatPromptExecutionSettings
# endregion init
# region chat completion
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
async def test_azure_ai_inference_chat_completion(
mock_complete,
azure_ai_inference_service,
model_id,
chat_history: ChatHistory,
mock_azure_ai_inference_chat_completion_response,
) -> None:
"""Test completion of AzureAIInferenceChatCompletion"""
user_message_content: str = "Hello"
chat_history.add_user_message(user_message_content)
settings = AzureAIInferenceChatPromptExecutionSettings()
mock_complete.return_value = mock_azure_ai_inference_chat_completion_response
responses = await azure_ai_inference_service.get_chat_message_contents(chat_history=chat_history, settings=settings)
mock_complete.assert_awaited_once_with(
messages=[UserMessage(content=user_message_content)],
model=model_id,
model_extras=None,
**settings.prepare_settings_dict(),
)
assert len(responses) == 1
assert responses[0].role == "assistant"
assert responses[0].content == "Hello"
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
async def test_azure_ai_inference_chat_completion_with_standard_parameters(
mock_complete,
azure_ai_inference_service,
model_id,
chat_history: ChatHistory,
mock_azure_ai_inference_chat_completion_response,
) -> None:
"""Test completion of AzureAIInferenceChatCompletion with standard OpenAI parameters"""
user_message_content: str = "Hello"
chat_history.add_user_message(user_message_content)
settings = AzureAIInferenceChatPromptExecutionSettings(
frequency_penalty=0.5,
max_tokens=100,
presence_penalty=0.5,
seed=123,
stop="stop",
temperature=0.5,
top_p=0.5,
)
mock_complete.return_value = mock_azure_ai_inference_chat_completion_response
responses = await azure_ai_inference_service.get_chat_message_contents(chat_history=chat_history, settings=settings)
mock_complete.assert_awaited_once_with(
messages=[UserMessage(content=user_message_content)],
model=model_id,
model_extras=None,
frequency_penalty=settings.frequency_penalty,
max_tokens=settings.max_tokens,
presence_penalty=settings.presence_penalty,
seed=settings.seed,
stop=settings.stop,
temperature=settings.temperature,
top_p=settings.top_p,
)
assert len(responses) == 1
assert responses[0].role == "assistant"
assert responses[0].content == "Hello"
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
async def test_azure_ai_inference_chat_completion_with_extra_parameters(
mock_complete,
azure_ai_inference_service,
model_id,
chat_history: ChatHistory,
mock_azure_ai_inference_chat_completion_response,
) -> None:
"""Test completion of AzureAIInferenceChatCompletion with extra parameters"""
user_message_content: str = "Hello"
chat_history.add_user_message(user_message_content)
extra_parameters = {"test_key": "test_value"}
settings = AzureAIInferenceChatPromptExecutionSettings(extra_parameters=extra_parameters)
mock_complete.return_value = mock_azure_ai_inference_chat_completion_response
responses = await azure_ai_inference_service.get_chat_message_contents(chat_history=chat_history, settings=settings)
mock_complete.assert_awaited_once_with(
messages=[UserMessage(content=user_message_content)],
model=model_id,
model_extras=settings.extra_parameters,
)
assert len(responses) == 1
assert responses[0].role == "assistant"
assert responses[0].content == "Hello"
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
async def test_azure_ai_inference_chat_completion_with_function_choice_behavior_fail_verification(
azure_ai_inference_service,
kernel,
chat_history: ChatHistory,
) -> None:
"""Test completion of AzureAIInferenceChatCompletion with function choice behavior expect verification failure"""
# Missing kernel
with pytest.raises(ServiceInvalidExecutionSettingsError):
settings = AzureAIInferenceChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
await azure_ai_inference_service.get_chat_message_contents(
chat_history=chat_history,
settings=settings,
arguments=KernelArguments(),
)
# More than 1 responses
with pytest.raises(ServiceInvalidExecutionSettingsError):
settings = AzureAIInferenceChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
extra_parameters={"n": 2},
)
await azure_ai_inference_service.get_chat_message_contents(
chat_history=chat_history,
settings=settings,
kernel=kernel,
arguments=KernelArguments(),
)
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
async def test_azure_ai_inference_chat_completion_with_function_choice_behavior(
mock_complete,
azure_ai_inference_service,
kernel: Kernel,
chat_history: ChatHistory,
mock_azure_ai_inference_chat_completion_response_with_tool_call,
mock_azure_ai_inference_chat_completion_response,
decorated_native_function,
disabled_model_diagnostics_test_env,
) -> None:
"""Test completion of AzureAIInferenceChatCompletion with function choice behavior"""
user_message_content: str = "Hello"
chat_history.add_user_message(user_message_content)
settings = AzureAIInferenceChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
settings.function_choice_behavior.maximum_auto_invoke_attempts = 1
# First call returns tool call, second call returns final response
mock_complete.side_effect = [
mock_azure_ai_inference_chat_completion_response_with_tool_call,
mock_azure_ai_inference_chat_completion_response,
]
kernel.add_function(plugin_name="TestPlugin", function=decorated_native_function)
responses = await azure_ai_inference_service.get_chat_message_contents(
chat_history=chat_history,
settings=settings,
kernel=kernel,
arguments=KernelArguments(),
)
# Completion should be called twice:
# One for the tool call and one for the last completion
# after the maximum_auto_invoke_attempts is reached
assert mock_complete.call_count == 2
assert len(responses) == 1
assert responses[0].role == "assistant"
assert responses[0].content == "Hello"
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
async def test_azure_ai_inference_chat_completion_with_function_choice_behavior_no_tool_call(
mock_complete,
azure_ai_inference_service,
model_id,
kernel,
chat_history: ChatHistory,
mock_azure_ai_inference_chat_completion_response,
) -> None:
"""Test completion of AzureAIInferenceChatCompletion with function choice behavior but no tool call"""
user_message_content: str = "Hello"
chat_history.add_user_message(user_message_content)
settings = AzureAIInferenceChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
mock_complete.return_value = mock_azure_ai_inference_chat_completion_response
responses = await azure_ai_inference_service.get_chat_message_contents(
chat_history=chat_history,
settings=settings,
kernel=kernel,
arguments=KernelArguments(),
)
mock_complete.assert_awaited_once_with(
messages=[UserMessage(content=user_message_content)],
model=model_id,
model_extras=None,
**settings.prepare_settings_dict(),
)
assert len(responses) == 1
assert responses[0].role == "assistant"
assert responses[0].content == "Hello"
class MockResponseModel(BaseModel):
a: int = Field(..., description="The a field")
b: str = Field(..., description="The b field")
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
async def test_azure_ai_inference_response_format_json_schema(
mock_complete,
azure_ai_inference_service,
model_id,
chat_history: ChatHistory,
mock_azure_ai_inference_chat_completion_response,
):
chat_history.add_user_message("Return an object with fields a and b.")
settings = AzureAIInferenceChatPromptExecutionSettings(
response_format=MockResponseModel,
structured_json_response=True,
)
mock_complete.return_value = mock_azure_ai_inference_chat_completion_response
_ = await azure_ai_inference_service.get_chat_message_contents(chat_history=chat_history, settings=settings)
assert mock_complete.call_count == 1
kwargs = mock_complete.call_args.kwargs
assert "response_format" in kwargs
response_format = kwargs["response_format"]
assert isinstance(response_format, JsonSchemaFormat)
assert response_format.name == "MockResponseModel"
assert response_format.strict is True
schema = response_format.schema
assert schema["title"] == "MockResponseModel"
assert "properties" in schema
assert "a" in schema["properties"]
assert schema["properties"]["a"]["type"] == "integer"
assert "b" in schema["properties"]
assert schema["properties"]["b"]["type"] == "string"
assert kwargs["messages"][0].content == "Return an object with fields a and b."
assert kwargs["model"] == model_id
# endregion chat completion
# region streaming chat completion
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
async def test_azure_ai_inference_streaming_chat_completion(
mock_complete,
azure_ai_inference_service,
model_id,
chat_history: ChatHistory,
mock_azure_ai_inference_streaming_chat_completion_response,
) -> None:
"""Test streaming completion of AzureAIInferenceChatCompletion"""
user_message_content: str = "Hello"
chat_history.add_user_message(user_message_content)
settings = AzureAIInferenceChatPromptExecutionSettings()
mock_complete.return_value = mock_azure_ai_inference_streaming_chat_completion_response
async for messages in azure_ai_inference_service.get_streaming_chat_message_contents(chat_history, settings):
assert len(messages) == 1
assert messages[0].role == "assistant"
assert messages[0].content == "Hello"
mock_complete.assert_awaited_once_with(
stream=True,
messages=[UserMessage(content=user_message_content)],
model=model_id,
model_extras=None,
**settings.prepare_settings_dict(),
)
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
async def test_azure_ai_inference_chat_streaming_completion_with_standard_parameters(
mock_complete,
azure_ai_inference_service,
model_id,
chat_history: ChatHistory,
mock_azure_ai_inference_streaming_chat_completion_response,
) -> None:
"""Test streaming completion of AzureAIInferenceChatCompletion with standard OpenAI parameters"""
user_message_content: str = "Hello"
chat_history.add_user_message(user_message_content)
settings = AzureAIInferenceChatPromptExecutionSettings(
frequency_penalty=0.5,
max_tokens=100,
presence_penalty=0.5,
seed=123,
stop="stop",
temperature=0.5,
top_p=0.5,
)
mock_complete.return_value = mock_azure_ai_inference_streaming_chat_completion_response
async for messages in azure_ai_inference_service.get_streaming_chat_message_contents(chat_history, settings):
assert len(messages) == 1
assert messages[0].role == "assistant"
assert messages[0].content == "Hello"
mock_complete.assert_awaited_once_with(
stream=True,
messages=[UserMessage(content=user_message_content)],
model=model_id,
model_extras=None,
frequency_penalty=settings.frequency_penalty,
max_tokens=settings.max_tokens,
presence_penalty=settings.presence_penalty,
seed=settings.seed,
stop=settings.stop,
temperature=settings.temperature,
top_p=settings.top_p,
)
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
async def test_azure_ai_inference_streaming_chat_completion_with_extra_parameters(
mock_complete,
azure_ai_inference_service,
model_id,
chat_history: ChatHistory,
mock_azure_ai_inference_streaming_chat_completion_response,
) -> None:
"""Test streaming completion of AzureAIInferenceChatCompletion with extra parameters"""
user_message_content: str = "Hello"
chat_history.add_user_message(user_message_content)
extra_parameters = {"test_key": "test_value"}
settings = AzureAIInferenceChatPromptExecutionSettings(extra_parameters=extra_parameters)
mock_complete.return_value = mock_azure_ai_inference_streaming_chat_completion_response
async for messages in azure_ai_inference_service.get_streaming_chat_message_contents(chat_history, settings):
assert len(messages) == 1
assert messages[0].role == "assistant"
assert messages[0].content == "Hello"
mock_complete.assert_awaited_once_with(
stream=True,
messages=[UserMessage(content=user_message_content)],
model=model_id,
model_extras=settings.extra_parameters,
)
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
async def test_azure_ai_inference_streaming_chat_completion_with_function_choice_behavior_fail_verification(
azure_ai_inference_service,
kernel,
chat_history: ChatHistory,
) -> None:
"""Test completion of AzureAIInferenceChatCompletion with function choice behavior expect verification failure"""
# Missing kernel
with pytest.raises(ServiceInvalidExecutionSettingsError):
settings = AzureAIInferenceChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
async for _ in azure_ai_inference_service.get_streaming_chat_message_contents(
chat_history,
settings,
arguments=KernelArguments(),
):
pass
# More than 1 responses
with pytest.raises(ServiceInvalidExecutionSettingsError):
settings = AzureAIInferenceChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
extra_parameters={"n": 2},
)
async for _ in azure_ai_inference_service.get_streaming_chat_message_contents(
chat_history,
settings,
kernel=kernel,
arguments=KernelArguments(),
):
pass
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
async def test_azure_ai_inference_streaming_chat_completion_with_function_choice_behavior(
mock_complete,
azure_ai_inference_service,
kernel: Kernel,
chat_history: ChatHistory,
mock_azure_ai_inference_streaming_chat_completion_response_with_tool_call,
decorated_native_function,
) -> None:
"""Test streaming completion of AzureAIInferenceChatCompletion with function choice behavior."""
user_message_content: str = "Hello"
chat_history.add_user_message(user_message_content)
settings = AzureAIInferenceChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
settings.function_choice_behavior.maximum_auto_invoke_attempts = 1
mock_complete.return_value = mock_azure_ai_inference_streaming_chat_completion_response_with_tool_call
kernel.add_function(plugin_name="TestPlugin", function=decorated_native_function)
all_messages = []
async for messages in azure_ai_inference_service.get_streaming_chat_message_contents(
chat_history,
settings,
kernel=kernel,
arguments=KernelArguments(),
):
all_messages.extend(messages)
# Assert the number of total messages
assert len(all_messages) == 2, f"Expected 2 messages, got {len(all_messages)}"
# Validate the first message
assert all_messages[0].role == "assistant", f"Unexpected role for first message: {all_messages[0].role}"
assert all_messages[0].content == "", f"Unexpected content for first message: {all_messages[0].content}"
assert all_messages[0].finish_reason == FinishReason.TOOL_CALLS, (
f"Unexpected finish reason for first message: {all_messages[0].finish_reason}"
)
# Validate the second message
assert all_messages[1].role == "tool", f"Unexpected role for second message: {all_messages[1].role}"
assert all_messages[1].content == "", f"Unexpected content for second message: {all_messages[1].content}"
assert all_messages[1].finish_reason is None
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
async def test_azure_ai_inference_streaming_chat_completion_with_function_choice_behavior_no_tool_call(
mock_complete,
azure_ai_inference_service,
model_id,
kernel,
chat_history: ChatHistory,
mock_azure_ai_inference_streaming_chat_completion_response,
) -> None:
"""Test streaming completion of AzureAIInferenceChatCompletion with function choice behavior but no tool call"""
user_message_content: str = "Hello"
chat_history.add_user_message(user_message_content)
settings = AzureAIInferenceChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
mock_complete.return_value = mock_azure_ai_inference_streaming_chat_completion_response
async for messages in azure_ai_inference_service.get_streaming_chat_message_contents(
chat_history,
settings,
kernel=kernel,
arguments=KernelArguments(),
):
assert len(messages) == 1
assert messages[0].role == "assistant"
assert messages[0].content == "Hello"
mock_complete.assert_awaited_once_with(
stream=True,
messages=[UserMessage(content=user_message_content)],
model=model_id,
model_extras=None,
**settings.prepare_settings_dict(),
)
class MockStreamingResponseModel(BaseModel):
foo: float = Field(..., description="Foo value")
bar: bool = Field(..., description="Bar value")
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
async def test_azure_ai_inference_streaming_response_format_json_schema(
mock_complete,
azure_ai_inference_service,
model_id,
chat_history: ChatHistory,
mock_azure_ai_inference_streaming_chat_completion_response,
):
chat_history.add_user_message("Stream a response with foo and bar.")
settings = AzureAIInferenceChatPromptExecutionSettings(
response_format=MockStreamingResponseModel,
structured_json_response=True,
)
mock_complete.return_value = mock_azure_ai_inference_streaming_chat_completion_response
messages = []
async for chunk in azure_ai_inference_service.get_streaming_chat_message_contents(chat_history, settings):
messages.extend(chunk)
assert mock_complete.call_count == 1
kwargs = mock_complete.call_args.kwargs
assert "response_format" in kwargs
response_format = kwargs["response_format"]
assert isinstance(response_format, JsonSchemaFormat)
assert response_format.name == "MockStreamingResponseModel"
assert response_format.strict is True
schema = response_format.schema
assert schema["title"] == "MockStreamingResponseModel"
assert "foo" in schema["properties"]
assert schema["properties"]["foo"]["type"] == "number"
assert "bar" in schema["properties"]
assert schema["properties"]["bar"]["type"] == "boolean"
assert kwargs["stream"] is True
assert kwargs["model"] == model_id
# endregion streaming chat completion
@@ -0,0 +1,163 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, patch
import pytest
from azure.ai.inference.aio import EmbeddingsClient
from azure.core.credentials import AzureKeyCredential
from semantic_kernel.connectors.ai.azure_ai_inference import (
AzureAIInferenceEmbeddingPromptExecutionSettings,
AzureAIInferenceTextEmbedding,
)
from semantic_kernel.connectors.ai.azure_ai_inference.azure_ai_inference_settings import AzureAIInferenceSettings
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
from semantic_kernel.utils.telemetry.user_agent import SEMANTIC_KERNEL_USER_AGENT
def test_azure_ai_inference_text_embedding_init(azure_ai_inference_unit_test_env, model_id) -> None:
"""Test initialization of AzureAIInferenceTextEmbedding"""
azure_ai_inference = AzureAIInferenceTextEmbedding(model_id)
assert azure_ai_inference.ai_model_id == model_id
assert azure_ai_inference.service_id == model_id
assert isinstance(azure_ai_inference.client, EmbeddingsClient)
@patch("azure.ai.inference.aio.EmbeddingsClient.__init__", return_value=None)
def test_azure_ai_inference_text_embedding_client_init(mock_client, azure_ai_inference_unit_test_env, model_id) -> None:
"""Test initialization of the Azure AI Inference client"""
endpoint = azure_ai_inference_unit_test_env["AZURE_AI_INFERENCE_ENDPOINT"]
api_key = azure_ai_inference_unit_test_env["AZURE_AI_INFERENCE_API_KEY"]
settings = AzureAIInferenceSettings(endpoint=endpoint, api_key=api_key)
_ = AzureAIInferenceTextEmbedding(model_id)
assert mock_client.call_count == 1
assert isinstance(mock_client.call_args.kwargs["endpoint"], str)
assert mock_client.call_args.kwargs["endpoint"] == str(settings.endpoint)
assert isinstance(mock_client.call_args.kwargs["credential"], AzureKeyCredential)
assert mock_client.call_args.kwargs["credential"].key == settings.api_key.get_secret_value()
assert mock_client.call_args.kwargs["user_agent"] == SEMANTIC_KERNEL_USER_AGENT
def test_azure_ai_inference_text_embedding_init_with_service_id(
azure_ai_inference_unit_test_env, model_id, service_id
) -> None:
"""Test initialization of AzureAIInferenceTextEmbedding"""
azure_ai_inference = AzureAIInferenceTextEmbedding(model_id, service_id=service_id)
assert azure_ai_inference.ai_model_id == model_id
assert azure_ai_inference.service_id == service_id
assert isinstance(azure_ai_inference.client, EmbeddingsClient)
def test_azure_ai_inference_text_embedding_init_with_api_version(azure_ai_inference_unit_test_env, model_id) -> None:
"""Test initialization of AzureAIInferenceTextEmbedding with api_version"""
azure_ai_inference = AzureAIInferenceTextEmbedding(model_id, api_version="2024-02-15-test")
assert azure_ai_inference.ai_model_id == model_id
assert isinstance(azure_ai_inference.client, EmbeddingsClient)
assert azure_ai_inference.client._config.api_version == "2024-02-15-test"
@pytest.mark.parametrize(
"azure_ai_inference_client",
[AzureAIInferenceTextEmbedding.__name__],
indirect=True,
)
def test_azure_ai_inference_chat_completion_init_with_custom_client(azure_ai_inference_client, model_id) -> None:
"""Test initialization of AzureAIInferenceTextEmbedding with custom client"""
client = azure_ai_inference_client
azure_ai_inference = AzureAIInferenceTextEmbedding(model_id, client=client)
assert azure_ai_inference.ai_model_id == model_id
assert azure_ai_inference.service_id == model_id
assert azure_ai_inference.client == client
@pytest.mark.parametrize("exclude_list", [["AZURE_AI_INFERENCE_ENDPOINT"]], indirect=True)
def test_azure_ai_inference_text_embedding_init_with_empty_endpoint(azure_ai_inference_unit_test_env, model_id) -> None:
"""Test initialization of AzureAIInferenceTextEmbedding with empty endpoint"""
with pytest.raises(ServiceInitializationError):
AzureAIInferenceTextEmbedding(model_id, env_file_path="fake_path")
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceTextEmbedding.__name__],
indirect=True,
)
@patch.object(EmbeddingsClient, "embed", new_callable=AsyncMock)
async def test_azure_ai_inference_text_embedding(
mock_embed,
azure_ai_inference_service,
model_id,
) -> None:
"""Test text embedding generation of AzureAIInferenceTextEmbedding without settings"""
texts = ["hello", "world"]
await azure_ai_inference_service.generate_embeddings(texts)
mock_embed.assert_awaited_once_with(
input=texts,
model=model_id,
model_extras=None,
dimensions=None,
encoding_format=None,
input_type=None,
)
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceTextEmbedding.__name__],
indirect=True,
)
@patch.object(EmbeddingsClient, "embed", new_callable=AsyncMock)
async def test_azure_ai_inference_text_embedding_with_standard_settings(
mock_embed,
azure_ai_inference_service,
model_id,
) -> None:
"""Test text embedding generation of AzureAIInferenceTextEmbedding with standard settings"""
texts = ["hello", "world"]
settings = AzureAIInferenceEmbeddingPromptExecutionSettings(
dimensions=1024, encoding_format="float", input_type="text"
)
await azure_ai_inference_service.generate_embeddings(texts, settings)
mock_embed.assert_awaited_once_with(
input=texts,
model=model_id,
model_extras=None,
dimensions=settings.dimensions,
encoding_format=settings.encoding_format,
input_type=settings.input_type,
)
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceTextEmbedding.__name__],
indirect=True,
)
@patch.object(EmbeddingsClient, "embed", new_callable=AsyncMock)
async def test_azure_ai_inference_text_embedding_with_extra_parameters(
mock_embed,
azure_ai_inference_service,
model_id,
) -> None:
"""Test text embedding generation of AzureAIInferenceTextEmbedding with extra parameters"""
texts = ["hello", "world"]
extra_parameters = {"test_key": "test_value"}
settings = AzureAIInferenceEmbeddingPromptExecutionSettings(extra_parameters=extra_parameters)
await azure_ai_inference_service.generate_embeddings(texts, settings)
mock_embed.assert_awaited_once_with(
input=texts,
model=model_id,
model_extras=extra_parameters,
dimensions=settings.dimensions,
encoding_format=settings.encoding_format,
input_type=settings.input_type,
)
@@ -0,0 +1,239 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, patch
import pytest
from azure.ai.inference.aio import ChatCompletionsClient
from semantic_kernel.connectors.ai.azure_ai_inference.azure_ai_inference_prompt_execution_settings import (
AzureAIInferenceChatPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.azure_ai_inference.services.azure_ai_inference_chat_completion import (
AzureAIInferenceChatCompletion,
)
from semantic_kernel.contents.chat_history import ChatHistory
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
@patch("azure.ai.inference.tracing.AIInferenceInstrumentor.uninstrument")
@patch("azure.ai.inference.tracing.AIInferenceInstrumentor.instrument")
async def test_azure_ai_inference_chat_completion_instrumentation(
mock_instrument,
mock_uninstrument,
mock_complete,
azure_ai_inference_service,
chat_history: ChatHistory,
mock_azure_ai_inference_chat_completion_response,
model_diagnostics_test_env,
) -> None:
"""Test completion of AzureAIInferenceChatCompletion"""
settings = AzureAIInferenceChatPromptExecutionSettings()
mock_complete.return_value = mock_azure_ai_inference_chat_completion_response
await azure_ai_inference_service.get_chat_message_contents(chat_history=chat_history, settings=settings)
mock_instrument.assert_called_once_with(enable_content_recording=True)
mock_uninstrument.assert_called_once()
@pytest.mark.parametrize(
"azure_ai_inference_service",
[
AzureAIInferenceChatCompletion.__name__,
],
indirect=True,
)
@pytest.mark.parametrize(
"override_env_param_dict",
[
{
"SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS": "False",
"SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE": "False",
},
],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
@patch("azure.ai.inference.tracing.AIInferenceInstrumentor.uninstrument")
@patch("azure.ai.inference.tracing.AIInferenceInstrumentor.instrument")
async def test_azure_ai_inference_chat_completion_not_instrumentation(
mock_instrument,
mock_uninstrument,
mock_complete,
azure_ai_inference_service,
chat_history: ChatHistory,
mock_azure_ai_inference_chat_completion_response,
model_diagnostics_test_env,
) -> None:
"""Test completion of AzureAIInferenceChatCompletion"""
settings = AzureAIInferenceChatPromptExecutionSettings()
mock_complete.return_value = mock_azure_ai_inference_chat_completion_response
await azure_ai_inference_service.get_chat_message_contents(chat_history=chat_history, settings=settings)
mock_instrument.assert_not_called()
mock_uninstrument.assert_not_called()
@pytest.mark.parametrize(
"azure_ai_inference_service",
[
AzureAIInferenceChatCompletion.__name__,
],
indirect=True,
)
@pytest.mark.parametrize(
"override_env_param_dict",
[
{
"SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS": "True",
"SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE": "False",
},
],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
@patch("azure.ai.inference.tracing.AIInferenceInstrumentor.uninstrument")
@patch("azure.ai.inference.tracing.AIInferenceInstrumentor.instrument")
async def test_azure_ai_inference_chat_completion_instrumentation_without_sensitive(
mock_instrument,
mock_uninstrument,
mock_complete,
azure_ai_inference_service,
chat_history: ChatHistory,
mock_azure_ai_inference_chat_completion_response,
model_diagnostics_test_env,
) -> None:
"""Test completion of AzureAIInferenceChatCompletion"""
settings = AzureAIInferenceChatPromptExecutionSettings()
mock_complete.return_value = mock_azure_ai_inference_chat_completion_response
await azure_ai_inference_service.get_chat_message_contents(chat_history=chat_history, settings=settings)
mock_instrument.assert_called_once_with(enable_content_recording=False)
mock_uninstrument.assert_called_once()
@pytest.mark.parametrize(
"azure_ai_inference_service",
[AzureAIInferenceChatCompletion.__name__],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
@patch("azure.ai.inference.tracing.AIInferenceInstrumentor.uninstrument")
@patch("azure.ai.inference.tracing.AIInferenceInstrumentor.instrument")
async def test_azure_ai_inference_streaming_chat_completion_instrumentation(
mock_instrument,
mock_uninstrument,
mock_complete,
azure_ai_inference_service,
chat_history: ChatHistory,
mock_azure_ai_inference_streaming_chat_completion_response,
model_diagnostics_test_env,
) -> None:
"""Test completion of AzureAIInferenceChatCompletion"""
settings = AzureAIInferenceChatPromptExecutionSettings()
mock_complete.return_value = mock_azure_ai_inference_streaming_chat_completion_response
async for _ in azure_ai_inference_service.get_streaming_chat_message_contents(
chat_history=chat_history, settings=settings
):
pass
mock_instrument.assert_called_once_with(enable_content_recording=True)
mock_uninstrument.assert_called_once()
@pytest.mark.parametrize(
"azure_ai_inference_service",
[
AzureAIInferenceChatCompletion.__name__,
],
indirect=True,
)
@pytest.mark.parametrize(
"override_env_param_dict",
[
{
"SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS": "False",
"SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE": "False",
},
],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
@patch("azure.ai.inference.tracing.AIInferenceInstrumentor.uninstrument")
@patch("azure.ai.inference.tracing.AIInferenceInstrumentor.instrument")
async def test_azure_ai_inference_streaming_chat_completion_not_instrumentation(
mock_instrument,
mock_uninstrument,
mock_complete,
azure_ai_inference_service,
chat_history: ChatHistory,
mock_azure_ai_inference_streaming_chat_completion_response,
model_diagnostics_test_env,
) -> None:
"""Test completion of AzureAIInferenceChatCompletion"""
settings = AzureAIInferenceChatPromptExecutionSettings()
mock_complete.return_value = mock_azure_ai_inference_streaming_chat_completion_response
async for _ in azure_ai_inference_service.get_streaming_chat_message_contents(
chat_history=chat_history, settings=settings
):
pass
mock_instrument.assert_not_called()
mock_uninstrument.assert_not_called()
@pytest.mark.parametrize(
"azure_ai_inference_service",
[
AzureAIInferenceChatCompletion.__name__,
],
indirect=True,
)
@pytest.mark.parametrize(
"override_env_param_dict",
[
{
"SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS": "True",
"SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE": "False",
},
],
indirect=True,
)
@patch.object(ChatCompletionsClient, "complete", new_callable=AsyncMock)
@patch("azure.ai.inference.tracing.AIInferenceInstrumentor.uninstrument")
@patch("azure.ai.inference.tracing.AIInferenceInstrumentor.instrument")
async def test_azure_ai_inference_streaming_chat_completion_instrumentation_without_sensitive(
mock_instrument,
mock_uninstrument,
mock_complete,
azure_ai_inference_service,
chat_history: ChatHistory,
mock_azure_ai_inference_streaming_chat_completion_response,
model_diagnostics_test_env,
) -> None:
"""Test completion of AzureAIInferenceChatCompletion"""
settings = AzureAIInferenceChatPromptExecutionSettings()
mock_complete.return_value = mock_azure_ai_inference_streaming_chat_completion_response
async for _ in azure_ai_inference_service.get_streaming_chat_message_contents(
chat_history=chat_history, settings=settings
):
pass
mock_instrument.assert_called_once_with(enable_content_recording=False)
mock_uninstrument.assert_called_once()
@@ -0,0 +1,169 @@
# Copyright (c) Microsoft. All rights reserved.
import pytest
from azure.ai.inference.models import (
AssistantMessage,
ImageContentItem,
SystemMessage,
TextContentItem,
ToolMessage,
UserMessage,
)
from semantic_kernel.connectors.ai.azure_ai_inference.services.utils import MESSAGE_CONVERTERS
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.function_call_content import FunctionCallContent
from semantic_kernel.contents.function_result_content import FunctionResultContent
from semantic_kernel.contents.image_content import ImageContent
from semantic_kernel.contents.text_content import TextContent
from semantic_kernel.contents.utils.author_role import AuthorRole
def test_message_convertors_contain_all_author_roles() -> None:
"""Test that all AuthorRoles are present in the MESSAGE_CONVERTERS dict."""
for role in AuthorRole:
assert role in MESSAGE_CONVERTERS
def test_format_system_message() -> None:
"""Test that a system message is formatted correctly."""
message = ChatMessageContent(role=AuthorRole.SYSTEM, content="test content")
system_message = MESSAGE_CONVERTERS[message.role](message)
assert isinstance(system_message, SystemMessage)
assert system_message.content == message.content
def test_format_user_message_with_no_image() -> None:
"""Test that a user message with no image items is formatted correctly."""
message = ChatMessageContent(role=AuthorRole.USER, content="test content")
user_message = MESSAGE_CONVERTERS[message.role](message)
assert isinstance(user_message, UserMessage)
assert user_message.content == message.content
def test_format_user_message_with_image() -> None:
"""Test that a user message with image items is formatted correctly"""
message = ChatMessageContent(
role=AuthorRole.USER,
items=[
TextContent(text="test text"),
ImageContent(uri="https://test.com/image.jpg"),
],
)
user_message = MESSAGE_CONVERTERS[message.role](message)
assert isinstance(user_message, UserMessage)
assert len(user_message.content) == 2
assert isinstance(user_message.content[0], TextContentItem)
assert isinstance(user_message.content[1], ImageContentItem)
def test_format_user_message_with_unsupported_items() -> None:
"""Test that a user message with unsupported items is formatted correctly"""
message = ChatMessageContent(
role=AuthorRole.USER,
items=[
TextContent(text="test text"),
ImageContent(), # ImageContent without uri or data_uri is unsupported
FunctionCallContent(id="test function"), # FunctionCallContent unsupported
],
)
user_message = MESSAGE_CONVERTERS[message.role](message)
assert isinstance(user_message, UserMessage)
assert len(user_message.content) == 1
assert isinstance(user_message.content[0], TextContentItem)
def test_format_assistant_message() -> None:
"""Test that an assistant message is formatted correctly."""
message = ChatMessageContent(role=AuthorRole.ASSISTANT, content="test content")
assistant_message = MESSAGE_CONVERTERS[message.role](message)
assert isinstance(assistant_message, AssistantMessage)
assert assistant_message.content == message.content
def test_format_assistant_message_with_tool_call() -> None:
"""Test that an assistant message with a tool call is formatted correctly."""
function_call_content = FunctionCallContent(id="test function")
message = ChatMessageContent(
role=AuthorRole.ASSISTANT,
items=[function_call_content],
)
assistant_message = MESSAGE_CONVERTERS[message.role](message)
assert isinstance(assistant_message, AssistantMessage)
assert assistant_message.content == message.content
assert len(assistant_message.tool_calls) == 1
assert assistant_message.tool_calls[0].id == function_call_content.id
def test_format_assistant_message_with_unsupported_items() -> None:
"""Test that an assistant message with unsupported items is formatted correctly."""
text_content = TextContent(text="test text")
message = ChatMessageContent(
role=AuthorRole.ASSISTANT,
items=[
text_content,
ImageContent(), # ImageContent is unsupported
],
)
assistant_message = MESSAGE_CONVERTERS[message.role](message)
assert isinstance(assistant_message, AssistantMessage)
assert assistant_message.content == message.content
def test_format_tool_message() -> None:
"""Test that a tool message is formatted correctly."""
function_result_content = FunctionResultContent(id="test function", result="test result")
message = ChatMessageContent(
role=AuthorRole.TOOL,
items=[function_result_content],
)
tool_message = MESSAGE_CONVERTERS[message.role](message)
assert isinstance(tool_message, ToolMessage)
assert tool_message.content == function_result_content.result
assert tool_message.tool_call_id == function_result_content.id
def test_format_tool_message_item_not_found_as_the_first_item() -> None:
"""Test that formatting a tool message where the function result item is not the first item."""
function_result_content = FunctionResultContent(id="test function", result="test result")
message = ChatMessageContent(
role=AuthorRole.TOOL,
items=[
TextContent(text="test text"),
function_result_content,
],
)
with pytest.raises(ValueError):
MESSAGE_CONVERTERS[message.role](message)
def test_format_tool_message_with_more_than_one_items() -> None:
"""Test that a tool message with more than one item is formatted correctly."""
function_result_content = FunctionResultContent(id="test function", result="test result")
message = ChatMessageContent(
role=AuthorRole.TOOL,
items=[
function_result_content,
TextContent(text="test text"),
],
)
tool_message = MESSAGE_CONVERTERS[message.role](message)
assert isinstance(tool_message, ToolMessage)
assert tool_message.content == function_result_content.result
assert tool_message.tool_call_id == function_result_content.id
@@ -0,0 +1,143 @@
# Copyright (c) Microsoft. All rights reserved.
from semantic_kernel.connectors.ai.azure_ai_inference import (
AzureAIInferenceChatPromptExecutionSettings,
AzureAIInferenceEmbeddingPromptExecutionSettings,
AzureAIInferencePromptExecutionSettings,
)
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
def test_default_azure_ai_inference_prompt_execution_settings():
settings = AzureAIInferencePromptExecutionSettings()
assert settings.frequency_penalty is None
assert settings.max_tokens is None
assert settings.presence_penalty is None
assert settings.seed is None
assert settings.stop is None
assert settings.temperature is None
assert settings.top_p is None
assert settings.extra_parameters is None
def test_custom_azure_ai_inference_prompt_execution_settings():
settings = AzureAIInferencePromptExecutionSettings(
frequency_penalty=0.5,
max_tokens=128,
presence_penalty=0.5,
seed=1,
stop="world",
temperature=0.5,
top_p=0.5,
extra_parameters={"key": "value"},
)
assert settings.frequency_penalty == 0.5
assert settings.max_tokens == 128
assert settings.presence_penalty == 0.5
assert settings.seed == 1
assert settings.stop == "world"
assert settings.temperature == 0.5
assert settings.top_p == 0.5
assert settings.extra_parameters == {"key": "value"}
def test_azure_ai_inference_prompt_execution_settings_from_default_completion_config():
settings = PromptExecutionSettings(service_id="test_service")
chat_settings = AzureAIInferenceChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.service_id == "test_service"
assert chat_settings.frequency_penalty is None
assert chat_settings.max_tokens is None
assert chat_settings.presence_penalty is None
assert chat_settings.seed is None
assert chat_settings.stop is None
assert chat_settings.temperature is None
assert chat_settings.top_p is None
assert chat_settings.extra_parameters is None
def test_azure_ai_inference_prompt_execution_settings_from_openai_prompt_execution_settings():
chat_settings = AzureAIInferenceChatPromptExecutionSettings(service_id="test_service", temperature=1.0)
new_settings = AzureAIInferencePromptExecutionSettings(service_id="test_2", temperature=0.0)
chat_settings.update_from_prompt_execution_settings(new_settings)
assert chat_settings.service_id == "test_2"
assert chat_settings.temperature == 0.0
def test_azure_ai_inference_prompt_execution_settings_from_custom_completion_config():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"frequency_penalty": 0.5,
"max_tokens": 128,
"presence_penalty": 0.5,
"seed": 1,
"stop": "world",
"temperature": 0.5,
"top_p": 0.5,
"extra_parameters": {"key": "value"},
},
)
chat_settings = AzureAIInferenceChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.service_id == "test_service"
assert chat_settings.frequency_penalty == 0.5
assert chat_settings.max_tokens == 128
assert chat_settings.presence_penalty == 0.5
assert chat_settings.seed == 1
assert chat_settings.stop == "world"
assert chat_settings.temperature == 0.5
assert chat_settings.top_p == 0.5
assert chat_settings.extra_parameters == {"key": "value"}
def test_azure_ai_inference_chat_prompt_execution_settings_from_custom_completion_config_with_functions():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"tools": [{"function": {}}],
},
)
chat_settings = AzureAIInferenceChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.tools == [{"function": {}}]
def test_create_options():
settings = AzureAIInferenceChatPromptExecutionSettings(
service_id="test_service",
extension_data={
"frequency_penalty": 0.5,
"max_tokens": 128,
"presence_penalty": 0.5,
"seed": 1,
"stop": "world",
"temperature": 0.5,
"top_p": 0.5,
"extra_parameters": {"key": "value"},
},
)
options = settings.prepare_settings_dict()
assert options["frequency_penalty"] == 0.5
assert options["max_tokens"] == 128
assert options["presence_penalty"] == 0.5
assert options["seed"] == 1
assert options["stop"] == "world"
assert options["temperature"] == 0.5
assert options["top_p"] == 0.5
assert options["extra_parameters"] == {"key": "value"}
assert "tools" not in options
assert "tool_config" not in options
def test_default_azure_ai_inference_embedding_prompt_execution_settings():
settings = AzureAIInferenceEmbeddingPromptExecutionSettings()
assert settings.dimensions is None
assert settings.encoding_format is None
assert settings.input_type is None
assert settings.extra_parameters is None
@@ -0,0 +1,320 @@
# Copyright (c) Microsoft. All rights reserved.
import json
from unittest.mock import Mock
import pytest
from semantic_kernel.connectors.ai.bedrock.services.model_provider.bedrock_model_provider import BedrockModelProvider
from semantic_kernel.contents.chat_history import ChatHistory
@pytest.fixture()
def model_id(request) -> str:
if hasattr(request, "param"):
return request.param
return "test_model_id"
@pytest.fixture()
def service_id() -> str:
return "test_service_id"
@pytest.fixture()
def chat_history() -> ChatHistory:
chat_history = ChatHistory(system_message="You are a helpful assistant.")
chat_history.add_user_message("Hello!")
chat_history.add_assistant_message("Hi! How can I help you today?")
chat_history.add_system_message("Be polite and respectful.")
chat_history.add_user_message("I need help with a task.")
return chat_history
@pytest.fixture()
def bedrock_unit_test_env(monkeypatch, exclude_list, override_env_param_dict):
"""Fixture to set environment variables for Amazon Bedrock AI connector unit tests."""
if exclude_list is None:
exclude_list = []
if override_env_param_dict is None:
override_env_param_dict = {}
env_vars = {
"BEDROCK_TEXT_MODEL_ID": "env_test_text_model_id",
"BEDROCK_CHAT_MODEL_ID": "env_test_chat_model_id",
"BEDROCK_EMBEDDING_MODEL_ID": "env_test_embedding_model_id",
"BEDROCK_MODEL_PROVIDER": "amazon",
}
env_vars.update(override_env_param_dict)
for key, value in env_vars.items():
if key not in exclude_list:
monkeypatch.setenv(key, value)
else:
monkeypatch.delenv(key, raising=False)
return env_vars
class MockBedrockClient(Mock):
def __init__(self, *args, **kwargs):
pass
def get_foundation_model(self, *args, **kwargs):
return {
"modelDetails": {
"responseStreamingSupported": True,
"inputModalities": ["TEXT"],
"outputModalities": ["TEXT", "EMBEDDING"],
}
}
class MockBedrockRuntimeClient(Mock):
def __init__(self, *args, **kwargs):
pass
def converse(self, *args, **kwargs):
pass
def converse_stream(self, *args, **kwargs):
pass
def invoke_model(self, *args, **kwargs):
pass
def invoke_model_with_response_stream(self, *args, **kwargs):
pass
# region mock chat completion responses
@pytest.fixture()
def mock_bedrock_chat_completion_response():
# https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-call.html#conversation-inference-call-response
return {
"output": {
"message": {
"role": "assistant",
"content": [
{
"text": "Hi! How can I help you today?",
}
],
}
},
"stopReason": "end_turn",
"usage": {
"inputTokens": 125,
"outputTokens": 60,
"totalTokens": 185,
},
}
@pytest.fixture()
def mock_bedrock_streaming_chat_completion_response():
# https://docs.aws.amazon.com/bedrock/latest/userguide/conversation-inference-call.html#conversation-inference-call-response
events = [
{"messageStart": {"role": "assistant"}},
{"contentBlockStart": {"contentBlockIndex": 0, "start": {}}},
{"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "Hi! "}}},
{"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "How can "}}},
{"contentBlockDelta": {"contentBlockIndex": 0, "delta": {"text": "I help you today?"}}},
{"contentBlockStop": {"contentBlockIndex": 0}},
{"messageStop": {"stopReason": "end_turn"}},
{
"metadata": {
"metrics": {"latencyMs": 1000},
"usage": {"inputTokens": 125, "outputTokens": 60, "totalTokens": 185},
}
},
]
def event_stream(events):
yield from events
return {"stream": event_stream(events)}
@pytest.fixture()
def mock_bedrock_streaming_chat_completion_invalid_response():
events = [
{"unknown": {}},
]
def event_stream(events):
yield from events
return {"stream": event_stream(events)}
# endregion
# region mock text completion responses
@pytest.fixture()
def output_text():
return "Hi! How can I help you today?"
@pytest.fixture()
def model_provider():
return BedrockModelProvider.AMAZON
@pytest.fixture()
def mock_bedrock_text_completion_response(
model_id: str,
output_text: str,
request,
):
# Check if model_provider fixture is requested by the test
model_provider = None
if "model_provider" in request.fixturenames:
model_provider = request.getfixturevalue("model_provider")
else:
model_provider = BedrockModelProvider.to_model_provider(model_id)
match model_provider:
case BedrockModelProvider.AMAZON:
body = {
"inputTextTokenCount": 10,
"results": [
{
"tokenCount": 10,
"outputText": output_text,
"completionReason": "FINISHED ",
}
],
}
case BedrockModelProvider.ANTHROPIC:
body = {
"completion": output_text,
"stop_reason": "stop_sequence",
"stop": "",
}
case BedrockModelProvider.COHERE:
body = {
"generations": [
{
"text": output_text,
}
],
}
case BedrockModelProvider.AI21LABS:
body = {
"completions": [
{
"data": {
"text": output_text,
}
}
],
}
case BedrockModelProvider.META:
body = {
"generation": output_text,
"prompt_token_count": 10,
"generation_token_count": 10,
}
case BedrockModelProvider.MISTRALAI:
body = {"outputs": [{"text": output_text}]}
mock = Mock()
mock.read.return_value = json.dumps(body)
return {"body": mock}
@pytest.fixture()
def mock_bedrock_streaming_text_completion_response(
model_id: str,
output_text: str,
request,
):
# Check if model_provider fixture is requested by the test
model_provider = None
if "model_provider" in request.fixturenames:
model_provider = request.getfixturevalue("model_provider")
else:
model_provider = BedrockModelProvider.to_model_provider(model_id)
match model_provider:
case BedrockModelProvider.AMAZON:
chunks = [
{
"chunk": {
"bytes": json.dumps({
"inputTextTokenCount": 10,
"totalOutputTextTokenCount": 10,
"outputText": chunk,
}).encode(),
}
}
for chunk in [output_text[i : i + 3] for i in range(0, len(output_text), 3)]
]
def event_stream(events):
yield from events
return {"body": event_stream(chunks)}
# endregion
# region mock text embedding responses
@pytest.fixture()
def mock_bedrock_text_embedding_response(
model_id: str,
request,
):
# Check if model_provider fixture is requested by the test
model_provider = None
if "model_provider" in request.fixturenames:
model_provider = request.getfixturevalue("model_provider")
else:
model_provider = BedrockModelProvider.to_model_provider(model_id)
match model_provider:
case BedrockModelProvider.AMAZON:
body = {
"embedding": [0.1, 0.2, 0.3],
}
case BedrockModelProvider.COHERE:
body = {
"embeddings": [[0.1, 0.2, 0.3]],
}
mock = Mock()
mock.read.return_value = json.dumps(body)
return {"body": mock}
@pytest.fixture()
def mock_bedrock_text_embedding_invalid_response(model_id: str):
model_provider = BedrockModelProvider.to_model_provider(model_id)
match model_provider:
case BedrockModelProvider.AMAZON:
body = {"embedding": 0.1}
case BedrockModelProvider.COHERE:
body = {"embeddings": 0.1}
mock = Mock()
mock.read.return_value = json.dumps(body)
return {"body": mock}
# endregion
@@ -0,0 +1,371 @@
# Copyright (c) Microsoft. All rights reserved.
from functools import reduce
from unittest.mock import Mock, patch
import boto3
import pytest
from semantic_kernel.connectors.ai.bedrock.bedrock_prompt_execution_settings import BedrockChatPromptExecutionSettings
from semantic_kernel.connectors.ai.bedrock.services.bedrock_chat_completion import BedrockChatCompletion
from semantic_kernel.connectors.ai.bedrock.services.model_provider.bedrock_model_provider import BedrockModelProvider
from semantic_kernel.connectors.ai.completion_usage import CompletionUsage
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from semantic_kernel.contents.text_content import TextContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.contents.utils.finish_reason import FinishReason
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError, ServiceInvalidResponseError
from tests.unit.connectors.ai.bedrock.conftest import MockBedrockClient, MockBedrockRuntimeClient
# region init
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_chat_completion_init(mock_client, bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Chat Completion service"""
bedrock_chat_completion = BedrockChatCompletion()
assert bedrock_chat_completion.ai_model_id == bedrock_unit_test_env["BEDROCK_CHAT_MODEL_ID"]
assert bedrock_chat_completion.service_id == bedrock_unit_test_env["BEDROCK_CHAT_MODEL_ID"]
assert bedrock_chat_completion.bedrock_model_provider == BedrockModelProvider(
bedrock_unit_test_env["BEDROCK_MODEL_PROVIDER"]
)
assert bedrock_chat_completion.bedrock_client is not None
assert bedrock_chat_completion.bedrock_runtime_client is not None
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_chat_completion_init_model_id_override(mock_client, bedrock_unit_test_env, model_id) -> None:
"""Test initialization of Amazon Bedrock Chat Completion service"""
bedrock_chat_completion = BedrockChatCompletion(model_id=model_id)
assert bedrock_chat_completion.ai_model_id == model_id
assert bedrock_chat_completion.service_id == model_id
assert bedrock_chat_completion.bedrock_client is not None
assert bedrock_chat_completion.bedrock_runtime_client is not None
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_chat_completion_init_custom_service_id(mock_client, bedrock_unit_test_env, service_id) -> None:
"""Test initialization of Amazon Bedrock Chat Completion service"""
bedrock_chat_completion = BedrockChatCompletion(service_id=service_id)
assert bedrock_chat_completion.service_id == service_id
assert bedrock_chat_completion.bedrock_client is not None
assert bedrock_chat_completion.bedrock_runtime_client is not None
def test_bedrock_chat_completion_init_custom_clients(bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Chat Completion service"""
bedrock_chat_completion = BedrockChatCompletion(
runtime_client=MockBedrockRuntimeClient(),
client=MockBedrockClient(),
)
assert isinstance(bedrock_chat_completion.bedrock_client, MockBedrockClient)
assert isinstance(bedrock_chat_completion.bedrock_runtime_client, MockBedrockRuntimeClient)
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_chat_completion_init_custom_client(mock_client, bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Chat Completion service"""
bedrock_chat_completion = BedrockChatCompletion(
client=MockBedrockClient(),
)
assert isinstance(bedrock_chat_completion.bedrock_client, MockBedrockClient)
assert bedrock_chat_completion.bedrock_runtime_client is not None
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_chat_completion_init_custom_runtime_client(mock_client, bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Chat Completion service"""
bedrock_chat_completion = BedrockChatCompletion(
runtime_client=MockBedrockRuntimeClient(),
)
assert bedrock_chat_completion.bedrock_client is not None
assert isinstance(bedrock_chat_completion.bedrock_runtime_client, MockBedrockRuntimeClient)
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_chat_completion_init_custom_bedrock_model_provider(mock_client, bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Chat Completion service"""
bedrock_chat_completion = BedrockChatCompletion(
model_provider=BedrockModelProvider.AMAZON,
)
assert bedrock_chat_completion.bedrock_model_provider == BedrockModelProvider.AMAZON
@pytest.mark.parametrize("exclude_list", [["BEDROCK_CHAT_MODEL_ID"]], indirect=True)
def test_bedrock_chat_completion_client_init_with_empty_model_id(bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Chat Completion service with empty model id"""
with pytest.raises(ServiceInitializationError, match="The Amazon Bedrock Chat Model ID is missing."):
BedrockChatCompletion(env_file_path="fake_env_file_path.env")
def test_bedrock_chat_completion_client_init_invalid_settings(bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Chat Completion service with invalid settings"""
with pytest.raises(
ServiceInitializationError, match="Failed to initialize the Amazon Bedrock Chat Completion Service."
):
BedrockChatCompletion(model_id=123) # Model ID must be a string
def test_bedrock_chat_completion_client_init_invalid_model_provider(bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Chat Completion service with invalid settings"""
with pytest.raises(
ServiceInitializationError, match="Failed to initialize the Amazon Bedrock Chat Completion Service."
):
BedrockChatCompletion(model_provider="invalid_provider")
@patch.object(boto3, "client", return_value=Mock())
def test_prompt_execution_settings_class(mock_client, bedrock_unit_test_env) -> None:
"""Test getting prompt execution settings class"""
bedrock_completion_client = BedrockChatCompletion()
assert bedrock_completion_client.get_prompt_execution_settings_class() == BedrockChatPromptExecutionSettings
# endregion
# region private methods
@patch.object(boto3, "client", return_value=Mock())
def test_prepare_chat_history_for_request(mock_client, bedrock_unit_test_env, chat_history) -> None:
"""Test preparing chat history for request"""
bedrock_chat_completion = BedrockChatCompletion()
parsed_chat_history = bedrock_chat_completion._prepare_chat_history_for_request(chat_history)
assert isinstance(parsed_chat_history, list)
assert len(parsed_chat_history) == len(chat_history) - 2 # Exclude system message
assert all([item["role"] in ["user", "assistant"] for item in parsed_chat_history])
@patch.object(boto3, "client", return_value=Mock())
def test_prepare_system_message_for_request(mock_client, bedrock_unit_test_env, chat_history) -> None:
"""Test preparing system message for request"""
bedrock_chat_completion = BedrockChatCompletion()
parsed_system_message = bedrock_chat_completion._prepare_system_messages_for_request(chat_history)
assert isinstance(parsed_system_message, list)
assert len(parsed_system_message) == 2
@pytest.mark.parametrize(
"model_id",
[
"amazon.titan",
"anthropic.claude",
"cohere.command",
"ai21.jamba",
"meta.llama",
"mistral.ai",
],
)
@patch.object(boto3, "client", return_value=Mock())
def test_prepare_settings_for_request(mock_client, model_id, chat_history) -> None:
"""Test preparing settings for request"""
bedrock_chat_completion = BedrockChatCompletion(model_id=model_id)
settings = BedrockChatPromptExecutionSettings()
parsed_settings = bedrock_chat_completion._prepare_settings_for_request(chat_history, settings)
assert isinstance(parsed_settings, dict)
assert parsed_settings["modelId"] == bedrock_chat_completion.ai_model_id
assert parsed_settings["messages"] == bedrock_chat_completion._prepare_chat_history_for_request(chat_history)
assert parsed_settings["system"] == bedrock_chat_completion._prepare_system_messages_for_request(chat_history)
assert isinstance(parsed_settings["inferenceConfig"], dict)
assert all([parsed_settings["inferenceConfig"].values()])
@pytest.mark.parametrize(
"model_id",
[
"arn:aws:bedrock:us-east-1:972143716085:application-inference-profile/123456",
],
)
@patch.object(boto3, "client", return_value=Mock())
def test_prepare_settings_for_request_with_application_inference_profile(mock_client, model_id, chat_history) -> None:
"""Test preparing settings for request"""
# Without a valid model provider, it should raise an error
bedrock_chat_completion = BedrockChatCompletion(model_id=model_id)
settings = BedrockChatPromptExecutionSettings()
with pytest.raises(
ValueError,
match=f"Model ID {model_id} does not contain a valid model provider name.",
):
bedrock_chat_completion._prepare_settings_for_request(chat_history, settings)
# With a valid model provider, it should not raise an error
bedrock_chat_completion = BedrockChatCompletion(model_id=model_id, model_provider=BedrockModelProvider.AMAZON)
parsed_settings = bedrock_chat_completion._prepare_settings_for_request(chat_history, settings)
assert isinstance(parsed_settings, dict)
assert parsed_settings["modelId"] == bedrock_chat_completion.ai_model_id
assert parsed_settings["messages"] == bedrock_chat_completion._prepare_chat_history_for_request(chat_history)
assert parsed_settings["system"] == bedrock_chat_completion._prepare_system_messages_for_request(chat_history)
assert isinstance(parsed_settings["inferenceConfig"], dict)
assert all([parsed_settings["inferenceConfig"].values()])
# endregion
# region chat completion
@pytest.mark.parametrize(
# These are fake model ids with the supported prefixes
"model_id",
[
"amazon.titan",
"anthropic.claude",
"cohere.command",
"ai21.jamba",
"meta.llama",
"mistral.ai",
],
)
async def test_bedrock_chat_completion(
model_id,
chat_history: ChatHistory,
mock_bedrock_chat_completion_response,
) -> None:
"""Test Amazon Bedrock Chat Completion complete method"""
with patch.object(
MockBedrockRuntimeClient, "converse", return_value=mock_bedrock_chat_completion_response
) as mock_converse:
# Setup
bedrock_chat_completion = BedrockChatCompletion(
model_id=model_id,
runtime_client=MockBedrockRuntimeClient(),
client=MockBedrockClient(),
)
# Act
settings = BedrockChatPromptExecutionSettings()
response = await bedrock_chat_completion.get_chat_message_contents(chat_history=chat_history, settings=settings)
# Assert
mock_converse.assert_called_once_with(
**(bedrock_chat_completion._prepare_settings_for_request(chat_history, settings))
)
assert isinstance(response, list)
assert len(response) == 1
assert isinstance(response[0], ChatMessageContent)
assert response[0].ai_model_id == model_id
assert response[0].role == AuthorRole.ASSISTANT
assert len(response[0].items) == 1
assert isinstance(response[0].items[0], TextContent)
assert response[0].finish_reason == FinishReason.STOP
assert response[0].metadata["usage"] == CompletionUsage(
prompt_tokens=mock_bedrock_chat_completion_response["usage"]["inputTokens"],
completion_tokens=mock_bedrock_chat_completion_response["usage"]["outputTokens"],
)
assert (
response[0].items[0].text
== mock_bedrock_chat_completion_response["output"]["message"]["content"][0]["text"]
)
assert response[0].inner_content == mock_bedrock_chat_completion_response
@pytest.mark.parametrize(
# These are fake model ids with the supported prefixes
"model_id",
[
"amazon.titan",
"anthropic.claude",
"cohere.command",
"ai21.jamba",
"meta.llama",
"mistral.ai",
],
)
async def test_bedrock_streaming_chat_completion(
model_id,
chat_history: ChatHistory,
mock_bedrock_streaming_chat_completion_response,
) -> None:
"""Test Amazon Bedrock Streaming Chat Completion complete method"""
with patch.object(
MockBedrockRuntimeClient, "converse_stream", return_value=mock_bedrock_streaming_chat_completion_response
) as mock_converse_stream:
# Setup
bedrock_chat_completion = BedrockChatCompletion(
model_id=model_id,
runtime_client=MockBedrockRuntimeClient(),
client=MockBedrockClient(),
)
# Act
settings = BedrockChatPromptExecutionSettings()
chunks: list[StreamingChatMessageContent] = []
async for streaming_messages in bedrock_chat_completion.get_streaming_chat_message_contents(
chat_history=chat_history, settings=settings
):
chunks.extend(streaming_messages)
response = reduce(lambda p, r: p + r, chunks)
# Assert
mock_converse_stream.assert_called_once_with(
**(bedrock_chat_completion._prepare_settings_for_request(chat_history, settings))
)
assert isinstance(response, StreamingChatMessageContent)
assert response.ai_model_id == model_id
assert response.role == AuthorRole.ASSISTANT
assert len(response.items) == 1
assert isinstance(response.inner_content, list)
assert len(response.inner_content) == 7
assert response.finish_reason == FinishReason.STOP
@pytest.mark.parametrize(
# These are fake model ids with the supported prefixes
"model_id",
[
"amazon.titan",
"anthropic.claude",
"cohere.command",
"ai21.jamba",
"meta.llama",
"mistral.ai",
],
)
async def test_bedrock_streaming_chat_completion_invalid_event(
model_id,
chat_history: ChatHistory,
mock_bedrock_streaming_chat_completion_invalid_response,
) -> None:
"""Test Amazon Bedrock Streaming Chat Completion complete method"""
with patch.object(
MockBedrockRuntimeClient,
"converse_stream",
return_value=mock_bedrock_streaming_chat_completion_invalid_response,
):
# Setup
bedrock_chat_completion = BedrockChatCompletion(
model_id=model_id,
runtime_client=MockBedrockRuntimeClient(),
client=MockBedrockClient(),
)
# Act
settings = BedrockChatPromptExecutionSettings()
with pytest.raises(ServiceInvalidResponseError):
async for chunk in bedrock_chat_completion.get_streaming_chat_message_contents(
chat_history=chat_history, settings=settings
):
pass
# endregion
@@ -0,0 +1,443 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import MagicMock
import pytest
from semantic_kernel.connectors.ai.bedrock.bedrock_prompt_execution_settings import BedrockChatPromptExecutionSettings
from semantic_kernel.connectors.ai.bedrock.services.model_provider.bedrock_model_provider import (
BedrockModelProvider,
)
from semantic_kernel.connectors.ai.bedrock.services.model_provider.utils import (
MESSAGE_CONVERTERS,
finish_reason_from_bedrock_to_semantic_kernel,
remove_none_recursively,
update_settings_from_function_choice_configuration,
)
from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
from semantic_kernel.connectors.ai.function_choice_type import FunctionChoiceType
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.function_call_content import FunctionCallContent
from semantic_kernel.contents.function_result_content import FunctionResultContent
from semantic_kernel.contents.image_content import ImageContent
from semantic_kernel.contents.text_content import TextContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.contents.utils.finish_reason import FinishReason
from semantic_kernel.exceptions.service_exceptions import ServiceInvalidRequestError
from semantic_kernel.kernel import Kernel
def test_remove_none_recursively():
data = {
"a": 1,
"b": None,
"c": {
"d": 2,
"e": None,
"f": {
"g": 3,
"h": None,
},
},
}
expected = {
"a": 1,
"c": {
"d": 2,
"f": {
"g": 3,
},
},
}
assert remove_none_recursively(data) == expected
def test_remove_recursively_max_depth():
data = {
"a": {"b": None},
}
assert remove_none_recursively(data, max_depth=1) == data
def test_update_settings_from_function_choice_configuration_auto(kernel: Kernel, custom_plugin_class) -> None:
kernel.add_plugin(plugin=custom_plugin_class(), plugin_name="custom_plugin")
settings = BedrockChatPromptExecutionSettings()
auto_function_choice_behavior = FunctionChoiceBehavior.Auto()
auto_function_choice_behavior.configure(
kernel,
update_settings_from_function_choice_configuration,
settings,
)
assert "auto" in settings.tool_choice
assert len(settings.tools) == 1
def test_update_settings_from_function_choice_configuration_auto_without_plugin(kernel: Kernel) -> None:
settings = BedrockChatPromptExecutionSettings()
auto_function_choice_behavior = FunctionChoiceBehavior.Auto()
auto_function_choice_behavior.configure(
kernel,
update_settings_from_function_choice_configuration,
settings,
)
assert settings.tool_choice is None
assert settings.tools is None
def test_update_settings_from_function_choice_configuration_none(kernel: Kernel) -> None:
settings = BedrockChatPromptExecutionSettings()
auto_function_choice_behavior = FunctionChoiceBehavior.NoneInvoke()
auto_function_choice_behavior.configure(
kernel,
update_settings_from_function_choice_configuration,
settings,
)
assert settings.tool_choice is None
assert settings.tools is None
def test_update_settings_from_function_choice_configuration_required_with_one_function(
kernel: Kernel,
custom_plugin_class,
) -> None:
kernel.add_plugin(plugin=custom_plugin_class(), plugin_name="custom_plugin")
settings = BedrockChatPromptExecutionSettings()
auto_function_choice_behavior = FunctionChoiceBehavior.Required()
auto_function_choice_behavior.configure(
kernel,
update_settings_from_function_choice_configuration,
settings,
)
assert "tool" in settings.tool_choice
assert len(settings.tools) == 1
def test_update_settings_from_function_choice_configuration_required_with_more_than_one_functions(
kernel: Kernel,
custom_plugin_class,
experimental_plugin_class,
) -> None:
kernel.add_plugin(plugin=custom_plugin_class(), plugin_name="custom_plugin")
kernel.add_plugin(plugin=experimental_plugin_class(), plugin_name="experimental_plugin")
settings = BedrockChatPromptExecutionSettings()
auto_function_choice_behavior = FunctionChoiceBehavior.Required()
auto_function_choice_behavior.configure(
kernel,
update_settings_from_function_choice_configuration,
settings,
)
assert "any" in settings.tool_choice
assert len(settings.tools) == 2
def test_inference_profile_with_bedrock_model() -> None:
"""Test the BedrockModelProvider class returns the correct model for a given inference profile."""
us_amazon_inference_profile = "us.amazon.nova-lite-v1:0"
assert BedrockModelProvider.to_model_provider(us_amazon_inference_profile) == BedrockModelProvider.AMAZON
us_anthropic_inference_profile = "us.anthropic.claude-3-sonnet-20240229-v1:0"
assert BedrockModelProvider.to_model_provider(us_anthropic_inference_profile) == BedrockModelProvider.ANTHROPIC
eu_meta_inference_profile = "eu.meta.llama3-2-3b-instruct-v1:0"
assert BedrockModelProvider.to_model_provider(eu_meta_inference_profile) == BedrockModelProvider.META
unknown_inference_profile = "unknown"
with pytest.raises(ValueError, match="Model ID unknown does not contain a valid model provider name."):
BedrockModelProvider.to_model_provider(unknown_inference_profile)
def test_remove_none_recursively_empty_dict() -> None:
"""Test that an empty dict returns an empty dict."""
assert remove_none_recursively({}) == {}
def test_remove_none_recursively_no_none() -> None:
"""Test that a dict with no None values remains the same."""
original = {"a": 1, "b": 2}
result = remove_none_recursively(original)
assert result == {"a": 1, "b": 2}
def test_remove_none_recursively_with_none() -> None:
"""Test that dict values of None are removed."""
original = {"a": 1, "b": None, "c": {"d": None, "e": 3}}
result = remove_none_recursively(original)
# 'b' should be removed and 'd' inside nested dict should be removed
assert result == {"a": 1, "c": {"e": 3}}
def test_remove_none_recursively_max_depth() -> None:
"""Test that the function respects max_depth."""
original = {"a": {"b": {"c": None}}}
# If max_depth=1, it won't go deep enough to remove 'c'.
result = remove_none_recursively(original, max_depth=1)
assert result == {"a": {"b": {"c": None}}}
# If max_depth=3, it should remove 'c'.
result = remove_none_recursively(original, max_depth=3)
assert result == {"a": {"b": {}}}
def test_format_system_message() -> None:
"""Test that system message is formatted correctly."""
content = ChatMessageContent(role=AuthorRole.SYSTEM, content="System message")
formatted = MESSAGE_CONVERTERS[AuthorRole.SYSTEM](content)
assert formatted == {"text": "System message"}
def test_format_user_message_text_only() -> None:
"""Test user message with only text content."""
text_item = TextContent(text="Hello!")
user_message = ChatMessageContent(role=AuthorRole.USER, items=[text_item])
formatted = MESSAGE_CONVERTERS[AuthorRole.USER](user_message)
assert formatted["role"] == "user"
assert len(formatted["content"]) == 1
assert formatted["content"][0] == {"text": "Hello!"}
def test_format_user_message_image_only() -> None:
"""Test user message with only image content."""
img_item = ImageContent(data=b"abc", mime_type="image/png")
user_message = ChatMessageContent(role=AuthorRole.USER, items=[img_item])
formatted = MESSAGE_CONVERTERS[AuthorRole.USER](user_message)
assert formatted["role"] == "user"
assert len(formatted["content"]) == 1
image_section = formatted["content"][0].get("image")
assert image_section["format"] == "png"
assert image_section["source"]["bytes"] == b"abc"
def test_format_user_message_unsupported_content() -> None:
"""Test user message raises error with unsupported content type."""
# We can simulate an unsupported content type by using FunctionCallContent.
func_call_item = FunctionCallContent(id="123", function_name="test_function", arguments="{}")
user_message = ChatMessageContent(role=AuthorRole.USER, items=[func_call_item])
with pytest.raises(ServiceInvalidRequestError) as exc:
MESSAGE_CONVERTERS[AuthorRole.USER](user_message)
assert "Only text and image content are supported in a user message." in str(exc.value)
def test_format_assistant_message_text_content() -> None:
"""Test assistant message with text content."""
text_item = TextContent(text="Assistant response")
assistant_message = ChatMessageContent(role=AuthorRole.ASSISTANT, items=[text_item])
formatted = MESSAGE_CONVERTERS[AuthorRole.ASSISTANT](assistant_message)
assert formatted["role"] == "assistant"
assert formatted["content"] == [{"text": "Assistant response"}]
def test_format_assistant_message_function_call_content() -> None:
"""Test assistant message with function call content."""
func_item = FunctionCallContent(
id="fc1", plugin_name="plugin", function_name="function", arguments='{"param": "value"}'
)
assistant_message = ChatMessageContent(role=AuthorRole.ASSISTANT, items=[func_item])
formatted = MESSAGE_CONVERTERS[AuthorRole.ASSISTANT](assistant_message)
assert len(formatted["content"]) == 1
tool_use = formatted["content"][0].get("toolUse")
assert tool_use
assert tool_use["toolUseId"] == "fc1"
assert tool_use["name"] == "plugin-function"
assert tool_use["input"] == {"param": "value"}
def test_format_assistant_message_image_content_raises() -> None:
"""Test assistant message with image raises error."""
img_item = ImageContent(data=b"abc", mime_type="image/jpeg")
assistant_message = ChatMessageContent(role=AuthorRole.ASSISTANT, items=[img_item])
with pytest.raises(ServiceInvalidRequestError) as exc:
MESSAGE_CONVERTERS[AuthorRole.ASSISTANT](assistant_message)
assert "Image content is not supported in an assistant message." in str(exc.value)
def test_format_assistant_message_unsupported_type() -> None:
"""Test assistant message with unsupported item content type."""
func_res_item = FunctionResultContent(id="res1", function_name="some_function", result="some_result")
assistant_message = ChatMessageContent(role=AuthorRole.ASSISTANT, items=[func_res_item])
with pytest.raises(ServiceInvalidRequestError) as exc:
MESSAGE_CONVERTERS[AuthorRole.ASSISTANT](assistant_message)
assert "Unsupported content type in an assistant message:" in str(exc.value)
def test_format_tool_message_text() -> None:
"""Test tool message with text content."""
text_item = TextContent(text="Some text")
tool_message = ChatMessageContent(role=AuthorRole.TOOL, items=[text_item])
formatted = MESSAGE_CONVERTERS[AuthorRole.TOOL](tool_message)
assert formatted["role"] == "user" # note that for a tool message, role set to 'user'
assert formatted["content"] == [{"text": "Some text"}]
def test_format_tool_message_function_result() -> None:
"""Test tool message with function result content."""
func_result_item = FunctionResultContent(id="res_id", function_name="test_function", result="some result")
tool_message = ChatMessageContent(role=AuthorRole.TOOL, items=[func_result_item])
formatted = MESSAGE_CONVERTERS[AuthorRole.TOOL](tool_message)
assert formatted["role"] == "user"
content = formatted["content"][0]
assert content.get("toolResult")
assert content["toolResult"]["toolUseId"] == "res_id"
assert content["toolResult"]["content"] == [{"text": "some result"}]
def test_format_tool_message_image_raises() -> None:
"""Test tool message with image content raises an error."""
img_item = ImageContent(data=b"xyz", mime_type="image/jpeg")
tool_message = ChatMessageContent(role=AuthorRole.TOOL, items=[img_item])
with pytest.raises(ServiceInvalidRequestError) as exc:
MESSAGE_CONVERTERS[AuthorRole.TOOL](tool_message)
assert "Image content is not supported in a tool message." in str(exc.value)
def test_finish_reason_from_bedrock_to_semantic_kernel_stop() -> None:
"""Test that 'stop_sequence' maps to FinishReason.STOP"""
reason = finish_reason_from_bedrock_to_semantic_kernel("stop_sequence")
assert reason == FinishReason.STOP
reason = finish_reason_from_bedrock_to_semantic_kernel("end_turn")
assert reason == FinishReason.STOP
def test_finish_reason_from_bedrock_to_semantic_kernel_length() -> None:
"""Test that 'max_tokens' maps to FinishReason.LENGTH"""
reason = finish_reason_from_bedrock_to_semantic_kernel("max_tokens")
assert reason == FinishReason.LENGTH
def test_finish_reason_from_bedrock_to_semantic_kernel_content_filtered() -> None:
"""Test that 'content_filtered' maps to FinishReason.CONTENT_FILTER"""
reason = finish_reason_from_bedrock_to_semantic_kernel("content_filtered")
assert reason == FinishReason.CONTENT_FILTER
def test_finish_reason_from_bedrock_to_semantic_kernel_tool_use() -> None:
"""Test that 'tool_use' maps to FinishReason.TOOL_CALLS"""
reason = finish_reason_from_bedrock_to_semantic_kernel("tool_use")
assert reason == FinishReason.TOOL_CALLS
def test_finish_reason_from_bedrock_to_semantic_kernel_unknown() -> None:
"""Test that unknown finish reason returns None"""
reason = finish_reason_from_bedrock_to_semantic_kernel("something_unknown")
assert reason is None
@pytest.fixture
def mock_bedrock_settings() -> BedrockChatPromptExecutionSettings:
"""Helper fixture for BedrockChatPromptExecutionSettings."""
return BedrockChatPromptExecutionSettings()
@pytest.fixture
def mock_function_choice_config() -> FunctionCallChoiceConfiguration:
"""Helper fixture for a sample FunctionCallChoiceConfiguration."""
# We'll create mock kernel functions with metadata
mock_func_1 = MagicMock()
mock_func_1.fully_qualified_name = "plugin-function1"
mock_func_1.description = "Function 1 description"
param1 = MagicMock()
param1.name = "param1"
param1.schema_data = {"type": "string"}
param1.is_required = True
param2 = MagicMock()
param2.name = "param2"
param2.schema_data = {"type": "integer"}
param2.is_required = False
mock_func_1.parameters = [
param1,
param2,
]
mock_func_2 = MagicMock()
mock_func_2.fully_qualified_name = "plugin-function2"
mock_func_2.description = "Function 2 description"
mock_func_2.parameters = []
config = FunctionCallChoiceConfiguration()
config.available_functions = [mock_func_1, mock_func_2]
return config
def test_update_settings_from_function_choice_configuration_none_type(
mock_function_choice_config, mock_bedrock_settings
) -> None:
"""Test that if the FunctionChoiceType is NONE it doesn't modify settings."""
update_settings_from_function_choice_configuration(
mock_function_choice_config, mock_bedrock_settings, FunctionChoiceType.NONE
)
assert mock_bedrock_settings.tool_choice is None
assert mock_bedrock_settings.tools is None
def test_update_settings_from_function_choice_configuration_auto_two_tools(
mock_function_choice_config, mock_bedrock_settings
) -> None:
"""Test that AUTO sets tool_choice to {"auto": {}} and sets tools list"""
update_settings_from_function_choice_configuration(
mock_function_choice_config, mock_bedrock_settings, FunctionChoiceType.AUTO
)
assert mock_bedrock_settings.tool_choice == {"auto": {}}
assert len(mock_bedrock_settings.tools) == 2
# Validate structure of first tool
tool_spec_1 = mock_bedrock_settings.tools[0].get("toolSpec")
assert tool_spec_1["name"] == "plugin-function1"
assert tool_spec_1["description"] == "Function 1 description"
def test_update_settings_from_function_choice_configuration_required_many(
mock_function_choice_config, mock_bedrock_settings
) -> None:
"""Test that REQUIRED with more than one function sets tool_choice to {"any": {}}."""
update_settings_from_function_choice_configuration(
mock_function_choice_config, mock_bedrock_settings, FunctionChoiceType.REQUIRED
)
assert mock_bedrock_settings.tool_choice == {"any": {}}
assert len(mock_bedrock_settings.tools) == 2
def test_update_settings_from_function_choice_configuration_required_one(mock_bedrock_settings) -> None:
"""Test that REQUIRED with a single function picks "tool" with that function name."""
single_func = MagicMock()
single_func.fully_qualified_name = "plugin-function"
single_func.description = "Only function"
single_func.parameters = []
config = FunctionCallChoiceConfiguration()
config.available_functions = [single_func]
update_settings_from_function_choice_configuration(config, mock_bedrock_settings, FunctionChoiceType.REQUIRED)
assert mock_bedrock_settings.tool_choice == {"tool": {"name": "plugin-function"}}
assert len(mock_bedrock_settings.tools) == 1
assert mock_bedrock_settings.tools[0]["toolSpec"]["name"] == "plugin-function"
@@ -0,0 +1,324 @@
# Copyright (c) Microsoft. All rights reserved.
import json
from functools import reduce
from unittest.mock import Mock, patch
import boto3
import pytest
from semantic_kernel.connectors.ai.bedrock.bedrock_prompt_execution_settings import BedrockTextPromptExecutionSettings
from semantic_kernel.connectors.ai.bedrock.services.bedrock_text_completion import BedrockTextCompletion
from semantic_kernel.connectors.ai.bedrock.services.model_provider.bedrock_model_provider import (
BedrockModelProvider,
get_text_completion_request_body,
)
from semantic_kernel.contents.streaming_text_content import StreamingTextContent
from semantic_kernel.contents.text_content import TextContent
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
from tests.unit.connectors.ai.bedrock.conftest import MockBedrockClient, MockBedrockRuntimeClient
# region init
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_text_completion_init(mock_client, bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Completion service"""
bedrock_text_completion = BedrockTextCompletion()
assert bedrock_text_completion.ai_model_id == bedrock_unit_test_env["BEDROCK_TEXT_MODEL_ID"]
assert bedrock_text_completion.service_id == bedrock_unit_test_env["BEDROCK_TEXT_MODEL_ID"]
assert bedrock_text_completion.bedrock_model_provider == BedrockModelProvider(
bedrock_unit_test_env["BEDROCK_MODEL_PROVIDER"]
)
assert bedrock_text_completion.bedrock_client is not None
assert bedrock_text_completion.bedrock_runtime_client is not None
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_text_completion_init_model_id_override(mock_client, bedrock_unit_test_env, model_id) -> None:
"""Test initialization of Amazon Bedrock Text Completion service"""
bedrock_text_completion = BedrockTextCompletion(model_id=model_id)
assert bedrock_text_completion.ai_model_id == model_id
assert bedrock_text_completion.service_id == model_id
assert bedrock_text_completion.bedrock_client is not None
assert bedrock_text_completion.bedrock_runtime_client is not None
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_text_completion_init_custom_service_id(mock_client, bedrock_unit_test_env, service_id) -> None:
"""Test initialization of Amazon Bedrock Text Completion service"""
bedrock_text_completion = BedrockTextCompletion(service_id=service_id)
assert bedrock_text_completion.service_id == service_id
assert bedrock_text_completion.bedrock_client is not None
assert bedrock_text_completion.bedrock_runtime_client is not None
def test_bedrock_text_completion_init_custom_clients(bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Completion service"""
bedrock_text_completion = BedrockTextCompletion(
runtime_client=MockBedrockRuntimeClient(),
client=MockBedrockClient(),
)
assert isinstance(bedrock_text_completion.bedrock_client, MockBedrockClient)
assert isinstance(bedrock_text_completion.bedrock_runtime_client, MockBedrockRuntimeClient)
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_text_completion_init_custom_client(mock_client, bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Completion service"""
bedrock_text_completion = BedrockTextCompletion(
client=MockBedrockClient(),
)
assert isinstance(bedrock_text_completion.bedrock_client, MockBedrockClient)
assert bedrock_text_completion.bedrock_runtime_client is not None
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_text_completion_init_custom_runtime_client(mock_client, bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Completion service"""
bedrock_text_completion = BedrockTextCompletion(
runtime_client=MockBedrockRuntimeClient(),
)
assert bedrock_text_completion.bedrock_client is not None
assert isinstance(bedrock_text_completion.bedrock_runtime_client, MockBedrockRuntimeClient)
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_text_completion_init_custom_bedrock_model_provider(mock_client, bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Completion service"""
bedrock_text_completion = BedrockTextCompletion(
model_provider=BedrockModelProvider.AMAZON,
)
assert bedrock_text_completion.bedrock_model_provider == BedrockModelProvider.AMAZON
@pytest.mark.parametrize("exclude_list", [["BEDROCK_TEXT_MODEL_ID"]], indirect=True)
def test_bedrock_text_completion_client_init_with_empty_model_id(bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Completion service with empty model id"""
with pytest.raises(ServiceInitializationError, match="The Amazon Bedrock Text Model ID is missing."):
BedrockTextCompletion(env_file_path="fake_env_file_path.env")
def test_bedrock_text_completion_client_init_invalid_settings(bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Completion service with invalid settings"""
with pytest.raises(
ServiceInitializationError, match="Failed to initialize the Amazon Bedrock Text Completion Service."
):
BedrockTextCompletion(model_id=123) # Model ID must be a string
def test_bedrock_text_completion_client_init_invalid_model_provider(bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Completion service with invalid settings"""
with pytest.raises(
ServiceInitializationError, match="Failed to initialize the Amazon Bedrock Text Completion Service."
):
BedrockTextCompletion(model_provider="invalid_provider")
@patch.object(boto3, "client", return_value=Mock())
def test_prompt_execution_settings_class(mock_client, bedrock_unit_test_env) -> None:
"""Test getting prompt execution settings class"""
bedrock_completion_client = BedrockTextCompletion()
assert bedrock_completion_client.get_prompt_execution_settings_class() == BedrockTextPromptExecutionSettings
# endregion
# region text completion
@pytest.mark.parametrize(
# These are fake model ids with the supported prefixes
"model_id",
[
"amazon.titan",
"anthropic.claude",
"cohere.command",
"ai21.jamba",
"meta.llama",
"mistral.ai",
],
indirect=True,
)
async def test_bedrock_text_completion(
model_id,
mock_bedrock_text_completion_response,
output_text,
) -> None:
"""Test Amazon Bedrock Text Completion complete method"""
with patch.object(
MockBedrockRuntimeClient, "invoke_model", return_value=mock_bedrock_text_completion_response
) as mock_model_invoke:
# Setup
bedrock_text_completion = BedrockTextCompletion(
model_id=model_id,
runtime_client=MockBedrockRuntimeClient(),
client=MockBedrockClient(),
)
# Act
settings = BedrockTextPromptExecutionSettings()
response = await bedrock_text_completion.get_text_contents("Hello!", settings=settings)
# Assert
mock_model_invoke.assert_called_once_with(
body=json.dumps(get_text_completion_request_body(model_id, "Hello!", settings)),
modelId=model_id,
accept="application/json",
contentType="application/json",
)
assert isinstance(response, list)
assert len(response) == 1
assert isinstance(response[0], TextContent)
assert response[0].ai_model_id == model_id
assert response[0].text == output_text
@pytest.mark.parametrize(
# These are fake model ids with the supported prefixes
"model_id",
[
"arn:aws:bedrock:us-east-1:972143716085:application-inference-profile/123456",
],
indirect=True,
)
async def test_bedrock_text_completion_with_application_inference_profile(
model_id,
mock_bedrock_text_completion_response,
output_text,
model_provider,
) -> None:
"""Test Amazon Bedrock Text Completion complete method"""
with patch.object(
MockBedrockRuntimeClient,
"invoke_model",
return_value=mock_bedrock_text_completion_response,
) as mock_model_invoke:
# Setup
bedrock_text_completion = BedrockTextCompletion(
model_id=model_id,
runtime_client=MockBedrockRuntimeClient(),
client=MockBedrockClient(),
model_provider=model_provider,
)
# Act
settings = BedrockTextPromptExecutionSettings()
await bedrock_text_completion.get_text_contents("Hello!", settings=settings)
# Assert
mock_model_invoke.assert_called_once_with(
body=json.dumps(get_text_completion_request_body(model_id, "Hello!", settings, model_provider)),
modelId=model_id,
accept="application/json",
contentType="application/json",
)
@pytest.mark.parametrize(
# These are fake model ids with the supported prefixes
"model_id",
[
"amazon.titan",
],
indirect=True,
)
async def test_bedrock_streaming_text_completion(
model_id,
mock_bedrock_streaming_text_completion_response,
output_text,
) -> None:
"""Test Amazon Bedrock Text Completion complete method"""
with patch.object(
MockBedrockRuntimeClient,
"invoke_model_with_response_stream",
return_value=mock_bedrock_streaming_text_completion_response,
) as mock_invoke_model_with_response_stream:
# Setup
bedrock_text_completion = BedrockTextCompletion(
model_id=model_id,
runtime_client=MockBedrockRuntimeClient(),
client=MockBedrockClient(),
)
# Act
settings = BedrockTextPromptExecutionSettings()
chunks: list[StreamingTextContent] = []
async for streaming_responses in bedrock_text_completion.get_streaming_text_contents(
"Hello!", settings=settings
):
chunks.extend(streaming_responses)
response = reduce(lambda p, r: p + r, chunks)
# Assert
mock_invoke_model_with_response_stream.assert_called_once_with(
body=json.dumps(get_text_completion_request_body(model_id, "Hello!", settings)),
modelId=model_id,
accept="application/json",
contentType="application/json",
)
assert isinstance(response, StreamingTextContent)
assert response.ai_model_id == model_id
assert response.text == output_text
assert response.choice_index == 0
assert isinstance(response.inner_content, list)
@pytest.mark.parametrize(
# These are fake model ids with the supported prefixes
"model_id",
[
"arn:aws:bedrock:us-east-1:972143716085:application-inference-profile/123456",
],
indirect=True,
)
async def test_bedrock_streaming_text_completion_with_application_inference_profile(
model_id,
mock_bedrock_streaming_text_completion_response,
output_text,
model_provider,
) -> None:
"""Test Amazon Bedrock Chat Completion complete method"""
with patch.object(
MockBedrockRuntimeClient,
"invoke_model_with_response_stream",
return_value=mock_bedrock_streaming_text_completion_response,
) as mock_invoke_model_with_response_stream:
# Setup
bedrock_text_completion = BedrockTextCompletion(
model_id=model_id,
runtime_client=MockBedrockRuntimeClient(),
client=MockBedrockClient(),
model_provider=model_provider,
)
# Act
settings = BedrockTextPromptExecutionSettings()
chunks: list[StreamingTextContent] = []
async for streaming_responses in bedrock_text_completion.get_streaming_text_contents(
"Hello!", settings=settings
):
chunks.extend(streaming_responses)
# Assert
mock_invoke_model_with_response_stream.assert_called_once_with(
body=json.dumps(get_text_completion_request_body(model_id, "Hello!", settings, model_provider)),
modelId=model_id,
accept="application/json",
contentType="application/json",
)
# endregion
@@ -0,0 +1,235 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import ANY, Mock, patch
import boto3
import pytest
from semantic_kernel.connectors.ai.bedrock.bedrock_prompt_execution_settings import (
BedrockEmbeddingPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.bedrock.services.bedrock_text_embedding import BedrockTextEmbedding
from semantic_kernel.connectors.ai.bedrock.services.model_provider.bedrock_model_provider import BedrockModelProvider
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError, ServiceInvalidResponseError
from tests.unit.connectors.ai.bedrock.conftest import MockBedrockClient, MockBedrockRuntimeClient
# region init
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_text_embedding_init(mock_client, bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Embedding service"""
bedrock_text_embedding = BedrockTextEmbedding()
assert bedrock_text_embedding.ai_model_id == bedrock_unit_test_env["BEDROCK_EMBEDDING_MODEL_ID"]
assert bedrock_text_embedding.service_id == bedrock_unit_test_env["BEDROCK_EMBEDDING_MODEL_ID"]
assert bedrock_text_embedding.bedrock_model_provider == BedrockModelProvider(
bedrock_unit_test_env["BEDROCK_MODEL_PROVIDER"]
)
assert bedrock_text_embedding.bedrock_client is not None
assert bedrock_text_embedding.bedrock_runtime_client is not None
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_text_embedding_init_model_id_override(mock_client, bedrock_unit_test_env, model_id) -> None:
"""Test initialization of Amazon Bedrock Text Embedding service"""
bedrock_text_embedding = BedrockTextEmbedding(model_id=model_id)
assert bedrock_text_embedding.ai_model_id == model_id
assert bedrock_text_embedding.service_id == model_id
assert bedrock_text_embedding.bedrock_client is not None
assert bedrock_text_embedding.bedrock_runtime_client is not None
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_text_embedding_init_custom_service_id(mock_client, bedrock_unit_test_env, service_id) -> None:
"""Test initialization of Amazon Bedrock Text Embedding service"""
bedrock_text_embedding = BedrockTextEmbedding(service_id=service_id)
assert bedrock_text_embedding.service_id == service_id
assert bedrock_text_embedding.bedrock_client is not None
assert bedrock_text_embedding.bedrock_runtime_client is not None
def test_bedrock_text_embedding_init_custom_clients(bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Embedding service"""
bedrock_text_embedding = BedrockTextEmbedding(
runtime_client=MockBedrockRuntimeClient(),
client=MockBedrockClient(),
)
assert isinstance(bedrock_text_embedding.bedrock_client, MockBedrockClient)
assert isinstance(bedrock_text_embedding.bedrock_runtime_client, MockBedrockRuntimeClient)
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_text_embedding_init_custom_client(mock_client, bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Embedding service"""
bedrock_text_embedding = BedrockTextEmbedding(
client=MockBedrockClient(),
)
assert isinstance(bedrock_text_embedding.bedrock_client, MockBedrockClient)
assert bedrock_text_embedding.bedrock_runtime_client is not None
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_text_embedding_init_custom_runtime_client(mock_client, bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Embedding service"""
bedrock_text_embedding = BedrockTextEmbedding(
runtime_client=MockBedrockRuntimeClient(),
)
assert bedrock_text_embedding.bedrock_client is not None
assert isinstance(bedrock_text_embedding.bedrock_runtime_client, MockBedrockRuntimeClient)
@patch.object(boto3, "client", return_value=Mock())
def test_bedrock_text_embedding_init_custom_bedrock_model_provider(mock_client, bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Embedding service"""
bedrock_text_embedding = BedrockTextEmbedding(
model_provider=BedrockModelProvider.AMAZON,
)
assert bedrock_text_embedding.bedrock_model_provider == BedrockModelProvider.AMAZON
@pytest.mark.parametrize("exclude_list", [["BEDROCK_EMBEDDING_MODEL_ID"]], indirect=True)
def test_bedrock_text_embedding_client_init_with_empty_model_id(bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Embedding service with empty model id"""
with pytest.raises(ServiceInitializationError, match="The Amazon Bedrock Text Embedding Model ID is missing."):
BedrockTextEmbedding(env_file_path="fake_env_file_path.env")
def test_bedrock_text_embedding_client_init_invalid_settings(bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Embedding service with invalid settings"""
with pytest.raises(
ServiceInitializationError, match="Failed to initialize the Amazon Bedrock Text Embedding Service."
):
BedrockTextEmbedding(model_id=123) # Model ID must be a string
def test_bedrock_text_embedding_client_init_invalid_model_provider(bedrock_unit_test_env) -> None:
"""Test initialization of Amazon Bedrock Text Embedding service with invalid settings"""
with pytest.raises(
ServiceInitializationError, match="Failed to initialize the Amazon Bedrock Text Embedding Service."
):
BedrockTextEmbedding(model_provider="invalid_provider")
@patch.object(boto3, "client", return_value=Mock())
def test_prompt_execution_settings_class(mock_client, bedrock_unit_test_env) -> None:
"""Test getting prompt execution settings class"""
bedrock_completion_client = BedrockTextEmbedding()
assert bedrock_completion_client.get_prompt_execution_settings_class() == BedrockEmbeddingPromptExecutionSettings
# endregion
@pytest.mark.parametrize(
# These are fake model ids with the supported prefixes
"model_id",
[
"amazon.titan",
"cohere.command",
],
indirect=True,
)
async def test_bedrock_text_embedding(model_id, mock_bedrock_text_embedding_response) -> None:
"""Test Bedrock text embedding generation"""
with patch.object(
MockBedrockRuntimeClient, "invoke_model", return_value=mock_bedrock_text_embedding_response
) as mock_model_invoke:
# Setup
bedrock_text_embedding = BedrockTextEmbedding(
model_id=model_id,
runtime_client=MockBedrockRuntimeClient(),
client=MockBedrockClient(),
)
# Act
settings = BedrockEmbeddingPromptExecutionSettings()
response = await bedrock_text_embedding.generate_embeddings(["hello", "world"], settings)
# Assert
mock_model_invoke.assert_called_with(
body=ANY,
modelId=model_id,
accept="application/json",
contentType="application/json",
)
assert mock_model_invoke.call_count == 2
assert len(response) == 2
@pytest.mark.parametrize(
# These are fake model ids with the supported prefixes
"model_id",
[
"arn:aws:bedrock:us-east-1:972143716085:application-inference-profile/123456",
],
indirect=True,
)
async def test_bedrock_text_embedding_with_application_inference_profile(
model_id,
mock_bedrock_text_embedding_response,
model_provider,
) -> None:
"""Test Bedrock text embedding generation"""
with patch.object(
MockBedrockRuntimeClient, "invoke_model", return_value=mock_bedrock_text_embedding_response
) as mock_model_invoke:
# Setup
bedrock_text_embedding = BedrockTextEmbedding(
model_id=model_id,
runtime_client=MockBedrockRuntimeClient(),
client=MockBedrockClient(),
model_provider=BedrockModelProvider.AMAZON,
)
# Act
settings = BedrockEmbeddingPromptExecutionSettings()
response = await bedrock_text_embedding.generate_embeddings(["hello", "world"], settings)
# Assert
mock_model_invoke.assert_called_with(
body=ANY,
modelId=model_id,
accept="application/json",
contentType="application/json",
)
assert mock_model_invoke.call_count == 2
assert len(response) == 2
@pytest.mark.parametrize(
# These are fake model ids with the supported prefixes
"model_id",
[
"amazon.titan",
"cohere.command",
],
indirect=True,
)
async def test_bedrock_text_embedding_with_invalid_response(
model_id, mock_bedrock_text_embedding_invalid_response
) -> None:
"""Test Bedrock text embedding generation with invalid response"""
with patch.object(
MockBedrockRuntimeClient, "invoke_model", return_value=mock_bedrock_text_embedding_invalid_response
):
# Setup
bedrock_text_embedding = BedrockTextEmbedding(
model_id=model_id,
runtime_client=MockBedrockRuntimeClient(),
client=MockBedrockClient(),
)
with pytest.raises(ServiceInvalidResponseError):
await bedrock_text_embedding.generate_embeddings(["hello", "world"])
@@ -0,0 +1,104 @@
# Copyright (c) Microsoft. All rights reserved.
from semantic_kernel.connectors.ai.bedrock import BedrockChatPromptExecutionSettings, BedrockPromptExecutionSettings
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
def test_default_bedrock_prompt_execution_settings():
settings = BedrockPromptExecutionSettings()
assert settings.temperature is None
assert settings.top_p is None
assert settings.top_k is None
assert settings.max_tokens is None
assert settings.stop == []
def test_custom_bedrock_prompt_execution_settings():
settings = BedrockPromptExecutionSettings(
temperature=0.5,
top_p=0.5,
top_k=10,
max_tokens=128,
stop=["world"],
)
assert settings.temperature == 0.5
assert settings.top_p == 0.5
assert settings.top_k == 10
assert settings.max_tokens == 128
assert settings.stop == ["world"]
def test_bedrock_prompt_execution_settings_from_default_completion_config():
settings = PromptExecutionSettings(service_id="test_service")
chat_settings = BedrockChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.service_id == "test_service"
assert chat_settings.temperature is None
assert chat_settings.top_p is None
assert chat_settings.top_k is None
assert chat_settings.max_tokens is None
assert chat_settings.stop == []
def test_bedrock_prompt_execution_settings_from_openai_prompt_execution_settings():
chat_settings = BedrockChatPromptExecutionSettings(service_id="test_service", temperature=1.0)
new_settings = BedrockPromptExecutionSettings(service_id="test_2", temperature=0.0)
chat_settings.update_from_prompt_execution_settings(new_settings)
assert chat_settings.service_id == "test_2"
assert chat_settings.temperature == 0.0
def test_bedrock_prompt_execution_settings_from_custom_completion_config():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"temperature": 0.5,
"top_p": 0.5,
"top_k": 10,
"max_tokens": 128,
"stop": ["world"],
},
)
chat_settings = BedrockChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.temperature == 0.5
assert chat_settings.top_p == 0.5
assert chat_settings.top_k == 10
assert chat_settings.max_tokens == 128
assert chat_settings.stop == ["world"]
def test_bedrock_chat_prompt_execution_settings_from_custom_completion_config_with_functions():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"tools": [{"function": {}}],
},
)
chat_settings = BedrockChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.tools == [{"function": {}}]
def test_create_options():
settings = BedrockPromptExecutionSettings(
service_id="test_service",
extension_data={
"temperature": 0.5,
"top_p": 0.5,
"top_k": 10,
"max_tokens": 128,
"stop": ["world"],
},
)
options = settings.prepare_settings_dict()
assert options["temperature"] == 0.5
assert options["top_p"] == 0.5
assert options["top_k"] == 10
assert options["max_tokens"] == 128
assert options["stop"] == ["world"]
@@ -0,0 +1,23 @@
# Copyright (c) Microsoft. All rights reserved.
import pytest
from semantic_kernel.contents.chat_history import ChatHistory
@pytest.fixture()
def service_id() -> str:
return "test_service_id"
@pytest.fixture()
def chat_history() -> ChatHistory:
chat_history = ChatHistory()
chat_history.add_system_message("system_prompt")
chat_history.add_user_message("test_prompt")
return chat_history
@pytest.fixture()
def prompt() -> str:
return "test_prompt"
@@ -0,0 +1,174 @@
# Copyright (c) Microsoft. All rights reserved.
from collections.abc import AsyncGenerator, AsyncIterator
from unittest.mock import MagicMock
import pytest
import pytest_asyncio
from google.genai.types import (
Candidate,
Content,
FinishReason,
GenerateContentResponse,
GenerateContentResponseUsageMetadata,
Part,
)
@pytest.fixture()
def google_ai_unit_test_env(monkeypatch, exclude_list, override_env_param_dict):
"""Fixture to set environment variables for Google AI Unit Tests."""
if exclude_list is None:
exclude_list = []
if override_env_param_dict is None:
override_env_param_dict = {}
env_vars = {
"GOOGLE_AI_GEMINI_MODEL_ID": "test-gemini-model-id",
"GOOGLE_AI_EMBEDDING_MODEL_ID": "test-embedding-model-id",
"GOOGLE_AI_API_KEY": "test-api-key",
"GOOGLE_AI_CLOUD_PROJECT_ID": "test-project-id",
"GOOGLE_AI_CLOUD_REGION": "test-region",
}
env_vars.update(override_env_param_dict)
for key, value in env_vars.items():
if key not in exclude_list:
monkeypatch.setenv(key, value)
else:
monkeypatch.delenv(key, raising=False)
return env_vars
@pytest.fixture()
def mock_google_ai_chat_completion_response() -> GenerateContentResponse:
"""Mock Google AI Chat Completion response."""
candidate = Candidate()
candidate.index = 0
candidate.content = Content(role="user", parts=[Part.from_text(text="Test content")])
candidate.finish_reason = FinishReason.STOP
response = GenerateContentResponse()
response.candidates = [candidate]
response.usage_metadata = GenerateContentResponseUsageMetadata(
prompt_token_count=0,
cached_content_token_count=0,
candidates_token_count=0,
total_token_count=0,
)
return response
@pytest.fixture()
def mock_google_ai_chat_completion_response_with_tool_call() -> GenerateContentResponse:
"""Mock Google AI Chat Completion response."""
candidate = Candidate()
candidate.index = 0
candidate.content = Content(
role="user",
parts=[
Part.from_function_call(
name="test_function",
args={"test_arg": "test_value"},
)
],
)
candidate.finish_reason = FinishReason.STOP
response = GenerateContentResponse()
response.candidates = [candidate]
response.usage_metadata = GenerateContentResponseUsageMetadata(
prompt_token_count=0,
cached_content_token_count=0,
candidates_token_count=0,
total_token_count=0,
)
return response
@pytest_asyncio.fixture
async def mock_google_ai_streaming_chat_completion_response() -> AsyncIterator[GenerateContentResponse]:
"""Mock Google AI streaming Chat Completion response."""
candidate = Candidate()
candidate.index = 0
candidate.content = Content(role="user", parts=[Part.from_text(text="Test content")])
candidate.finish_reason = FinishReason.STOP
response = GenerateContentResponse()
response.candidates = [candidate]
response.usage_metadata = GenerateContentResponseUsageMetadata(
prompt_token_count=0,
cached_content_token_count=0,
candidates_token_count=0,
total_token_count=0,
)
iterable = MagicMock(spec=AsyncGenerator)
iterable.__aiter__.return_value = [response]
return iterable
@pytest_asyncio.fixture
async def mock_google_ai_streaming_chat_completion_response_with_tool_call() -> AsyncIterator[GenerateContentResponse]:
"""Mock Google AI streaming Chat Completion response with tool call."""
candidate = Candidate()
candidate.index = 0
candidate.content = Content(
role="user",
parts=[
Part.from_function_call(
name="getLightStatus",
args={"arg1": "test_value"},
)
],
)
candidate.finish_reason = FinishReason.STOP
response = GenerateContentResponse()
response.candidates = [candidate]
response.usage_metadata = GenerateContentResponseUsageMetadata(
prompt_token_count=0,
cached_content_token_count=0,
candidates_token_count=0,
total_token_count=0,
)
iterable = MagicMock(spec=AsyncGenerator)
iterable.__aiter__.return_value = [response]
return iterable
@pytest.fixture()
def mock_google_ai_text_completion_response() -> GenerateContentResponse:
"""Mock Google AI Text Completion response."""
candidate = Candidate()
candidate.index = 0
candidate.content = Content(parts=[Part.from_text(text="Test content")])
response = GenerateContentResponse()
response.candidates = [candidate]
return response
@pytest_asyncio.fixture
async def mock_google_ai_streaming_text_completion_response() -> AsyncIterator[GenerateContentResponse]:
"""Mock Google AI streaming Text Completion response."""
candidate = Candidate()
candidate.index = 0
candidate.content = Content(parts=[Part.from_text(text="Test content")])
response = GenerateContentResponse()
response.candidates = [candidate]
iterable = MagicMock(spec=AsyncGenerator)
iterable.__aiter__.return_value = [response]
return iterable
@@ -0,0 +1,650 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, patch
import pytest
from google.genai import Client
from google.genai.models import AsyncModels
from google.genai.types import Content, GenerateContentConfigDict
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
from semantic_kernel.connectors.ai.google.google_ai.google_ai_prompt_execution_settings import (
GoogleAIChatPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.google.google_ai.google_ai_settings import GoogleAISettings
from semantic_kernel.connectors.ai.google.google_ai.services.google_ai_chat_completion import GoogleAIChatCompletion
from semantic_kernel.connectors.ai.google.shared_utils import filter_system_message
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.utils.finish_reason import FinishReason
from semantic_kernel.exceptions.service_exceptions import (
ServiceInitializationError,
ServiceInvalidExecutionSettingsError,
)
from semantic_kernel.kernel import Kernel
# region init
def test_google_ai_chat_completion_init(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAIChatCompletion"""
model_id = google_ai_unit_test_env["GOOGLE_AI_GEMINI_MODEL_ID"]
api_key = google_ai_unit_test_env["GOOGLE_AI_API_KEY"]
google_ai_chat_completion = GoogleAIChatCompletion()
assert google_ai_chat_completion.ai_model_id == model_id
assert google_ai_chat_completion.service_id == model_id
assert isinstance(google_ai_chat_completion.service_settings, GoogleAISettings)
assert google_ai_chat_completion.service_settings.gemini_model_id == model_id
assert google_ai_chat_completion.service_settings.api_key.get_secret_value() == api_key
def test_google_ai_chat_completion_init_with_service_id(google_ai_unit_test_env, service_id) -> None:
"""Test initialization of GoogleAIChatCompletion with a service_id that is not the model_id"""
google_ai_chat_completion = GoogleAIChatCompletion(service_id=service_id)
assert google_ai_chat_completion.service_id == service_id
def test_google_ai_chat_completion_init_with_model_id_in_argument(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAIChatCompletion with model_id in argument"""
google_ai_chat_completion = GoogleAIChatCompletion(gemini_model_id="custom_model_id")
assert google_ai_chat_completion.ai_model_id == "custom_model_id"
assert google_ai_chat_completion.service_id == "custom_model_id"
@pytest.mark.parametrize("exclude_list", [["GOOGLE_AI_GEMINI_MODEL_ID"]], indirect=True)
def test_google_ai_chat_completion_init_with_empty_model_id(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAIChatCompletion with an empty model_id"""
with pytest.raises(ServiceInitializationError, match="The Google AI Gemini model ID is required."):
GoogleAIChatCompletion(env_file_path="fake_env_file_path.env")
@pytest.mark.parametrize("exclude_list", [["GOOGLE_AI_API_KEY"]], indirect=True)
def test_google_ai_chat_completion_init_with_empty_api_key(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAIChatCompletion with an empty api_key"""
with pytest.raises(ServiceInitializationError, match="API key is required when use_vertexai is False."):
GoogleAIChatCompletion(env_file_path="fake_env_file_path.env")
@pytest.mark.parametrize("exclude_list", [["GOOGLE_AI_CLOUD_PROJECT_ID"]], indirect=True)
def test_google_ai_chat_completion_init_with_use_vertexai_missing_project_id(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAIChatCompletion with use_vertexai true but missing project_id"""
with pytest.raises(ServiceInitializationError, match="Project ID must be provided when use_vertexai is True."):
GoogleAIChatCompletion(use_vertexai=True, env_file_path="fake_env_file_path.env")
@pytest.mark.parametrize("exclude_list", [["GOOGLE_AI_CLOUD_REGION"]], indirect=True)
def test_google_ai_chat_completion_init_with_use_vertexai_missing_region(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAIChatCompletion with use_vertexai true but missing region"""
with pytest.raises(ServiceInitializationError, match="Region must be provided when use_vertexai is True."):
GoogleAIChatCompletion(use_vertexai=True, env_file_path="fake_env_file_path.env")
@pytest.mark.parametrize("exclude_list", [["GOOGLE_AI_API_KEY"]], indirect=True)
def test_google_ai_chat_completion_init_with_use_vertexai_no_api_key(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAIChatCompletion succeeds with use_vertexai=True and no api_key"""
chat_completion = GoogleAIChatCompletion(use_vertexai=True)
assert chat_completion.service_settings.use_vertexai is True
def test_prompt_execution_settings_class(google_ai_unit_test_env) -> None:
google_ai_chat_completion = GoogleAIChatCompletion()
assert google_ai_chat_completion.get_prompt_execution_settings_class() == GoogleAIChatPromptExecutionSettings
# endregion init
# region chat completion
@patch.object(AsyncModels, "generate_content", new_callable=AsyncMock)
async def test_google_ai_chat_completion(
mock_google_ai_model_generate_content,
google_ai_unit_test_env,
chat_history: ChatHistory,
mock_google_ai_chat_completion_response,
) -> None:
"""Test chat completion with GoogleAIChatCompletion"""
settings = GoogleAIChatPromptExecutionSettings(top_k=5, temperature=0.7)
mock_google_ai_model_generate_content.return_value = mock_google_ai_chat_completion_response
google_ai_chat_completion = GoogleAIChatCompletion()
responses: list[ChatMessageContent] = await google_ai_chat_completion.get_chat_message_contents(
chat_history, settings
)
mock_google_ai_model_generate_content.assert_called_once_with(
model=google_ai_chat_completion.service_settings.gemini_model_id,
contents=google_ai_chat_completion._prepare_chat_history_for_request(chat_history),
config=GenerateContentConfigDict(
**settings.prepare_settings_dict(),
system_instruction=filter_system_message(chat_history),
),
)
assert len(responses) == 1
assert responses[0].role == "assistant"
assert responses[0].content == mock_google_ai_chat_completion_response.candidates[0].content.parts[0].text
assert responses[0].finish_reason == FinishReason.STOP
assert "usage" in responses[0].metadata
assert "prompt_feedback" in responses[0].metadata
assert responses[0].inner_content == mock_google_ai_chat_completion_response
async def test_google_ai_chat_completion_with_custom_client(
chat_history: ChatHistory,
google_ai_unit_test_env,
mock_google_ai_chat_completion_response,
) -> None:
"""Test chat completion with GoogleAIChatCompletion using a custom client"""
# Create a custom client with a fake API key for testing
custom_client = Client(api_key="fake-api-key-for-testing")
# Mock the custom client's generate_content method
mock_generate_content = AsyncMock(return_value=mock_google_ai_chat_completion_response)
custom_client.aio.models.generate_content = mock_generate_content
google_ai_chat_completion = GoogleAIChatCompletion(client=custom_client)
responses: list[ChatMessageContent] = await google_ai_chat_completion.get_chat_message_contents(
chat_history,
GoogleAIChatPromptExecutionSettings(),
)
custom_client.close()
# Verify that the custom client was used and returned the expected response
assert len(responses) == 1
assert responses[0].role == "assistant"
assert responses[0].content == mock_google_ai_chat_completion_response.candidates[0].content.parts[0].text
assert responses[0].finish_reason == FinishReason.STOP
# Verify that the custom client's method was called
mock_generate_content.assert_called_once()
async def test_google_ai_chat_completion_with_function_choice_behavior_fail_verification(
chat_history: ChatHistory,
google_ai_unit_test_env,
) -> None:
"""Test completion of GoogleAIChatCompletion with function choice behavior expect verification failure"""
# Missing kernel
with pytest.raises(ServiceInvalidExecutionSettingsError):
settings = GoogleAIChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
google_ai_chat_completion = GoogleAIChatCompletion()
await google_ai_chat_completion.get_chat_message_contents(
chat_history=chat_history,
settings=settings,
)
@patch.object(AsyncModels, "generate_content", new_callable=AsyncMock)
async def test_google_ai_chat_completion_with_function_choice_behavior(
mock_google_ai_model_generate_content,
google_ai_unit_test_env,
kernel,
chat_history: ChatHistory,
mock_google_ai_chat_completion_response_with_tool_call,
) -> None:
"""Test completion of GoogleAIChatCompletion with function choice behavior"""
mock_google_ai_model_generate_content.return_value = mock_google_ai_chat_completion_response_with_tool_call
settings = GoogleAIChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
settings.function_choice_behavior.maximum_auto_invoke_attempts = 1
google_ai_chat_completion = GoogleAIChatCompletion()
responses = await google_ai_chat_completion.get_chat_message_contents(
chat_history=chat_history,
settings=settings,
kernel=kernel,
)
# The function should be called twice:
# One for the tool call and one for the last completion
# after the maximum_auto_invoke_attempts is reached
assert mock_google_ai_model_generate_content.call_count == 2
assert len(responses) == 1
assert responses[0].role == "assistant"
# Google doesn't return STOP as the finish reason for tool calls
assert responses[0].finish_reason == FinishReason.STOP
@patch.object(AsyncModels, "generate_content", new_callable=AsyncMock)
async def test_google_ai_chat_completion_with_function_choice_behavior_no_tool_call(
mock_google_ai_model_generate_content,
google_ai_unit_test_env,
kernel,
chat_history: ChatHistory,
mock_google_ai_chat_completion_response,
) -> None:
"""Test completion of GoogleAIChatCompletion with function choice behavior but no tool call returned"""
mock_google_ai_model_generate_content.return_value = mock_google_ai_chat_completion_response
settings = GoogleAIChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
settings.function_choice_behavior.maximum_auto_invoke_attempts = 1
google_ai_chat_completion = GoogleAIChatCompletion()
responses = await google_ai_chat_completion.get_chat_message_contents(
chat_history=chat_history,
settings=settings,
kernel=kernel,
)
mock_google_ai_model_generate_content.assert_awaited_once_with(
model=google_ai_chat_completion.service_settings.gemini_model_id,
contents=google_ai_chat_completion._prepare_chat_history_for_request(chat_history),
config=GenerateContentConfigDict(
**settings.prepare_settings_dict(),
system_instruction=filter_system_message(chat_history),
),
)
assert len(responses) == 1
assert responses[0].role == "assistant"
assert responses[0].content == mock_google_ai_chat_completion_response.candidates[0].content.parts[0].text
# endregion chat completion
# region streaming chat completion
@patch.object(AsyncModels, "generate_content_stream", new_callable=AsyncMock)
async def test_google_ai_streaming_chat_completion(
mock_google_ai_model_generate_content_stream,
google_ai_unit_test_env,
chat_history: ChatHistory,
mock_google_ai_streaming_chat_completion_response,
) -> None:
"""Test streaming chat completion with GoogleAIChatCompletion"""
settings = GoogleAIChatPromptExecutionSettings()
mock_google_ai_model_generate_content_stream.return_value = mock_google_ai_streaming_chat_completion_response
google_ai_chat_completion = GoogleAIChatCompletion()
async for messages in google_ai_chat_completion.get_streaming_chat_message_contents(chat_history, settings):
assert len(messages) == 1
assert messages[0].role == "assistant"
assert messages[0].finish_reason == FinishReason.STOP
assert "usage" in messages[0].metadata
assert "prompt_feedback" in messages[0].metadata
mock_google_ai_model_generate_content_stream.assert_called_once_with(
model=google_ai_chat_completion.service_settings.gemini_model_id,
contents=google_ai_chat_completion._prepare_chat_history_for_request(chat_history),
config=GenerateContentConfigDict(
**settings.prepare_settings_dict(),
system_instruction=filter_system_message(chat_history),
),
)
async def test_google_ai_streaming_chat_completion_with_custom_client(
chat_history: ChatHistory,
google_ai_unit_test_env,
mock_google_ai_streaming_chat_completion_response,
) -> None:
"""Test streaming chat completion with GoogleAIChatCompletion using a custom client"""
# Create a custom client with a fake API key for testing
custom_client = Client(api_key="fake-api-key-for-testing")
# Mock the custom client's generate_content_stream method
mock_generate_content_stream = AsyncMock(return_value=mock_google_ai_streaming_chat_completion_response)
custom_client.aio.models.generate_content_stream = mock_generate_content_stream
google_ai_chat_completion = GoogleAIChatCompletion(client=custom_client)
all_messages = []
async for messages in google_ai_chat_completion.get_streaming_chat_message_contents(
chat_history, GoogleAIChatPromptExecutionSettings()
):
all_messages.extend(messages)
assert len(messages) == 1
assert messages[0].role == "assistant"
assert messages[0].finish_reason == FinishReason.STOP
assert "usage" in messages[0].metadata
assert "prompt_feedback" in messages[0].metadata
custom_client.close()
# Verify that the custom client was used and returned the expected response
assert len(all_messages) > 0
# Verify that the custom client's method was called
mock_generate_content_stream.assert_called_once()
async def test_google_ai_streaming_chat_completion_with_function_choice_behavior_fail_verification(
chat_history: ChatHistory,
google_ai_unit_test_env,
) -> None:
"""Test streaming chat completion of GoogleAIChatCompletion with function choice
behavior expect verification failure"""
# Missing kernel
with pytest.raises(ServiceInvalidExecutionSettingsError):
settings = GoogleAIChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
google_ai_chat_completion = GoogleAIChatCompletion()
async for _ in google_ai_chat_completion.get_streaming_chat_message_contents(
chat_history=chat_history,
settings=settings,
):
pass
@patch.object(AsyncModels, "generate_content_stream", new_callable=AsyncMock)
async def test_google_ai_streaming_chat_completion_with_function_choice_behavior(
mock_google_ai_model_generate_content_stream,
google_ai_unit_test_env,
kernel: Kernel,
chat_history: ChatHistory,
mock_google_ai_streaming_chat_completion_response_with_tool_call,
decorated_native_function,
) -> None:
"""Test streaming chat completion of GoogleAIChatCompletion with function choice behavior"""
mock_google_ai_model_generate_content_stream.return_value = (
mock_google_ai_streaming_chat_completion_response_with_tool_call
)
settings = GoogleAIChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
settings.function_choice_behavior.maximum_auto_invoke_attempts = 1
google_ai_chat_completion = GoogleAIChatCompletion()
kernel.add_function(plugin_name="TestPlugin", function=decorated_native_function)
all_messages = []
async for messages in google_ai_chat_completion.get_streaming_chat_message_contents(
chat_history,
settings,
kernel=kernel,
):
all_messages.extend(messages)
assert len(all_messages) == 2, f"Expected 2 messages, got {len(all_messages)}"
# Validate the first message
assert all_messages[0].role == "assistant", f"Unexpected role for first message: {all_messages[0].role}"
assert all_messages[0].content == "", f"Unexpected content for first message: {all_messages[0].content}"
assert all_messages[0].finish_reason == FinishReason.STOP, (
f"Unexpected finish reason for first message: {all_messages[0].finish_reason}"
)
# Validate the second message
assert all_messages[1].role == "tool", f"Unexpected role for second message: {all_messages[1].role}"
assert all_messages[1].content == "", f"Unexpected content for second message: {all_messages[1].content}"
assert all_messages[1].finish_reason is None
@patch.object(AsyncModels, "generate_content_stream", new_callable=AsyncMock)
async def test_google_ai_streaming_chat_completion_with_function_choice_behavior_no_tool_call(
mock_google_ai_model_generate_content_stream,
google_ai_unit_test_env,
kernel,
chat_history: ChatHistory,
mock_google_ai_streaming_chat_completion_response,
) -> None:
"""Test completion of GoogleAIChatCompletion with function choice behavior but no tool call returned"""
mock_google_ai_model_generate_content_stream.return_value = mock_google_ai_streaming_chat_completion_response
settings = GoogleAIChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
settings.function_choice_behavior.maximum_auto_invoke_attempts = 1
google_ai_chat_completion = GoogleAIChatCompletion()
async for messages in google_ai_chat_completion.get_streaming_chat_message_contents(
chat_history=chat_history,
settings=settings,
kernel=kernel,
):
assert len(messages) == 1
assert messages[0].role == "assistant"
assert messages[0].content == "Test content"
mock_google_ai_model_generate_content_stream.assert_awaited_once_with(
model=google_ai_chat_completion.service_settings.gemini_model_id,
contents=google_ai_chat_completion._prepare_chat_history_for_request(chat_history),
config=GenerateContentConfigDict(
**settings.prepare_settings_dict(),
system_instruction=filter_system_message(chat_history),
),
)
# endregion streaming chat completion
def test_google_ai_chat_completion_parse_chat_history_correctly(google_ai_unit_test_env) -> None:
"""Test _prepare_chat_history_for_request method"""
google_ai_chat_completion = GoogleAIChatCompletion()
chat_history = ChatHistory()
chat_history.add_system_message("test_system_message")
chat_history.add_user_message("test_user_message")
chat_history.add_assistant_message("test_assistant_message")
parsed_chat_history = google_ai_chat_completion._prepare_chat_history_for_request(chat_history)
assert isinstance(parsed_chat_history, list)
# System message should be ignored
assert len(parsed_chat_history) == 2
assert all(isinstance(message, Content) for message in parsed_chat_history)
assert parsed_chat_history[0].role == "user"
assert parsed_chat_history[0].parts[0].text == "test_user_message"
assert parsed_chat_history[1].role == "model"
assert parsed_chat_history[1].parts[0].text == "test_assistant_message"
# region deserialization (Part → FunctionCallContent round-trip)
def test_create_chat_message_content_with_thought_signature(google_ai_unit_test_env) -> None:
"""Test that thought_signature from a Part is deserialized into FunctionCallContent.metadata."""
from google.genai.types import (
Candidate,
Content,
GenerateContentResponse,
GenerateContentResponseUsageMetadata,
Part,
)
from google.genai.types import (
FinishReason as GFinishReason,
)
from semantic_kernel.contents.function_call_content import FunctionCallContent
thought_sig_value = b"test-thought-sig-bytes"
part = Part.from_function_call(name="test_function", args={"key": "value"})
part.thought_signature = thought_sig_value
candidate = Candidate()
candidate.index = 0
candidate.content = Content(role="user", parts=[part])
candidate.finish_reason = GFinishReason.STOP
response = GenerateContentResponse()
response.candidates = [candidate]
response.usage_metadata = GenerateContentResponseUsageMetadata(
prompt_token_count=0, cached_content_token_count=0, candidates_token_count=0, total_token_count=0
)
completion = GoogleAIChatCompletion()
result = completion._create_chat_message_content(response, candidate)
fc_items = [item for item in result.items if isinstance(item, FunctionCallContent)]
assert len(fc_items) == 1
assert fc_items[0].metadata is not None
assert fc_items[0].metadata["thought_signature"] == thought_sig_value
def test_create_chat_message_content_without_thought_signature(google_ai_unit_test_env) -> None:
"""Test that FunctionCallContent works when Part has no thought_signature."""
from google.genai.types import (
Candidate,
Content,
GenerateContentResponse,
GenerateContentResponseUsageMetadata,
Part,
)
from google.genai.types import (
FinishReason as GFinishReason,
)
from semantic_kernel.contents.function_call_content import FunctionCallContent
part = Part.from_function_call(name="test_function", args={"key": "value"})
candidate = Candidate()
candidate.index = 0
candidate.content = Content(role="user", parts=[part])
candidate.finish_reason = GFinishReason.STOP
response = GenerateContentResponse()
response.candidates = [candidate]
response.usage_metadata = GenerateContentResponseUsageMetadata(
prompt_token_count=0, cached_content_token_count=0, candidates_token_count=0, total_token_count=0
)
completion = GoogleAIChatCompletion()
result = completion._create_chat_message_content(response, candidate)
fc_items = [item for item in result.items if isinstance(item, FunctionCallContent)]
assert len(fc_items) == 1
assert "thought_signature" not in fc_items[0].metadata
def test_create_streaming_chat_message_content_with_thought_signature(google_ai_unit_test_env) -> None:
"""Test that thought_signature from a Part is deserialized in streaming path."""
from google.genai.types import (
Candidate,
Content,
GenerateContentResponse,
GenerateContentResponseUsageMetadata,
Part,
)
from google.genai.types import (
FinishReason as GFinishReason,
)
from semantic_kernel.contents.function_call_content import FunctionCallContent
thought_sig_value = b"streaming-thought-sig"
part = Part.from_function_call(name="stream_func", args={"a": "b"})
part.thought_signature = thought_sig_value
candidate = Candidate()
candidate.index = 0
candidate.content = Content(role="user", parts=[part])
candidate.finish_reason = GFinishReason.STOP
chunk = GenerateContentResponse()
chunk.candidates = [candidate]
chunk.usage_metadata = GenerateContentResponseUsageMetadata(
prompt_token_count=0, cached_content_token_count=0, candidates_token_count=0, total_token_count=0
)
completion = GoogleAIChatCompletion()
result = completion._create_streaming_chat_message_content(chunk, candidate)
fc_items = [item for item in result.items if isinstance(item, FunctionCallContent)]
assert len(fc_items) == 1
assert fc_items[0].metadata is not None
assert fc_items[0].metadata["thought_signature"] == thought_sig_value
def test_create_streaming_chat_message_content_without_thought_signature(google_ai_unit_test_env) -> None:
"""Test that streaming FunctionCallContent works when Part lacks thought_signature."""
from google.genai.types import (
Candidate,
Content,
GenerateContentResponse,
GenerateContentResponseUsageMetadata,
Part,
)
from google.genai.types import (
FinishReason as GFinishReason,
)
from semantic_kernel.contents.function_call_content import FunctionCallContent
part = Part.from_function_call(name="stream_func", args={"a": "b"})
candidate = Candidate()
candidate.index = 0
candidate.content = Content(role="user", parts=[part])
candidate.finish_reason = GFinishReason.STOP
chunk = GenerateContentResponse()
chunk.candidates = [candidate]
chunk.usage_metadata = GenerateContentResponseUsageMetadata(
prompt_token_count=0, cached_content_token_count=0, candidates_token_count=0, total_token_count=0
)
completion = GoogleAIChatCompletion()
result = completion._create_streaming_chat_message_content(chunk, candidate)
fc_items = [item for item in result.items if isinstance(item, FunctionCallContent)]
assert len(fc_items) == 1
assert "thought_signature" not in fc_items[0].metadata
def test_create_chat_message_content_getattr_guard_on_missing_attribute(google_ai_unit_test_env) -> None:
"""Test that getattr guard handles SDK versions where thought_signature doesn't exist on Part."""
from unittest.mock import MagicMock
from google.genai.types import (
GenerateContentResponse,
GenerateContentResponseUsageMetadata,
)
from semantic_kernel.contents.function_call_content import FunctionCallContent
# Create a mock Part that lacks 'thought_signature' attribute entirely
mock_part = MagicMock()
mock_part.text = None
mock_part.function_call.name = "test_func"
mock_part.function_call.args = {"x": "y"}
del mock_part.thought_signature # simulate older SDK without the field
# Use a fully-mocked candidate to avoid Content pydantic validation
mock_candidate = MagicMock()
mock_candidate.index = 0
mock_candidate.content.parts = [mock_part]
mock_candidate.finish_reason = 1 # STOP
response = GenerateContentResponse()
response.candidates = [mock_candidate]
response.usage_metadata = GenerateContentResponseUsageMetadata(
prompt_token_count=0, cached_content_token_count=0, candidates_token_count=0, total_token_count=0
)
completion = GoogleAIChatCompletion()
result = completion._create_chat_message_content(response, mock_candidate)
fc_items = [item for item in result.items if isinstance(item, FunctionCallContent)]
assert len(fc_items) == 1
assert "thought_signature" not in fc_items[0].metadata
# endregion deserialization
@@ -0,0 +1,212 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, patch
import pytest
from google.genai import Client
from google.genai.models import AsyncModels
from google.genai.types import GenerateContentConfigDict
from semantic_kernel.connectors.ai.google.google_ai.google_ai_prompt_execution_settings import (
GoogleAITextPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.google.google_ai.google_ai_settings import GoogleAISettings
from semantic_kernel.connectors.ai.google.google_ai.services.google_ai_text_completion import GoogleAITextCompletion
from semantic_kernel.contents.text_content import TextContent
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
# region init
def test_google_ai_text_completion_init(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAITextCompletion"""
model_id = google_ai_unit_test_env["GOOGLE_AI_GEMINI_MODEL_ID"]
api_key = google_ai_unit_test_env["GOOGLE_AI_API_KEY"]
google_ai_text_completion = GoogleAITextCompletion()
assert google_ai_text_completion.ai_model_id == model_id
assert google_ai_text_completion.service_id == model_id
assert isinstance(google_ai_text_completion.service_settings, GoogleAISettings)
assert google_ai_text_completion.service_settings.gemini_model_id == model_id
assert google_ai_text_completion.service_settings.api_key.get_secret_value() == api_key
def test_google_ai_text_completion_init_with_service_id(google_ai_unit_test_env, service_id) -> None:
"""Test initialization of GoogleAITextCompletion with a service_id that is not the model_id"""
google_ai_text_completion = GoogleAITextCompletion(service_id=service_id)
assert google_ai_text_completion.service_id == service_id
def test_google_ai_text_completion_init_with_model_id_in_argument(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAIChatCompletion with model_id in argument"""
google_ai_text_completion = GoogleAITextCompletion(gemini_model_id="custom_model_id")
assert google_ai_text_completion.ai_model_id == "custom_model_id"
assert google_ai_text_completion.service_id == "custom_model_id"
@pytest.mark.parametrize("exclude_list", [["GOOGLE_AI_GEMINI_MODEL_ID"]], indirect=True)
def test_google_ai_text_completion_init_with_empty_model_id(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAITextCompletion with an empty model_id"""
with pytest.raises(ServiceInitializationError, match="The Google AI Gemini model ID is required."):
GoogleAITextCompletion(env_file_path="fake_env_file_path.env")
@pytest.mark.parametrize("exclude_list", [["GOOGLE_AI_API_KEY"]], indirect=True)
def test_google_ai_text_completion_init_with_empty_api_key(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAITextCompletion with an empty api_key"""
with pytest.raises(ServiceInitializationError, match="The API key is required when use_vertexai is False."):
GoogleAITextCompletion(env_file_path="fake_env_file_path.env")
@pytest.mark.parametrize("exclude_list", ["GOOGLE_AI_CLOUD_PROJECT_ID"], indirect=True)
def test_google_ai_text_completion_init_with_use_vertexai_missing_project_id(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAITextCompletion with use_vertexai true but missing project_id"""
with pytest.raises(ServiceInitializationError, match="Project ID must be provided when use_vertexai is True."):
GoogleAITextCompletion(use_vertexai=True, env_file_path="fake_env_file_path.env")
@pytest.mark.parametrize("exclude_list", [["GOOGLE_AI_CLOUD_REGION"]], indirect=True)
def test_google_ai_text_completion_init_with_use_vertexai_missing_region(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAITextCompletion with use_vertexai true but missing region"""
with pytest.raises(ServiceInitializationError, match="Region must be provided when use_vertexai is True."):
GoogleAITextCompletion(use_vertexai=True, env_file_path="fake_env_file_path.env")
@pytest.mark.parametrize("exclude_list", [["GOOGLE_AI_API_KEY"]], indirect=True)
def test_google_ai_text_completion_init_with_use_vertexai_no_api_key(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAITextCompletion succeeds with use_vertexai=True and no api_key"""
text_completion = GoogleAITextCompletion(use_vertexai=True)
assert text_completion.service_settings.use_vertexai is True
def test_prompt_execution_settings_class(google_ai_unit_test_env) -> None:
google_ai_text_completion = GoogleAITextCompletion()
assert google_ai_text_completion.get_prompt_execution_settings_class() == GoogleAITextPromptExecutionSettings
# endregion init
# region text completion
@patch.object(AsyncModels, "generate_content", new_callable=AsyncMock)
async def test_google_ai_text_completion(
mock_google_model_generate_content,
google_ai_unit_test_env,
prompt: str,
mock_google_ai_text_completion_response,
) -> None:
"""Test text completion with GoogleAITextCompletion"""
settings = GoogleAITextPromptExecutionSettings()
mock_google_model_generate_content.return_value = mock_google_ai_text_completion_response
google_ai_text_completion = GoogleAITextCompletion()
responses: list[TextContent] = await google_ai_text_completion.get_text_contents(prompt, settings)
mock_google_model_generate_content.assert_called_once_with(
model=google_ai_text_completion.service_settings.gemini_model_id,
contents=prompt,
config=GenerateContentConfigDict(**settings.prepare_settings_dict()),
)
assert len(responses) == 1
assert responses[0].text == mock_google_ai_text_completion_response.candidates[0].content.parts[0].text
assert "usage" in responses[0].metadata
assert "prompt_feedback" in responses[0].metadata
assert responses[0].inner_content == mock_google_ai_text_completion_response
async def test_google_ai_text_completion_with_custom_client(
prompt: str,
google_ai_unit_test_env,
mock_google_ai_text_completion_response,
) -> None:
"""Test text completion with GoogleAITextCompletion using a custom client"""
# Create a custom client with a fake API key for testing
custom_client = Client(api_key="fake-api-key-for-testing")
# Mock the custom client's generate_content method
mock_generate_content = AsyncMock(return_value=mock_google_ai_text_completion_response)
custom_client.aio.models.generate_content = mock_generate_content
google_ai_text_completion = GoogleAITextCompletion(client=custom_client)
responses: list[TextContent] = await google_ai_text_completion.get_text_contents(
prompt,
GoogleAITextPromptExecutionSettings(),
)
custom_client.close()
# Verify that the custom client was used and returned the expected response
assert len(responses) == 1
assert responses[0].text == mock_google_ai_text_completion_response.candidates[0].content.parts[0].text
assert "usage" in responses[0].metadata
assert "prompt_feedback" in responses[0].metadata
assert responses[0].inner_content == mock_google_ai_text_completion_response
# Verify that the custom client's method was called
mock_generate_content.assert_called_once()
# endregion text completion
# region streaming text completion
@patch.object(AsyncModels, "generate_content_stream", new_callable=AsyncMock)
async def test_google_ai_streaming_text_completion(
mock_google_model_generate_content_stream,
google_ai_unit_test_env,
prompt: str,
mock_google_ai_streaming_text_completion_response,
) -> None:
"""Test streaming text completion with GoogleAITextCompletion"""
settings = GoogleAITextPromptExecutionSettings()
mock_google_model_generate_content_stream.return_value = mock_google_ai_streaming_text_completion_response
google_ai_text_completion = GoogleAITextCompletion()
async for chunks in google_ai_text_completion.get_streaming_text_contents(prompt, settings):
assert len(chunks) == 1
assert "usage" in chunks[0].metadata
assert "prompt_feedback" in chunks[0].metadata
mock_google_model_generate_content_stream.assert_called_once_with(
model=google_ai_text_completion.service_settings.gemini_model_id,
contents=prompt,
config=GenerateContentConfigDict(**settings.prepare_settings_dict()),
)
async def test_google_ai_streaming_text_completion_with_custom_client(
prompt: str,
google_ai_unit_test_env,
mock_google_ai_streaming_text_completion_response,
) -> None:
"""Test streaming text completion with GoogleAITextCompletion using a custom client"""
# Create a custom client with a fake API key for testing
custom_client = Client(api_key="fake-api-key-for-testing")
# Mock the custom client's generate_content_stream method
mock_generate_content_stream = AsyncMock(return_value=mock_google_ai_streaming_text_completion_response)
custom_client.aio.models.generate_content_stream = mock_generate_content_stream
google_ai_text_completion = GoogleAITextCompletion(client=custom_client)
async for chunks in google_ai_text_completion.get_streaming_text_contents(
prompt, GoogleAITextPromptExecutionSettings()
):
assert len(chunks) == 1
assert "usage" in chunks[0].metadata
assert "prompt_feedback" in chunks[0].metadata
custom_client.close()
# Verify that the custom client's method was called
mock_generate_content_stream.assert_called_once()
# endregion streaming text completion
@@ -0,0 +1,237 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, patch
import pytest
from google.genai import Client
from google.genai.models import AsyncModels
from google.genai.types import ContentEmbedding, EmbedContentConfigDict, EmbedContentResponse
from numpy import array, ndarray
from semantic_kernel.connectors.ai.google.google_ai.google_ai_prompt_execution_settings import (
GoogleAIEmbeddingPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.google.google_ai.google_ai_settings import GoogleAISettings
from semantic_kernel.connectors.ai.google.google_ai.services.google_ai_text_embedding import GoogleAITextEmbedding
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
# region init
def test_google_ai_text_embedding_init(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAITextEmbedding"""
model_id = google_ai_unit_test_env["GOOGLE_AI_EMBEDDING_MODEL_ID"]
api_key = google_ai_unit_test_env["GOOGLE_AI_API_KEY"]
google_ai_text_embedding = GoogleAITextEmbedding()
assert google_ai_text_embedding.ai_model_id == model_id
assert google_ai_text_embedding.service_id == model_id
assert isinstance(google_ai_text_embedding.service_settings, GoogleAISettings)
assert google_ai_text_embedding.service_settings.embedding_model_id == model_id
assert google_ai_text_embedding.service_settings.api_key.get_secret_value() == api_key
def test_google_ai_text_embedding_init_with_service_id(google_ai_unit_test_env, service_id) -> None:
"""Test initialization of GoogleAITextEmbedding with a service_id that is not the model_id"""
google_ai_text_embedding = GoogleAITextEmbedding(service_id=service_id)
assert google_ai_text_embedding.service_id == service_id
def test_google_ai_text_embedding_init_with_model_id_in_argument(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAITextEmbedding with model_id in argument"""
google_ai_chat_completion = GoogleAITextEmbedding(embedding_model_id="custom_model_id")
assert google_ai_chat_completion.ai_model_id == "custom_model_id"
assert google_ai_chat_completion.service_id == "custom_model_id"
@pytest.mark.parametrize("exclude_list", [["GOOGLE_AI_EMBEDDING_MODEL_ID"]], indirect=True)
def test_google_ai_text_embedding_init_with_empty_model_id(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAITextEmbedding with an empty model_id"""
with pytest.raises(ServiceInitializationError, match="The Google AI embedding model ID is required."):
GoogleAITextEmbedding(env_file_path="fake_env_file_path.env")
@pytest.mark.parametrize("exclude_list", [["GOOGLE_AI_API_KEY"]], indirect=True)
def test_google_ai_text_embedding_init_with_empty_api_key(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAITextEmbedding with an empty api_key"""
with pytest.raises(ServiceInitializationError, match="The API key is required when use_vertexai is False."):
GoogleAITextEmbedding(env_file_path="fake_env_file_path.env")
@pytest.mark.parametrize("exclude_list", [["GOOGLE_AI_API_KEY"]], indirect=True)
def test_google_ai_text_embedding_init_with_use_vertexai_no_api_key(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAITextEmbedding succeeds with use_vertexai=True and no api_key"""
embedding = GoogleAITextEmbedding(use_vertexai=True)
assert embedding.service_settings.use_vertexai is True
@pytest.mark.parametrize("exclude_list", [["GOOGLE_AI_CLOUD_PROJECT_ID"]], indirect=True)
def test_google_ai_text_embedding_init_with_use_vertexai_missing_project_id(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAITextEmbedding with use_vertexai true but missing project ID"""
with pytest.raises(ServiceInitializationError, match="Project ID must be provided when use_vertexai is True."):
GoogleAITextEmbedding(use_vertexai=True, env_file_path="fake_env_file_path.env")
@pytest.mark.parametrize("exclude_list", [["GOOGLE_AI_CLOUD_REGION"]], indirect=True)
def test_google_ai_text_embedding_init_with_use_vertexai_missing_region(google_ai_unit_test_env) -> None:
"""Test initialization of GoogleAITextEmbedding with use_vertexai true but missing region"""
with pytest.raises(ServiceInitializationError, match="Region must be provided when use_vertexai is True."):
GoogleAITextEmbedding(use_vertexai=True, env_file_path="fake_env_file_path.env")
def test_prompt_execution_settings_class(google_ai_unit_test_env) -> None:
google_ai_text_embedding = GoogleAITextEmbedding()
assert google_ai_text_embedding.get_prompt_execution_settings_class() == GoogleAIEmbeddingPromptExecutionSettings
# endregion init
@patch.object(AsyncModels, "embed_content", new_callable=AsyncMock)
async def test_embedding(mock_google_model_embed_content, google_ai_unit_test_env, prompt):
"""Test that the service initializes and generates embeddings correctly."""
model_id = google_ai_unit_test_env["GOOGLE_AI_EMBEDDING_MODEL_ID"]
mock_google_model_embed_content.return_value = EmbedContentResponse(
embeddings=[ContentEmbedding(values=[0.1, 0.2, 0.3])]
)
settings = GoogleAIEmbeddingPromptExecutionSettings()
google_ai_text_embedding = GoogleAITextEmbedding()
response: ndarray = await google_ai_text_embedding.generate_embeddings(
[prompt],
settings=settings,
)
assert len(response) == 1
assert response.all() == array([0.1, 0.2, 0.3]).all()
mock_google_model_embed_content.assert_called_once_with(
model=model_id,
contents=[prompt],
config=EmbedContentConfigDict(output_dimensionality=settings.output_dimensionality),
)
@patch.object(AsyncModels, "embed_content", new_callable=AsyncMock)
async def test_embedding_with_settings(mock_google_model_embed_content, google_ai_unit_test_env, prompt):
"""Test that the service initializes and generates embeddings correctly."""
model_id = google_ai_unit_test_env["GOOGLE_AI_EMBEDDING_MODEL_ID"]
mock_google_model_embed_content.return_value = EmbedContentResponse(
embeddings=[ContentEmbedding(values=[0.1, 0.2, 0.3])]
)
settings = GoogleAIEmbeddingPromptExecutionSettings()
settings.output_dimensionality = 3
google_ai_text_embedding = GoogleAITextEmbedding()
response: ndarray = await google_ai_text_embedding.generate_embeddings(
[prompt],
settings=settings,
)
assert len(response) == 1
assert response.all() == array([0.1, 0.2, 0.3]).all()
mock_google_model_embed_content.assert_called_once_with(
model=model_id,
contents=[prompt],
config=EmbedContentConfigDict(**settings.prepare_settings_dict()),
)
@patch.object(AsyncModels, "embed_content", new_callable=AsyncMock)
async def test_embedding_without_settings(mock_google_model_embed_content, google_ai_unit_test_env, prompt):
"""Test that the service initializes and generates embeddings correctly without settings."""
model_id = google_ai_unit_test_env["GOOGLE_AI_EMBEDDING_MODEL_ID"]
mock_google_model_embed_content.return_value = EmbedContentResponse(
embeddings=[ContentEmbedding(values=[0.1, 0.2, 0.3])]
)
google_ai_text_embedding = GoogleAITextEmbedding()
response: ndarray = await google_ai_text_embedding.generate_embeddings([prompt])
assert len(response) == 1
assert response.all() == array([0.1, 0.2, 0.3]).all()
mock_google_model_embed_content.assert_called_once_with(
model=model_id,
contents=[prompt],
config=EmbedContentConfigDict(output_dimensionality=None),
)
@patch.object(AsyncModels, "embed_content", new_callable=AsyncMock)
async def test_embedding_list_input(mock_google_model_embed_content, google_ai_unit_test_env, prompt):
"""Test that the service initializes and generates embeddings correctly with a list of prompts."""
model_id = google_ai_unit_test_env["GOOGLE_AI_EMBEDDING_MODEL_ID"]
mock_google_model_embed_content.return_value = EmbedContentResponse(
embeddings=[ContentEmbedding(values=[0.1, 0.2, 0.3]), ContentEmbedding(values=[0.1, 0.2, 0.3])]
)
settings = GoogleAIEmbeddingPromptExecutionSettings()
google_ai_text_embedding = GoogleAITextEmbedding()
response: ndarray = await google_ai_text_embedding.generate_embeddings(
[prompt, prompt],
settings=settings,
)
assert len(response) == 2
assert response.all() == array([[0.1, 0.2, 0.3], [0.1, 0.2, 0.3]]).all()
mock_google_model_embed_content.assert_called_once_with(
model=model_id,
contents=[prompt, prompt],
config=EmbedContentConfigDict(output_dimensionality=settings.output_dimensionality),
)
@patch.object(AsyncModels, "embed_content", new_callable=AsyncMock)
async def test_raw_embedding(mock_google_model_embed_content, google_ai_unit_test_env, prompt):
"""Test that the service initializes and generates embeddings correctly."""
model_id = google_ai_unit_test_env["GOOGLE_AI_EMBEDDING_MODEL_ID"]
mock_google_model_embed_content.return_value = EmbedContentResponse(
embeddings=[ContentEmbedding(values=[0.1, 0.2, 0.3])]
)
settings = GoogleAIEmbeddingPromptExecutionSettings()
google_ai_text_embedding = GoogleAITextEmbedding()
response: list[list[float]] = await google_ai_text_embedding.generate_raw_embeddings(
[prompt],
settings=settings,
)
assert len(response) == 1
assert response[0] == [0.1, 0.2, 0.3]
mock_google_model_embed_content.assert_called_once_with(
model=model_id,
contents=[prompt],
config=EmbedContentConfigDict(output_dimensionality=settings.output_dimensionality),
)
async def test_embedding_with_custom_client(google_ai_unit_test_env, prompt) -> None:
"""Test embedding with GoogleAITextEmbedding using a custom client"""
# Create a custom client with a fake API key for testing
custom_client = Client(api_key="fake-api-key-for-testing")
# Mock the custom client's embed_content method
mock_embed_content = AsyncMock(
return_value=EmbedContentResponse(embeddings=[ContentEmbedding(values=[0.1, 0.2, 0.3])])
)
custom_client.aio.models.embed_content = mock_embed_content
google_ai_text_embedding = GoogleAITextEmbedding(client=custom_client)
response: list[list[float]] = await google_ai_text_embedding.generate_embeddings(
[prompt],
GoogleAIEmbeddingPromptExecutionSettings(),
)
custom_client.close()
# Verify that the custom client was used and returned the expected response
assert len(response) == 1
assert response.all() == array([0.1, 0.2, 0.3]).all()
# Verify that the custom client's method was called
mock_embed_content.assert_called_once()
@@ -0,0 +1,201 @@
# Copyright (c) Microsoft. All rights reserved.
import pytest
from google.genai.types import FinishReason, Part
from semantic_kernel.connectors.ai.google.google_ai.services.utils import (
finish_reason_from_google_ai_to_semantic_kernel,
format_assistant_message,
format_user_message,
kernel_function_metadata_to_google_ai_function_call_format,
)
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.function_call_content import FunctionCallContent
from semantic_kernel.contents.function_result_content import FunctionResultContent
from semantic_kernel.contents.image_content import ImageContent
from semantic_kernel.contents.text_content import TextContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.contents.utils.finish_reason import FinishReason as SemanticKernelFinishReason
from semantic_kernel.exceptions.service_exceptions import ServiceInvalidRequestError
from semantic_kernel.functions.kernel_function_metadata import KernelFunctionMetadata
from semantic_kernel.functions.kernel_parameter_metadata import KernelParameterMetadata
def test_finish_reason_from_google_ai_to_semantic_kernel():
"""Test finish_reason_from_google_ai_to_semantic_kernel."""
assert finish_reason_from_google_ai_to_semantic_kernel(FinishReason.STOP) == SemanticKernelFinishReason.STOP
assert finish_reason_from_google_ai_to_semantic_kernel(FinishReason.MAX_TOKENS) == SemanticKernelFinishReason.LENGTH
assert (
finish_reason_from_google_ai_to_semantic_kernel(FinishReason.SAFETY)
== SemanticKernelFinishReason.CONTENT_FILTER
)
assert finish_reason_from_google_ai_to_semantic_kernel(FinishReason.OTHER) is None
assert finish_reason_from_google_ai_to_semantic_kernel(None) is None
def test_format_user_message():
"""Test format_user_message."""
user_message = ChatMessageContent(role=AuthorRole.USER, content="User message")
formatted_user_message = format_user_message(user_message)
assert len(formatted_user_message) == 1
assert isinstance(formatted_user_message[0], Part)
assert formatted_user_message[0].text == "User message"
# Test with an image content
image_content = ImageContent(data="image data", mime_type="image/png")
user_message = ChatMessageContent(
role=AuthorRole.USER,
items=[
TextContent(text="Text content"),
image_content,
],
)
formatted_user_message = format_user_message(user_message)
assert len(formatted_user_message) == 2
assert isinstance(formatted_user_message[0], Part)
assert formatted_user_message[0].text == "Text content"
assert isinstance(formatted_user_message[1], Part)
assert formatted_user_message[1].inline_data.mime_type == "image/png"
assert formatted_user_message[1].inline_data.data == image_content.data
def test_format_user_message_throws_with_unsupported_items() -> None:
"""Test format_user_message with unsupported items."""
# Test with unsupported items, any item other than TextContent and ImageContent should raise an error
user_message = ChatMessageContent(
role=AuthorRole.USER,
items=[
FunctionCallContent(),
],
)
with pytest.raises(ServiceInvalidRequestError):
format_user_message(user_message)
# Test with an ImageContent that has no data_uri
user_message = ChatMessageContent(
role=AuthorRole.USER,
items=[
ImageContent(data_uri=""),
],
)
with pytest.raises(ServiceInvalidRequestError):
format_user_message(user_message)
def test_format_assistant_message() -> None:
assistant_message = ChatMessageContent(
role=AuthorRole.ASSISTANT,
items=[
TextContent(text="test"),
FunctionCallContent(name="test_function", arguments={}),
ImageContent(data="image data", mime_type="image/png"),
],
)
formatted_assistant_message = format_assistant_message(assistant_message)
assert isinstance(formatted_assistant_message, list)
assert len(formatted_assistant_message) == 3
assert isinstance(formatted_assistant_message[0], Part)
assert formatted_assistant_message[0].text == "test"
assert isinstance(formatted_assistant_message[1], Part)
assert formatted_assistant_message[1].function_call.name == "test_function"
assert formatted_assistant_message[1].function_call.args == {}
assert isinstance(formatted_assistant_message[2], Part)
assert formatted_assistant_message[2].inline_data
def test_format_assistant_message_with_unsupported_items() -> None:
assistant_message = ChatMessageContent(
role=AuthorRole.ASSISTANT,
items=[
FunctionResultContent(id="test_id", function_name="test_function"),
],
)
with pytest.raises(ServiceInvalidRequestError):
format_assistant_message(assistant_message)
def test_format_assistant_message_with_thought_signature() -> None:
"""Test that thought_signature is preserved in function call parts."""
import base64
thought_sig = base64.b64encode(b"test_thought_signature_data")
assistant_message = ChatMessageContent(
role=AuthorRole.ASSISTANT,
items=[
FunctionCallContent(
name="test_function",
arguments={"arg1": "value1"},
metadata={"thought_signature": thought_sig},
),
],
)
formatted = format_assistant_message(assistant_message)
assert len(formatted) == 1
assert isinstance(formatted[0], Part)
assert formatted[0].function_call.name == "test_function"
assert formatted[0].function_call.args == {"arg1": "value1"}
assert formatted[0].thought_signature == thought_sig
def test_format_assistant_message_without_thought_signature() -> None:
"""Test that function calls without thought_signature still work."""
assistant_message = ChatMessageContent(
role=AuthorRole.ASSISTANT,
items=[
FunctionCallContent(
name="test_function",
arguments={"arg1": "value1"},
),
],
)
formatted = format_assistant_message(assistant_message)
assert len(formatted) == 1
assert isinstance(formatted[0], Part)
assert formatted[0].function_call.name == "test_function"
assert formatted[0].function_call.args == {"arg1": "value1"}
assert not getattr(formatted[0], "thought_signature", None)
def test_google_ai_function_call_format_sanitizes_anyof_schema() -> None:
"""Integration test: anyOf in param schema_data is sanitized in the output dict."""
metadata = KernelFunctionMetadata(
name="test_func",
description="A test function",
is_prompt=False,
parameters=[
KernelParameterMetadata(
name="messages",
description="The user messages",
is_required=True,
schema_data={
"anyOf": [
{"type": "string"},
{"type": "array", "items": {"type": "string"}},
],
"description": "The user messages",
},
),
],
)
result = kernel_function_metadata_to_google_ai_function_call_format(metadata)
param_schema = result["parameters"]["properties"]["messages"]
assert "anyOf" not in param_schema
assert param_schema["type"] == "string"
def test_google_ai_function_call_format_empty_parameters() -> None:
"""Integration test: metadata with no parameters produces parameters=None."""
metadata = KernelFunctionMetadata(
name="no_params_func",
description="No parameters",
is_prompt=False,
parameters=[],
)
result = kernel_function_metadata_to_google_ai_function_call_format(metadata)
assert result["parameters"] is None
@@ -0,0 +1,133 @@
# Copyright (c) Microsoft. All rights reserved.
from semantic_kernel.connectors.ai.google.google_ai import (
GoogleAIChatPromptExecutionSettings,
GoogleAIEmbeddingPromptExecutionSettings,
GoogleAIPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
def test_default_google_ai_prompt_execution_settings():
settings = GoogleAIPromptExecutionSettings()
assert settings.stop_sequences is None
assert settings.response_mime_type is None
assert settings.response_schema is None
assert settings.candidate_count is None
assert settings.max_output_tokens is None
assert settings.temperature is None
assert settings.top_p is None
assert settings.top_k is None
def test_custom_google_ai_prompt_execution_settings():
settings = GoogleAIPromptExecutionSettings(
stop_sequences=["world"],
response_mime_type="text/plain",
candidate_count=1,
max_output_tokens=128,
temperature=0.5,
top_p=0.5,
top_k=10,
)
assert settings.stop_sequences == ["world"]
assert settings.response_mime_type == "text/plain"
assert settings.candidate_count == 1
assert settings.max_output_tokens == 128
assert settings.temperature == 0.5
assert settings.top_p == 0.5
assert settings.top_k == 10
def test_google_ai_prompt_execution_settings_from_default_completion_config():
settings = PromptExecutionSettings(service_id="test_service")
chat_settings = GoogleAIChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.service_id == "test_service"
assert chat_settings.stop_sequences is None
assert chat_settings.response_mime_type is None
assert chat_settings.response_schema is None
assert chat_settings.candidate_count is None
assert chat_settings.max_output_tokens is None
assert chat_settings.temperature is None
assert chat_settings.top_p is None
assert chat_settings.top_k is None
def test_google_ai_prompt_execution_settings_from_openai_prompt_execution_settings():
chat_settings = GoogleAIChatPromptExecutionSettings(service_id="test_service", temperature=1.0)
new_settings = GoogleAIPromptExecutionSettings(service_id="test_2", temperature=0.0)
chat_settings.update_from_prompt_execution_settings(new_settings)
assert chat_settings.service_id == "test_2"
assert chat_settings.temperature == 0.0
def test_google_ai_prompt_execution_settings_from_custom_completion_config():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"stop_sequences": ["world"],
"response_mime_type": "text/plain",
"candidate_count": 1,
"max_output_tokens": 128,
"temperature": 0.5,
"top_p": 0.5,
"top_k": 10,
},
)
chat_settings = GoogleAIChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.stop_sequences == ["world"]
assert chat_settings.response_mime_type == "text/plain"
assert chat_settings.candidate_count == 1
assert chat_settings.max_output_tokens == 128
assert chat_settings.temperature == 0.5
assert chat_settings.top_p == 0.5
assert chat_settings.top_k == 10
def test_google_ai_chat_prompt_execution_settings_from_custom_completion_config_with_functions():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"tools": [{"function": {}}],
},
)
chat_settings = GoogleAIChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.tools == [{"function": {}}]
def test_create_options():
settings = GoogleAIChatPromptExecutionSettings(
service_id="test_service",
extension_data={
"stop_sequences": ["world"],
"response_mime_type": "text/plain",
"candidate_count": 1,
"max_output_tokens": 128,
"temperature": 0.5,
"top_p": 0.5,
"top_k": 10,
},
)
options = settings.prepare_settings_dict()
assert options["stop_sequences"] == ["world"]
assert options["response_mime_type"] == "text/plain"
assert options["candidate_count"] == 1
assert options["max_output_tokens"] == 128
assert options["temperature"] == 0.5
assert options["top_p"] == 0.5
assert options["top_k"] == 10
assert "tools" not in options
assert "tool_config" not in options
def test_default_google_ai_embedding_prompt_execution_settings():
settings = GoogleAIEmbeddingPromptExecutionSettings()
assert settings.output_dimensionality is None
@@ -0,0 +1,268 @@
# Copyright (c) Microsoft. All rights reserved.
import pytest
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType
from semantic_kernel.connectors.ai.google.shared_utils import (
FUNCTION_CHOICE_TYPE_TO_GOOGLE_FUNCTION_CALLING_MODE,
GEMINI_FUNCTION_NAME_SEPARATOR,
collapse_function_call_results_in_chat_history,
filter_system_message,
format_gemini_function_name_to_kernel_function_fully_qualified_name,
sanitize_schema_for_google_ai,
)
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.function_call_content import FunctionCallContent
from semantic_kernel.contents.function_result_content import FunctionResultContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions.service_exceptions import ServiceInvalidRequestError
def test_first_system_message():
"""Test filter_system_message."""
# Test with a single system message
chat_history = ChatHistory()
chat_history.add_system_message("System message")
chat_history.add_user_message("User message")
assert filter_system_message(chat_history) == "System message"
# Test with no system message
chat_history = ChatHistory()
chat_history.add_user_message("User message")
assert filter_system_message(chat_history) is None
# Test with multiple system messages
chat_history = ChatHistory()
chat_history.add_system_message("System message 1")
chat_history.add_system_message("System message 2")
with pytest.raises(ServiceInvalidRequestError):
filter_system_message(chat_history)
def test_function_choice_type_to_google_function_calling_mode_contain_all_types() -> None:
assert FunctionChoiceType.AUTO in FUNCTION_CHOICE_TYPE_TO_GOOGLE_FUNCTION_CALLING_MODE
assert FunctionChoiceType.NONE in FUNCTION_CHOICE_TYPE_TO_GOOGLE_FUNCTION_CALLING_MODE
assert FunctionChoiceType.REQUIRED in FUNCTION_CHOICE_TYPE_TO_GOOGLE_FUNCTION_CALLING_MODE
def test_format_gemini_function_name_to_kernel_function_fully_qualified_name() -> None:
# Contains the separator
gemini_function_name = f"plugin{GEMINI_FUNCTION_NAME_SEPARATOR}function"
assert (
format_gemini_function_name_to_kernel_function_fully_qualified_name(gemini_function_name) == "plugin-function"
)
# Doesn't contain the separator
gemini_function_name = "function"
assert format_gemini_function_name_to_kernel_function_fully_qualified_name(gemini_function_name) == "function"
def test_collapse_function_call_results_in_chat_history() -> None:
chat_history = ChatHistory()
chat_history.extend([
ChatMessageContent(
AuthorRole.ASSISTANT,
items=[
FunctionCallContent(id="function1", name="function1"),
FunctionCallContent(id="function2", name="function2"),
],
),
# The following two messages should be collapsed into a single message
ChatMessageContent(
AuthorRole.TOOL,
items=[FunctionResultContent(id="function1", name="function1", result="result1")],
),
ChatMessageContent(
AuthorRole.TOOL,
items=[FunctionResultContent(id="function2", name="function2", result="result2")],
),
ChatMessageContent(AuthorRole.ASSISTANT, content="Assistant message"),
ChatMessageContent(AuthorRole.USER, content="User message"),
ChatMessageContent(
AuthorRole.ASSISTANT,
items=[FunctionCallContent(id="function3", name="function3")],
),
ChatMessageContent(
AuthorRole.TOOL,
items=[FunctionResultContent(id="function3", name="function3", result="result3")],
),
ChatMessageContent(AuthorRole.ASSISTANT, content="Assistant message"),
])
assert len(chat_history.messages) == 8
collapse_function_call_results_in_chat_history(chat_history)
assert len(chat_history.messages) == 7
assert len(chat_history.messages[1].items) == 2
# --- sanitize_schema_for_google_ai tests ---
def test_sanitize_schema_none():
"""Test that None input returns None."""
assert sanitize_schema_for_google_ai(None) is None
def test_sanitize_schema_simple_passthrough():
"""Test that a simple schema passes through unchanged."""
schema = {"type": "string", "description": "A name"}
result = sanitize_schema_for_google_ai(schema)
assert result == {"type": "string", "description": "A name"}
def test_sanitize_schema_type_as_list_with_null():
"""type: ["string", "null"] should become type: "string" + nullable: true."""
schema = {"type": ["string", "null"], "description": "Optional field"}
result = sanitize_schema_for_google_ai(schema)
assert result == {"type": "string", "nullable": True, "description": "Optional field"}
def test_sanitize_schema_type_as_list_without_null():
"""type: ["string", "integer"] should pick the first type."""
schema = {"type": ["string", "integer"]}
result = sanitize_schema_for_google_ai(schema)
assert result == {"type": "string"}
def test_sanitize_schema_anyof_with_null():
"""AnyOf with null variant should become the non-null type + nullable."""
schema = {
"anyOf": [{"type": "string"}, {"type": "null"}],
"description": "Optional param",
}
result = sanitize_schema_for_google_ai(schema)
assert result == {"type": "string", "nullable": True, "description": "Optional param"}
def test_sanitize_schema_anyof_without_null():
"""AnyOf without null should pick the first variant."""
schema = {
"anyOf": [
{"type": "string"},
{"type": "array", "items": {"type": "string"}},
],
}
result = sanitize_schema_for_google_ai(schema)
assert result == {"type": "string"}
def test_sanitize_schema_oneof():
"""OneOf should be handled the same as anyOf."""
schema = {
"oneOf": [{"type": "integer"}, {"type": "null"}],
}
result = sanitize_schema_for_google_ai(schema)
assert result == {"type": "integer", "nullable": True}
def test_sanitize_schema_nested_properties():
"""AnyOf inside nested properties should be sanitized recursively."""
schema = {
"type": "object",
"properties": {
"name": {"type": "string"},
"value": {"anyOf": [{"type": "number"}, {"type": "null"}]},
},
}
result = sanitize_schema_for_google_ai(schema)
assert result == {
"type": "object",
"properties": {
"name": {"type": "string"},
"value": {"type": "number", "nullable": True},
},
}
def test_sanitize_schema_nested_items():
"""AnyOf inside array items should be sanitized recursively."""
schema = {
"type": "array",
"items": {"anyOf": [{"type": "string"}, {"type": "integer"}]},
}
result = sanitize_schema_for_google_ai(schema)
assert result == {
"type": "array",
"items": {"type": "string"},
}
def test_sanitize_schema_does_not_mutate_original():
"""The original schema dict should not be modified."""
schema = {
"anyOf": [{"type": "string"}, {"type": "null"}],
"description": "test",
}
original = {"anyOf": [{"type": "string"}, {"type": "null"}], "description": "test"}
sanitize_schema_for_google_ai(schema)
assert schema == original
def test_sanitize_schema_agent_messages_param():
"""Reproducer for issue #12442: str | list[str] parameter schema."""
schema = {
"anyOf": [
{"type": "string"},
{"type": "array", "items": {"type": "string"}},
],
"description": "The user messages for the agent.",
}
result = sanitize_schema_for_google_ai(schema)
assert "anyOf" not in result
assert result["type"] == "string"
assert result["description"] == "The user messages for the agent."
def test_sanitize_schema_allof():
"""AllOf should be handled like anyOf/oneOf, picking the first variant."""
schema = {
"allOf": [
{"type": "object", "properties": {"name": {"type": "string"}}},
{"type": "object", "properties": {"age": {"type": "integer"}}},
],
}
result = sanitize_schema_for_google_ai(schema)
assert "allOf" not in result
assert result["type"] == "object"
assert "name" in result["properties"]
def test_sanitize_schema_allof_with_null():
"""AllOf with a null variant should produce nullable: true."""
schema = {
"allOf": [{"type": "string"}, {"type": "null"}],
}
result = sanitize_schema_for_google_ai(schema)
assert "allOf" not in result
assert result["type"] == "string"
assert result["nullable"] is True
def test_sanitize_schema_all_null_type_list():
"""type: ["null"] should fall back to type: "string" + nullable: true."""
schema = {"type": ["null"]}
result = sanitize_schema_for_google_ai(schema)
assert result == {"type": "string", "nullable": True}
def test_sanitize_schema_all_null_anyof():
"""AnyOf where all variants are null should fall back to type: "string"."""
schema = {"anyOf": [{"type": "null"}]}
result = sanitize_schema_for_google_ai(schema)
assert result == {"type": "string", "nullable": True}
def test_sanitize_schema_chosen_variant_keeps_own_description():
"""When the chosen anyOf variant has its own description, do not overwrite it."""
schema = {
"anyOf": [
{"type": "string", "description": "inner desc"},
{"type": "null"},
],
"description": "outer desc",
}
result = sanitize_schema_for_google_ai(schema)
assert result["description"] == "inner desc"
assert result["nullable"] is True
@@ -0,0 +1,189 @@
# Copyright (c) Microsoft. All rights reserved.
from collections.abc import AsyncGenerator, AsyncIterable
from unittest.mock import MagicMock
import pytest
from google.cloud.aiplatform_v1beta1.types.content import Candidate, Content, Part
from google.cloud.aiplatform_v1beta1.types.prediction_service import GenerateContentResponse
from google.cloud.aiplatform_v1beta1.types.tool import FunctionCall
from vertexai.generative_models import GenerationResponse
from vertexai.language_models import TextEmbedding
@pytest.fixture()
def vertex_ai_unit_test_env(monkeypatch, exclude_list, override_env_param_dict):
"""Fixture to set environment variables for Vertex AI Unit Tests."""
if exclude_list is None:
exclude_list = []
if override_env_param_dict is None:
override_env_param_dict = {}
env_vars = {
"VERTEX_AI_GEMINI_MODEL_ID": "test-gemini-model-id",
"VERTEX_AI_EMBEDDING_MODEL_ID": "test-embedding-model-id",
"VERTEX_AI_PROJECT_ID": "test-project-id",
}
env_vars.update(override_env_param_dict)
for key, value in env_vars.items():
if key not in exclude_list:
monkeypatch.setenv(key, value)
else:
monkeypatch.delenv(key, raising=False)
return env_vars
@pytest.fixture()
def mock_vertex_ai_chat_completion_response() -> GenerationResponse:
"""Mock Vertex AI Chat Completion response."""
candidate = Candidate()
candidate.index = 0
candidate.content = Content(role="user", parts=[Part(text="Test content")])
candidate.finish_reason = Candidate.FinishReason.STOP
response = GenerateContentResponse()
response.candidates.append(candidate)
response.usage_metadata = GenerateContentResponse.UsageMetadata(
prompt_token_count=0,
candidates_token_count=0,
total_token_count=0,
)
return GenerationResponse._from_gapic(response)
@pytest.fixture()
def mock_vertex_ai_chat_completion_response_with_tool_call() -> GenerationResponse:
"""Mock Vertex AI Chat Completion response."""
candidate = Candidate()
candidate.index = 0
candidate.content = Content(
role="user",
parts=[
Part(
function_call=FunctionCall(
name="test_function",
args={"test_arg": "test_value"},
)
)
],
)
candidate.finish_reason = Candidate.FinishReason.STOP
response = GenerateContentResponse()
response.candidates.append(candidate)
response.usage_metadata = GenerateContentResponse.UsageMetadata(
prompt_token_count=0,
candidates_token_count=0,
total_token_count=0,
)
return GenerationResponse._from_gapic(response)
@pytest.fixture()
def mock_vertex_ai_streaming_chat_completion_response() -> AsyncIterable[GenerationResponse]:
"""Mock Vertex AI streaming Chat Completion response."""
candidate = Candidate()
candidate.index = 0
candidate.content = Content(role="user", parts=[Part(text="Test content")])
candidate.finish_reason = Candidate.FinishReason.STOP
response = GenerateContentResponse()
response.candidates.append(candidate)
response.usage_metadata = GenerateContentResponse.UsageMetadata(
prompt_token_count=0,
candidates_token_count=0,
total_token_count=0,
)
iterable = MagicMock(spec=AsyncGenerator)
iterable.__aiter__.return_value = [GenerationResponse._from_gapic(response)]
return iterable
@pytest.fixture()
def mock_vertex_ai_streaming_chat_completion_response_with_tool_call() -> AsyncIterable[GenerationResponse]:
"""Mock Vertex AI streaming Chat Completion response."""
candidate = Candidate()
candidate.index = 0
candidate.content = Content(
role="user",
parts=[
Part(
function_call=FunctionCall(
name="getLightStatus",
args={"arg1": "test_value"},
)
)
],
)
candidate.finish_reason = Candidate.FinishReason.STOP
response = GenerateContentResponse()
response.candidates.append(candidate)
response.usage_metadata = GenerateContentResponse.UsageMetadata(
prompt_token_count=0,
candidates_token_count=0,
total_token_count=0,
)
iterable = MagicMock(spec=AsyncGenerator)
iterable.__aiter__.return_value = [GenerationResponse._from_gapic(response)]
return iterable
@pytest.fixture()
def mock_vertex_ai_text_completion_response() -> GenerationResponse:
"""Mock Vertex AI Text Completion response."""
candidate = Candidate()
candidate.index = 0
candidate.content = Content(parts=[Part(text="Test content")])
response = GenerateContentResponse()
response.candidates.append(candidate)
response.usage_metadata = GenerateContentResponse.UsageMetadata(
prompt_token_count=0,
candidates_token_count=0,
total_token_count=0,
)
return GenerationResponse._from_gapic(response)
@pytest.fixture()
def mock_vertex_ai_streaming_text_completion_response() -> AsyncIterable[GenerationResponse]:
"""Mock Vertex AI streaming Text Completion response."""
candidate = Candidate()
candidate.index = 0
candidate.content = Content(parts=[Part(text="Test content")])
response = GenerateContentResponse()
response.candidates.append(candidate)
response.usage_metadata = GenerateContentResponse.UsageMetadata(
prompt_token_count=0,
candidates_token_count=0,
total_token_count=0,
)
iterable = MagicMock(spec=AsyncGenerator)
iterable.__aiter__.return_value = [GenerationResponse._from_gapic(response)]
return iterable
class MockTextEmbeddingModel:
async def get_embeddings_async(
self,
texts: list[str],
*,
auto_truncate: bool = True,
output_dimensionality: int | None = None,
) -> list[TextEmbedding]:
pass
@@ -0,0 +1,545 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, patch
import pytest
from vertexai.generative_models import Content, GenerativeModel
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
from semantic_kernel.connectors.ai.google.vertex_ai.services.vertex_ai_chat_completion import VertexAIChatCompletion
from semantic_kernel.connectors.ai.google.vertex_ai.vertex_ai_prompt_execution_settings import (
VertexAIChatPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.google.vertex_ai.vertex_ai_settings import VertexAISettings
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.utils.finish_reason import FinishReason
from semantic_kernel.exceptions.service_exceptions import (
ServiceInitializationError,
ServiceInvalidExecutionSettingsError,
)
from semantic_kernel.kernel import Kernel
# region init
def test_vertex_ai_chat_completion_init(vertex_ai_unit_test_env) -> None:
"""Test initialization of VertexAIChatCompletion"""
model_id = vertex_ai_unit_test_env["VERTEX_AI_GEMINI_MODEL_ID"]
project_id = vertex_ai_unit_test_env["VERTEX_AI_PROJECT_ID"]
vertex_ai_chat_completion = VertexAIChatCompletion()
assert vertex_ai_chat_completion.ai_model_id == model_id
assert vertex_ai_chat_completion.service_id == model_id
assert isinstance(vertex_ai_chat_completion.service_settings, VertexAISettings)
assert vertex_ai_chat_completion.service_settings.gemini_model_id == model_id
assert vertex_ai_chat_completion.service_settings.project_id == project_id
def test_vertex_ai_chat_completion_init_with_service_id(vertex_ai_unit_test_env, service_id) -> None:
"""Test initialization of VertexAIChatCompletion with a service id that is not the model id"""
vertex_ai_chat_completion = VertexAIChatCompletion(service_id=service_id)
assert vertex_ai_chat_completion.service_id == service_id
def test_vertex_ai_chat_completion_init_with_model_id_in_argument(vertex_ai_unit_test_env) -> None:
"""Test initialization of VertexAIChatCompletion with model id in argument"""
vertex_ai_chat_completion = VertexAIChatCompletion(gemini_model_id="custom_model_id")
assert vertex_ai_chat_completion.ai_model_id == "custom_model_id"
assert vertex_ai_chat_completion.service_id == "custom_model_id"
@pytest.mark.parametrize("exclude_list", [["VERTEX_AI_GEMINI_MODEL_ID"]], indirect=True)
def test_vertex_ai_chat_completion_init_with_empty_model_id(vertex_ai_unit_test_env) -> None:
"""Test initialization of VertexAIChatCompletion with an empty model id"""
with pytest.raises(ServiceInitializationError):
VertexAIChatCompletion(env_file_path="fake_env_file_path.env")
@pytest.mark.parametrize("exclude_list", [["VERTEX_AI_PROJECT_ID"]], indirect=True)
def test_vertex_ai_chat_completion_init_with_empty_project_id(vertex_ai_unit_test_env) -> None:
"""Test initialization of VertexAIChatCompletion with an empty project id"""
with pytest.raises(ServiceInitializationError):
VertexAIChatCompletion(env_file_path="fake_env_file_path.env")
def test_prompt_execution_settings_class(vertex_ai_unit_test_env) -> None:
vertex_ai_chat_completion = VertexAIChatCompletion()
assert vertex_ai_chat_completion.get_prompt_execution_settings_class() == VertexAIChatPromptExecutionSettings
# endregion init
# region chat completion
@patch.object(GenerativeModel, "generate_content_async", new_callable=AsyncMock)
async def test_vertex_ai_chat_completion(
mock_vertex_ai_model_generate_content_async,
vertex_ai_unit_test_env,
chat_history: ChatHistory,
mock_vertex_ai_chat_completion_response,
) -> None:
"""Test chat completion with VertexAIChatCompletion"""
settings = VertexAIChatPromptExecutionSettings()
mock_vertex_ai_model_generate_content_async.return_value = mock_vertex_ai_chat_completion_response
vertex_ai_chat_completion = VertexAIChatCompletion()
responses: list[ChatMessageContent] = await vertex_ai_chat_completion.get_chat_message_contents(
chat_history, settings
)
# Verify the call was made once
mock_vertex_ai_model_generate_content_async.assert_called_once()
# Get the actual call arguments
call_args = mock_vertex_ai_model_generate_content_async.call_args
# Verify the contents
contents = call_args.kwargs["contents"]
assert len(contents) == 1
assert contents[0].role == "user"
assert len(contents[0].parts) == 1
assert contents[0].parts[0].text == "test_prompt"
# Verify other arguments
assert call_args.kwargs["generation_config"] == settings.prepare_settings_dict()
assert call_args.kwargs["tools"] is None
assert call_args.kwargs["tool_config"] is None
assert len(responses) == 1
assert responses[0].role == "assistant"
assert responses[0].content == mock_vertex_ai_chat_completion_response.candidates[0].content.parts[0].text
assert responses[0].finish_reason == FinishReason.STOP
assert "usage" in responses[0].metadata
assert "prompt_feedback" in responses[0].metadata
assert responses[0].inner_content == mock_vertex_ai_chat_completion_response
async def test_vertex_ai_chat_completion_with_function_choice_behavior_fail_verification(
chat_history: ChatHistory,
vertex_ai_unit_test_env,
) -> None:
"""Test completion of VertexAIChatCompletion with function choice behavior expect verification failure"""
# Missing kernel
with pytest.raises(ServiceInvalidExecutionSettingsError):
settings = VertexAIChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
vertex_ai_chat_completion = VertexAIChatCompletion()
await vertex_ai_chat_completion.get_chat_message_contents(
chat_history=chat_history,
settings=settings,
)
@patch.object(GenerativeModel, "generate_content_async", new_callable=AsyncMock)
async def test_vertex_ai_chat_completion_with_function_choice_behavior(
mock_vertex_ai_model_generate_content_async,
vertex_ai_unit_test_env,
kernel,
chat_history: ChatHistory,
mock_vertex_ai_chat_completion_response_with_tool_call,
) -> None:
"""Test completion of VertexAIChatCompletion with function choice behavior"""
mock_vertex_ai_model_generate_content_async.return_value = mock_vertex_ai_chat_completion_response_with_tool_call
settings = VertexAIChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
settings.function_choice_behavior.maximum_auto_invoke_attempts = 1
vertex_ai_chat_completion = VertexAIChatCompletion()
responses = await vertex_ai_chat_completion.get_chat_message_contents(
chat_history=chat_history,
settings=settings,
kernel=kernel,
)
# The function should be called twice:
# One for the tool call and one for the last completion
# after the maximum_auto_invoke_attempts is reached
assert mock_vertex_ai_model_generate_content_async.call_count == 2
assert len(responses) == 1
assert responses[0].role == "assistant"
# Google doesn't return STOP as the finish reason for tool calls
assert responses[0].finish_reason == FinishReason.STOP
@patch.object(GenerativeModel, "generate_content_async", new_callable=AsyncMock)
async def test_vertex_ai_chat_completion_with_function_choice_behavior_no_tool_call(
mock_vertex_ai_model_generate_content_async,
vertex_ai_unit_test_env,
kernel,
chat_history: ChatHistory,
mock_vertex_ai_chat_completion_response,
) -> None:
"""Test completion of VertexAIChatCompletion with function choice behavior but no tool call returned"""
mock_vertex_ai_model_generate_content_async.return_value = mock_vertex_ai_chat_completion_response
settings = VertexAIChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
settings.function_choice_behavior.maximum_auto_invoke_attempts = 1
vertex_ai_chat_completion = VertexAIChatCompletion()
responses = await vertex_ai_chat_completion.get_chat_message_contents(
chat_history=chat_history,
settings=settings,
kernel=kernel,
)
# Verify the call was made once
mock_vertex_ai_model_generate_content_async.assert_awaited_once()
# Get the actual call arguments
call_args = mock_vertex_ai_model_generate_content_async.await_args
# Verify the contents
contents = call_args.kwargs["contents"]
assert len(contents) == 1
assert contents[0].role == "user"
assert len(contents[0].parts) == 1
assert contents[0].parts[0].text == "test_prompt"
# Verify other arguments
assert call_args.kwargs["generation_config"] == settings.prepare_settings_dict()
assert call_args.kwargs["tools"] is None
assert call_args.kwargs["tool_config"] is None
assert len(responses) == 1
assert responses[0].role == "assistant"
assert responses[0].content == mock_vertex_ai_chat_completion_response.candidates[0].content.parts[0].text
# endregion chat completion
# region streaming chat completion
@patch.object(GenerativeModel, "generate_content_async", new_callable=AsyncMock)
async def test_vertex_ai_streaming_chat_completion(
mock_vertex_ai_model_generate_content_async,
vertex_ai_unit_test_env,
chat_history: ChatHistory,
mock_vertex_ai_streaming_chat_completion_response,
) -> None:
"""Test chat completion with VertexAIChatCompletion"""
settings = VertexAIChatPromptExecutionSettings()
mock_vertex_ai_model_generate_content_async.return_value = mock_vertex_ai_streaming_chat_completion_response
vertex_ai_chat_completion = VertexAIChatCompletion()
async for messages in vertex_ai_chat_completion.get_streaming_chat_message_contents(chat_history, settings):
assert len(messages) == 1
assert messages[0].role == "assistant"
assert messages[0].finish_reason == FinishReason.STOP
assert "usage" in messages[0].metadata
assert "prompt_feedback" in messages[0].metadata
# Verify the call was made once
mock_vertex_ai_model_generate_content_async.assert_called_once()
# Get the actual call arguments
call_args = mock_vertex_ai_model_generate_content_async.call_args
# Verify the contents
contents = call_args.kwargs["contents"]
assert len(contents) == 1
assert contents[0].role == "user"
assert len(contents[0].parts) == 1
assert contents[0].parts[0].text == "test_prompt"
# Verify other arguments
assert call_args.kwargs["generation_config"] == settings.prepare_settings_dict()
assert call_args.kwargs["tools"] is None
assert call_args.kwargs["tool_config"] is None
assert call_args.kwargs["stream"] is True
async def test_vertex_ai_streaming_chat_completion_with_function_choice_behavior_fail_verification(
chat_history: ChatHistory,
vertex_ai_unit_test_env,
) -> None:
"""Test streaming chat completion of VertexAIChatCompletion with function choice
behavior expect verification failure"""
# Missing kernel
with pytest.raises(ServiceInvalidExecutionSettingsError):
settings = VertexAIChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
vertex_ai_chat_completion = VertexAIChatCompletion()
async for _ in vertex_ai_chat_completion.get_streaming_chat_message_contents(
chat_history=chat_history,
settings=settings,
):
pass
@patch.object(GenerativeModel, "generate_content_async", new_callable=AsyncMock)
async def test_vertex_ai_streaming_chat_completion_with_function_choice_behavior(
mock_vertex_ai_model_generate_content_async,
vertex_ai_unit_test_env,
kernel: Kernel,
chat_history: ChatHistory,
mock_vertex_ai_streaming_chat_completion_response_with_tool_call,
decorated_native_function,
) -> None:
"""Test streaming chat completion of VertexAIChatCompletion with function choice behavior"""
mock_vertex_ai_model_generate_content_async.return_value = (
mock_vertex_ai_streaming_chat_completion_response_with_tool_call
)
settings = VertexAIChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
settings.function_choice_behavior.maximum_auto_invoke_attempts = 1
vertex_ai_chat_completion = VertexAIChatCompletion()
kernel.add_function(plugin_name="TestPlugin", function=decorated_native_function)
all_messages = []
async for messages in vertex_ai_chat_completion.get_streaming_chat_message_contents(
chat_history,
settings,
kernel=kernel,
):
all_messages.extend(messages)
assert len(all_messages) == 2, f"Expected 2 messages, got {len(all_messages)}"
# Validate the first message
assert all_messages[0].role == "assistant", f"Unexpected role for first message: {all_messages[0].role}"
assert all_messages[0].content == "", f"Unexpected content for first message: {all_messages[0].content}"
assert all_messages[0].finish_reason == FinishReason.STOP, (
f"Unexpected finish reason for first message: {all_messages[0].finish_reason}"
)
# Validate the second message
assert all_messages[1].role == "tool", f"Unexpected role for second message: {all_messages[1].role}"
assert all_messages[1].content == "", f"Unexpected content for second message: {all_messages[1].content}"
assert all_messages[1].finish_reason is None
@patch.object(GenerativeModel, "generate_content_async", new_callable=AsyncMock)
async def test_vertex_ai_streaming_chat_completion_with_function_choice_behavior_no_tool_call(
mock_vertex_ai_model_generate_content_async,
vertex_ai_unit_test_env,
kernel,
chat_history: ChatHistory,
mock_vertex_ai_streaming_chat_completion_response,
) -> None:
"""Test completion of VertexAIChatCompletion with function choice behavior but no tool call returned"""
mock_vertex_ai_model_generate_content_async.return_value = mock_vertex_ai_streaming_chat_completion_response
settings = VertexAIChatPromptExecutionSettings(
function_choice_behavior=FunctionChoiceBehavior.Auto(),
)
settings.function_choice_behavior.maximum_auto_invoke_attempts = 1
vertex_ai_chat_completion = VertexAIChatCompletion()
async for messages in vertex_ai_chat_completion.get_streaming_chat_message_contents(
chat_history=chat_history,
settings=settings,
kernel=kernel,
):
assert len(messages) == 1
assert messages[0].role == "assistant"
# Verify the call was made once
mock_vertex_ai_model_generate_content_async.assert_awaited_once()
# Get the actual call arguments
call_args = mock_vertex_ai_model_generate_content_async.await_args
# Verify the contents
contents = call_args.kwargs["contents"]
assert len(contents) == 1
assert contents[0].role == "user"
assert len(contents[0].parts) == 1
assert contents[0].parts[0].text == "test_prompt"
# Verify other arguments
assert call_args.kwargs["generation_config"] == settings.prepare_settings_dict()
assert call_args.kwargs["tools"] is None
assert call_args.kwargs["tool_config"] is None
assert call_args.kwargs["stream"] is True
# endregion streaming chat completion
def test_vertex_ai_chat_completion_parse_chat_history_correctly(vertex_ai_unit_test_env) -> None:
"""Test _prepare_chat_history_for_request method"""
vertex_ai_chat_completion = VertexAIChatCompletion()
chat_history = ChatHistory()
chat_history.add_system_message("test_system_message")
chat_history.add_user_message("test_user_message")
chat_history.add_assistant_message("test_assistant_message")
parsed_chat_history = vertex_ai_chat_completion._prepare_chat_history_for_request(chat_history)
assert isinstance(parsed_chat_history, list)
# System message should be ignored
assert len(parsed_chat_history) == 2
assert all(isinstance(message, Content) for message in parsed_chat_history)
assert parsed_chat_history[0].role == "user"
assert parsed_chat_history[0].parts[0].text == "test_user_message"
assert parsed_chat_history[1].role == "model"
assert parsed_chat_history[1].parts[0].text == "test_assistant_message"
# region thought_signature deserialization tests
def test_create_chat_message_content_with_thought_signature(vertex_ai_unit_test_env) -> None:
"""Test that thought_signature from a Part dict is deserialized into FunctionCallContent.metadata."""
from unittest.mock import MagicMock
from google.cloud.aiplatform_v1beta1.types.content import Candidate as GapicCandidate
from semantic_kernel.contents.function_call_content import FunctionCallContent
thought_sig_value = "vertex-test-thought-sig"
# Create a mock Part whose to_dict() returns thought_signature
mock_part = MagicMock()
mock_part.text = None
mock_part.to_dict.return_value = {
"function_call": {"name": "test_function", "args": {"key": "value"}},
"thought_signature": thought_sig_value,
}
mock_part.function_call.name = "test_function"
mock_part.function_call.args = {"key": "value"}
# Build a mock candidate with the mock part
mock_candidate = MagicMock()
mock_candidate.index = 0
mock_candidate.content.parts = [mock_part]
mock_candidate.finish_reason = GapicCandidate.FinishReason.STOP
# Build a mock response
mock_response = MagicMock()
completion = VertexAIChatCompletion()
result = completion._create_chat_message_content(mock_response, mock_candidate)
fc_items = [item for item in result.items if isinstance(item, FunctionCallContent)]
assert len(fc_items) == 1
assert fc_items[0].metadata is not None
assert fc_items[0].metadata["thought_signature"] == thought_sig_value
def test_create_chat_message_content_without_thought_signature(vertex_ai_unit_test_env) -> None:
"""Test that FunctionCallContent works when Part dict has no thought_signature."""
from unittest.mock import MagicMock
from google.cloud.aiplatform_v1beta1.types.content import Candidate as GapicCandidate
from semantic_kernel.contents.function_call_content import FunctionCallContent
mock_part = MagicMock()
mock_part.text = None
mock_part.to_dict.return_value = {
"function_call": {"name": "test_function", "args": {"key": "value"}},
}
mock_part.function_call.name = "test_function"
mock_part.function_call.args = {"key": "value"}
mock_candidate = MagicMock()
mock_candidate.index = 0
mock_candidate.content.parts = [mock_part]
mock_candidate.finish_reason = GapicCandidate.FinishReason.STOP
mock_response = MagicMock()
completion = VertexAIChatCompletion()
result = completion._create_chat_message_content(mock_response, mock_candidate)
fc_items = [item for item in result.items if isinstance(item, FunctionCallContent)]
assert len(fc_items) == 1
assert "thought_signature" not in fc_items[0].metadata
def test_create_streaming_chat_message_content_with_thought_signature(vertex_ai_unit_test_env) -> None:
"""Test that thought_signature from a Part dict is deserialized in streaming path."""
from unittest.mock import MagicMock
from google.cloud.aiplatform_v1beta1.types.content import Candidate as GapicCandidate
from semantic_kernel.contents.function_call_content import FunctionCallContent
thought_sig_value = "vertex-streaming-thought-sig"
mock_part = MagicMock()
mock_part.text = None
mock_part.to_dict.return_value = {
"function_call": {"name": "stream_func", "args": {"a": "b"}},
"thought_signature": thought_sig_value,
}
mock_part.function_call.name = "stream_func"
mock_part.function_call.args = {"a": "b"}
mock_candidate = MagicMock()
mock_candidate.index = 0
mock_candidate.content.parts = [mock_part]
mock_candidate.finish_reason = GapicCandidate.FinishReason.STOP
mock_chunk = MagicMock()
completion = VertexAIChatCompletion()
result = completion._create_streaming_chat_message_content(mock_chunk, mock_candidate, function_invoke_attempt=0)
fc_items = [item for item in result.items if isinstance(item, FunctionCallContent)]
assert len(fc_items) == 1
assert fc_items[0].metadata is not None
assert fc_items[0].metadata["thought_signature"] == thought_sig_value
def test_create_streaming_chat_message_content_without_thought_signature(vertex_ai_unit_test_env) -> None:
"""Test that streaming FunctionCallContent works when Part dict lacks thought_signature."""
from unittest.mock import MagicMock
from google.cloud.aiplatform_v1beta1.types.content import Candidate as GapicCandidate
from semantic_kernel.contents.function_call_content import FunctionCallContent
mock_part = MagicMock()
mock_part.text = None
mock_part.to_dict.return_value = {
"function_call": {"name": "stream_func", "args": {"a": "b"}},
}
mock_part.function_call.name = "stream_func"
mock_part.function_call.args = {"a": "b"}
mock_candidate = MagicMock()
mock_candidate.index = 0
mock_candidate.content.parts = [mock_part]
mock_candidate.finish_reason = GapicCandidate.FinishReason.STOP
mock_chunk = MagicMock()
completion = VertexAIChatCompletion()
result = completion._create_streaming_chat_message_content(mock_chunk, mock_candidate, function_invoke_attempt=0)
fc_items = [item for item in result.items if isinstance(item, FunctionCallContent)]
assert len(fc_items) == 1
assert "thought_signature" not in fc_items[0].metadata
# endregion thought_signature deserialization tests
@@ -0,0 +1,130 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, patch
import pytest
from vertexai.generative_models import GenerativeModel
from semantic_kernel.connectors.ai.google.vertex_ai.services.vertex_ai_text_completion import VertexAITextCompletion
from semantic_kernel.connectors.ai.google.vertex_ai.vertex_ai_prompt_execution_settings import (
VertexAITextPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.google.vertex_ai.vertex_ai_settings import VertexAISettings
from semantic_kernel.contents.text_content import TextContent
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
# region init
def test_vertex_ai_text_completion_init(vertex_ai_unit_test_env) -> None:
"""Test initialization of VertexAITextCompletion"""
model_id = vertex_ai_unit_test_env["VERTEX_AI_GEMINI_MODEL_ID"]
project_id = vertex_ai_unit_test_env["VERTEX_AI_PROJECT_ID"]
vertex_ai_text_completion = VertexAITextCompletion()
assert vertex_ai_text_completion.ai_model_id == model_id
assert vertex_ai_text_completion.service_id == model_id
assert isinstance(vertex_ai_text_completion.service_settings, VertexAISettings)
assert vertex_ai_text_completion.service_settings.gemini_model_id == model_id
assert vertex_ai_text_completion.service_settings.project_id == project_id
def test_vertex_ai_text_completion_init_with_service_id(vertex_ai_unit_test_env, service_id) -> None:
"""Test initialization of VertexAITextCompletion with a service id that is not the model id"""
vertex_ai_text_completion = VertexAITextCompletion(service_id=service_id)
assert vertex_ai_text_completion.service_id == service_id
def test_vertex_ai_text_completion_init_with_model_id_in_argument(vertex_ai_unit_test_env) -> None:
"""Test initialization of VertexAIChatCompletion with model id in argument"""
vertex_ai_text_completion = VertexAITextCompletion(gemini_model_id="custom_model_id")
assert vertex_ai_text_completion.ai_model_id == "custom_model_id"
assert vertex_ai_text_completion.service_id == "custom_model_id"
@pytest.mark.parametrize("exclude_list", [["VERTEX_AI_GEMINI_MODEL_ID"]], indirect=True)
def test_vertex_ai_text_completion_init_with_empty_model_id(vertex_ai_unit_test_env) -> None:
"""Test initialization of VertexAITextCompletion with an empty model id"""
with pytest.raises(ServiceInitializationError):
VertexAITextCompletion(env_file_path="fake_env_file_path.env")
@pytest.mark.parametrize("exclude_list", [["VERTEX_AI_PROJECT_ID"]], indirect=True)
def test_vertex_ai_text_completion_init_with_empty_project_id(vertex_ai_unit_test_env) -> None:
"""Test initialization of VertexAITextCompletion with an empty project id"""
with pytest.raises(ServiceInitializationError):
VertexAITextCompletion(env_file_path="fake_env_file_path.env")
def test_prompt_execution_settings_class(vertex_ai_unit_test_env) -> None:
vertex_ai_text_completion = VertexAITextCompletion()
assert vertex_ai_text_completion.get_prompt_execution_settings_class() == VertexAITextPromptExecutionSettings
# endregion init
# region text completion
@patch.object(GenerativeModel, "generate_content_async", new_callable=AsyncMock)
async def test_vertex_ai_text_completion(
mock_vertex_ai_model_generate_content_async,
vertex_ai_unit_test_env,
prompt: str,
mock_vertex_ai_text_completion_response,
) -> None:
"""Test text completion with VertexAITextCompletion"""
settings = VertexAITextPromptExecutionSettings()
mock_vertex_ai_model_generate_content_async.return_value = mock_vertex_ai_text_completion_response
vertex_ai_text_completion = VertexAITextCompletion()
responses: list[TextContent] = await vertex_ai_text_completion.get_text_contents(prompt, settings)
mock_vertex_ai_model_generate_content_async.assert_called_once_with(
contents=prompt,
generation_config=settings.prepare_settings_dict(),
)
assert len(responses) == 1
assert responses[0].text == mock_vertex_ai_text_completion_response.candidates[0].content.parts[0].text
assert "usage" in responses[0].metadata
assert "prompt_feedback" in responses[0].metadata
assert responses[0].inner_content == mock_vertex_ai_text_completion_response
# endregion text completion
# region streaming text completion
@patch.object(GenerativeModel, "generate_content_async", new_callable=AsyncMock)
async def test_vertex_ai_streaming_text_completion(
mock_vertex_ai_model_generate_content_async,
vertex_ai_unit_test_env,
prompt: str,
mock_vertex_ai_streaming_text_completion_response,
) -> None:
"""Test streaming text completion with VertexAITextCompletion"""
settings = VertexAITextPromptExecutionSettings()
mock_vertex_ai_model_generate_content_async.return_value = mock_vertex_ai_streaming_text_completion_response
vertex_ai_text_completion = VertexAITextCompletion()
async for chunks in vertex_ai_text_completion.get_streaming_text_contents(prompt, settings):
assert len(chunks) == 1
assert "usage" in chunks[0].metadata
assert "prompt_feedback" in chunks[0].metadata
mock_vertex_ai_model_generate_content_async.assert_called_once_with(
contents=prompt,
generation_config=settings.prepare_settings_dict(),
stream=True,
)
# endregion streaming text completion
@@ -0,0 +1,160 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, patch
import pytest
from numpy import array, ndarray
from vertexai.language_models import TextEmbedding, TextEmbeddingModel
from semantic_kernel.connectors.ai.google.vertex_ai.services.vertex_ai_text_embedding import VertexAITextEmbedding
from semantic_kernel.connectors.ai.google.vertex_ai.vertex_ai_prompt_execution_settings import (
VertexAIEmbeddingPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.google.vertex_ai.vertex_ai_settings import VertexAISettings
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
from tests.unit.connectors.ai.google.vertex_ai.conftest import MockTextEmbeddingModel
# region init
def test_vertex_ai_text_embedding_init(vertex_ai_unit_test_env) -> None:
"""Test initialization of VertexAITextEmbedding"""
model_id = vertex_ai_unit_test_env["VERTEX_AI_EMBEDDING_MODEL_ID"]
project_id = vertex_ai_unit_test_env["VERTEX_AI_PROJECT_ID"]
vertex_ai_text_embedding = VertexAITextEmbedding()
assert vertex_ai_text_embedding.ai_model_id == model_id
assert vertex_ai_text_embedding.service_id == model_id
assert isinstance(vertex_ai_text_embedding.service_settings, VertexAISettings)
assert vertex_ai_text_embedding.service_settings.embedding_model_id == model_id
assert vertex_ai_text_embedding.service_settings.project_id == project_id
def test_vertex_ai_text_embedding_init_with_service_id(vertex_ai_unit_test_env, service_id) -> None:
"""Test initialization of VertexAITextEmbedding with a service id that is not the model id"""
vertex_ai_text_embedding = VertexAITextEmbedding(service_id=service_id)
assert vertex_ai_text_embedding.service_id == service_id
def test_vertex_ai_text_embedding_init_with_model_id_in_argument(vertex_ai_unit_test_env) -> None:
"""Test initialization of VertexAITextEmbedding with model id in argument"""
vertex_ai_chat_completion = VertexAITextEmbedding(embedding_model_id="custom_model_id")
assert vertex_ai_chat_completion.ai_model_id == "custom_model_id"
assert vertex_ai_chat_completion.service_id == "custom_model_id"
@pytest.mark.parametrize("exclude_list", [["VERTEX_AI_EMBEDDING_MODEL_ID"]], indirect=True)
def test_vertex_ai_text_embedding_init_with_empty_model_id(vertex_ai_unit_test_env) -> None:
"""Test initialization of VertexAITextEmbedding with an empty model id"""
with pytest.raises(ServiceInitializationError):
VertexAITextEmbedding(env_file_path="fake_env_file_path.env")
@pytest.mark.parametrize("exclude_list", [["VERTEX_AI_PROJECT_ID"]], indirect=True)
def test_vertex_ai_text_embedding_init_with_empty_project_id(vertex_ai_unit_test_env) -> None:
"""Test initialization of VertexAITextEmbedding with an empty project id"""
with pytest.raises(ServiceInitializationError):
VertexAITextEmbedding(env_file_path="fake_env_file_path.env")
def test_prompt_execution_settings_class(vertex_ai_unit_test_env) -> None:
vertex_ai_text_embedding = VertexAITextEmbedding()
assert vertex_ai_text_embedding.get_prompt_execution_settings_class() == VertexAIEmbeddingPromptExecutionSettings
# endregion init
@patch.object(TextEmbeddingModel, "from_pretrained")
@patch.object(MockTextEmbeddingModel, "get_embeddings_async", new_callable=AsyncMock)
async def test_embedding(mock_embedding_client, mock_from_pretrained, vertex_ai_unit_test_env, prompt):
"""Test that the service initializes and generates embeddings correctly."""
mock_from_pretrained.return_value = MockTextEmbeddingModel()
mock_embedding_client.return_value = [TextEmbedding(values=[0.1, 0.2, 0.3])]
settings = VertexAIEmbeddingPromptExecutionSettings()
vertex_ai_text_embedding = VertexAITextEmbedding()
response: ndarray = await vertex_ai_text_embedding.generate_embeddings(
[prompt],
settings=settings,
)
assert len(response) == 1
assert response.all() == array([0.1, 0.2, 0.3]).all()
mock_embedding_client.assert_called_once_with([prompt])
@patch.object(TextEmbeddingModel, "from_pretrained")
@patch.object(MockTextEmbeddingModel, "get_embeddings_async", new_callable=AsyncMock)
async def test_embedding_with_settings(mock_embedding_client, mock_from_pretrained, vertex_ai_unit_test_env, prompt):
"""Test that the service initializes and generates embeddings correctly."""
mock_from_pretrained.return_value = MockTextEmbeddingModel()
mock_embedding_client.return_value = [TextEmbedding(values=[0.1, 0.2, 0.3])]
settings = VertexAIEmbeddingPromptExecutionSettings()
settings.output_dimensionality = 3
settings.auto_truncate = True
vertex_ai_text_embedding = VertexAITextEmbedding()
response: ndarray = await vertex_ai_text_embedding.generate_embeddings(
[prompt],
settings=settings,
)
assert len(response) == 1
assert response.all() == array([0.1, 0.2, 0.3]).all()
mock_embedding_client.assert_called_once_with(
[prompt],
**settings.prepare_settings_dict(),
)
@patch.object(TextEmbeddingModel, "from_pretrained")
@patch.object(MockTextEmbeddingModel, "get_embeddings_async", new_callable=AsyncMock)
async def test_embedding_without_settings(mock_embedding_client, mock_from_pretrained, vertex_ai_unit_test_env, prompt):
"""Test that the service initializes and generates embeddings correctly without settings."""
mock_from_pretrained.return_value = MockTextEmbeddingModel()
mock_embedding_client.return_value = [TextEmbedding(values=[0.1, 0.2, 0.3])]
vertex_ai_text_embedding = VertexAITextEmbedding()
response: ndarray = await vertex_ai_text_embedding.generate_embeddings([prompt])
assert len(response) == 1
assert response.all() == array([0.1, 0.2, 0.3]).all()
mock_embedding_client.assert_called_once_with([prompt])
@patch.object(TextEmbeddingModel, "from_pretrained")
@patch.object(MockTextEmbeddingModel, "get_embeddings_async", new_callable=AsyncMock)
async def test_embedding_list_input(mock_embedding_client, mock_from_pretrained, vertex_ai_unit_test_env, prompt):
"""Test that the service initializes and generates embeddings correctly with a list of prompts."""
mock_from_pretrained.return_value = MockTextEmbeddingModel()
mock_embedding_client.return_value = [TextEmbedding(values=[0.1, 0.2, 0.3]), TextEmbedding(values=[0.1, 0.2, 0.3])]
vertex_ai_text_embedding = VertexAITextEmbedding()
response: ndarray = await vertex_ai_text_embedding.generate_embeddings([prompt, prompt])
assert len(response) == 2
assert response.all() == array([[0.1, 0.2, 0.3], [0.1, 0.2, 0.3]]).all()
mock_embedding_client.assert_called_once_with([prompt, prompt])
@patch.object(TextEmbeddingModel, "from_pretrained")
@patch.object(MockTextEmbeddingModel, "get_embeddings_async", new_callable=AsyncMock)
async def test_raw_embedding(mock_embedding_client, mock_from_pretrained, vertex_ai_unit_test_env, prompt):
"""Test that the service initializes and generates embeddings correctly."""
mock_from_pretrained.return_value = MockTextEmbeddingModel()
mock_embedding_client.return_value = [TextEmbedding(values=[0.1, 0.2, 0.3])]
settings = VertexAIEmbeddingPromptExecutionSettings()
vertex_ai_text_embedding = VertexAITextEmbedding()
response: ndarray = await vertex_ai_text_embedding.generate_raw_embeddings([prompt], settings)
assert len(response) == 1
assert response[0] == [0.1, 0.2, 0.3]
mock_embedding_client.assert_called_once_with([prompt])
@@ -0,0 +1,199 @@
# Copyright (c) Microsoft. All rights reserved.
import pytest
from google.cloud.aiplatform_v1beta1.types.content import Candidate
from vertexai.generative_models import FunctionDeclaration, Part
from semantic_kernel.connectors.ai.google.vertex_ai.services.utils import (
finish_reason_from_vertex_ai_to_semantic_kernel,
format_assistant_message,
format_user_message,
kernel_function_metadata_to_vertex_ai_function_call_format,
)
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.function_call_content import FunctionCallContent
from semantic_kernel.contents.function_result_content import FunctionResultContent
from semantic_kernel.contents.image_content import ImageContent
from semantic_kernel.contents.text_content import TextContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.contents.utils.finish_reason import FinishReason
from semantic_kernel.exceptions.service_exceptions import ServiceInvalidRequestError
from semantic_kernel.functions.kernel_function_metadata import KernelFunctionMetadata
from semantic_kernel.functions.kernel_parameter_metadata import KernelParameterMetadata
def test_finish_reason_from_vertex_ai_to_semantic_kernel():
"""Test finish_reason_from_vertex_ai_to_semantic_kernel."""
assert finish_reason_from_vertex_ai_to_semantic_kernel(Candidate.FinishReason.STOP) == FinishReason.STOP
assert finish_reason_from_vertex_ai_to_semantic_kernel(Candidate.FinishReason.MAX_TOKENS) == FinishReason.LENGTH
assert finish_reason_from_vertex_ai_to_semantic_kernel(Candidate.FinishReason.SAFETY) == FinishReason.CONTENT_FILTER
assert finish_reason_from_vertex_ai_to_semantic_kernel(Candidate.FinishReason.OTHER) is None
def test_format_user_message():
"""Test format_user_message."""
user_message = ChatMessageContent(role=AuthorRole.USER, content="User message")
formatted_user_message = format_user_message(user_message)
assert len(formatted_user_message) == 1
assert isinstance(formatted_user_message[0], Part)
assert formatted_user_message[0].text == "User message"
# Test with an image content
image_content = ImageContent(data="image data", mime_type="image/png")
user_message = ChatMessageContent(
role=AuthorRole.USER,
items=[
TextContent(text="Text content"),
image_content,
],
)
formatted_user_message = format_user_message(user_message)
assert len(formatted_user_message) == 2
assert isinstance(formatted_user_message[0], Part)
assert formatted_user_message[0].text == "Text content"
assert isinstance(formatted_user_message[1], Part)
assert formatted_user_message[1].inline_data.mime_type == "image/png"
assert formatted_user_message[1].inline_data.data == image_content.data
def test_format_user_message_throws_with_unsupported_items() -> None:
"""Test format_user_message with unsupported items."""
# Test with unsupported items, any item other than TextContent and ImageContent should raise an error
user_message = ChatMessageContent(
role=AuthorRole.USER,
items=[
FunctionCallContent(),
],
)
with pytest.raises(ServiceInvalidRequestError):
format_user_message(user_message)
# Test with an ImageContent that has no data_uri
user_message = ChatMessageContent(
role=AuthorRole.USER,
items=[
ImageContent(data_uri=""),
],
)
with pytest.raises(ServiceInvalidRequestError):
format_user_message(user_message)
def test_format_assistant_message() -> None:
assistant_message = ChatMessageContent(
role=AuthorRole.ASSISTANT,
items=[
TextContent(text="test"),
FunctionCallContent(name="test_function", arguments={}),
ImageContent(data="image data", mime_type="image/png"),
],
)
formatted_assistant_message = format_assistant_message(assistant_message)
assert isinstance(formatted_assistant_message, list)
assert len(formatted_assistant_message) == 3
assert isinstance(formatted_assistant_message[0], Part)
assert formatted_assistant_message[0].text == "test"
assert isinstance(formatted_assistant_message[1], Part)
assert formatted_assistant_message[1].function_call.name == "test_function"
assert formatted_assistant_message[1].function_call.args == {}
assert isinstance(formatted_assistant_message[2], Part)
assert formatted_assistant_message[2].inline_data
def test_format_assistant_message_with_unsupported_items() -> None:
assistant_message = ChatMessageContent(
role=AuthorRole.ASSISTANT,
items=[
FunctionResultContent(id="test_id", function_name="test_function"),
],
)
with pytest.raises(ServiceInvalidRequestError):
format_assistant_message(assistant_message)
def test_format_assistant_message_with_thought_signature() -> None:
"""Test that thought_signature is preserved in function call parts for Vertex AI."""
import base64
thought_sig = base64.b64encode(b"test_thought_signature_data").decode("utf-8")
assistant_message = ChatMessageContent(
role=AuthorRole.ASSISTANT,
items=[
FunctionCallContent(
name="test_function",
arguments={"arg1": "value1"},
metadata={"thought_signature": thought_sig},
),
],
)
formatted = format_assistant_message(assistant_message)
assert len(formatted) == 1
assert isinstance(formatted[0], Part)
assert formatted[0].function_call.name == "test_function"
assert formatted[0].function_call.args == {"arg1": "value1"}
part_dict = formatted[0].to_dict()
assert "thought_signature" in part_dict
assert part_dict["thought_signature"] == thought_sig
def test_format_assistant_message_without_thought_signature() -> None:
"""Test that function calls without thought_signature still work for Vertex AI."""
assistant_message = ChatMessageContent(
role=AuthorRole.ASSISTANT,
items=[
FunctionCallContent(
name="test_function",
arguments={"arg1": "value1"},
),
],
)
formatted = format_assistant_message(assistant_message)
assert len(formatted) == 1
assert isinstance(formatted[0], Part)
assert formatted[0].function_call.name == "test_function"
assert formatted[0].function_call.args == {"arg1": "value1"}
part_dict = formatted[0].to_dict()
assert "thought_signature" not in part_dict
def test_vertex_ai_function_call_format_sanitizes_anyof_schema() -> None:
"""Integration test: anyOf in param schema_data is sanitized in the FunctionDeclaration."""
metadata = KernelFunctionMetadata(
name="test_func",
description="A test function",
is_prompt=False,
parameters=[
KernelParameterMetadata(
name="messages",
description="The user messages",
is_required=True,
schema_data={
"anyOf": [
{"type": "string"},
{"type": "array", "items": {"type": "string"}},
],
"description": "The user messages",
},
),
],
)
result = kernel_function_metadata_to_vertex_ai_function_call_format(metadata)
assert isinstance(result, FunctionDeclaration)
def test_vertex_ai_function_call_format_empty_parameters() -> None:
"""Integration test: metadata with no parameters produces empty properties, no crash."""
metadata = KernelFunctionMetadata(
name="no_params_func",
description="No parameters",
is_prompt=False,
parameters=[],
)
result = kernel_function_metadata_to_vertex_ai_function_call_format(metadata)
assert isinstance(result, FunctionDeclaration)
@@ -0,0 +1,136 @@
# Copyright (c) Microsoft. All rights reserved.
from semantic_kernel.connectors.ai.google.vertex_ai import (
VertexAIChatPromptExecutionSettings,
VertexAIPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.google.vertex_ai.vertex_ai_prompt_execution_settings import (
VertexAIEmbeddingPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
def test_default_vertex_ai_prompt_execution_settings():
settings = VertexAIPromptExecutionSettings()
assert settings.stop_sequences is None
assert settings.response_mime_type is None
assert settings.response_schema is None
assert settings.candidate_count is None
assert settings.max_output_tokens is None
assert settings.temperature is None
assert settings.top_p is None
assert settings.top_k is None
def test_custom_vertex_ai_prompt_execution_settings():
settings = VertexAIPromptExecutionSettings(
stop_sequences=["world"],
response_mime_type="text/plain",
candidate_count=1,
max_output_tokens=128,
temperature=0.5,
top_p=0.5,
top_k=10,
)
assert settings.stop_sequences == ["world"]
assert settings.response_mime_type == "text/plain"
assert settings.candidate_count == 1
assert settings.max_output_tokens == 128
assert settings.temperature == 0.5
assert settings.top_p == 0.5
assert settings.top_k == 10
def test_vertex_ai_prompt_execution_settings_from_default_completion_config():
settings = PromptExecutionSettings(service_id="test_service")
chat_settings = VertexAIChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.service_id == "test_service"
assert chat_settings.stop_sequences is None
assert chat_settings.response_mime_type is None
assert chat_settings.response_schema is None
assert chat_settings.candidate_count is None
assert chat_settings.max_output_tokens is None
assert chat_settings.temperature is None
assert chat_settings.top_p is None
assert chat_settings.top_k is None
def test_vertex_ai_prompt_execution_settings_from_openai_prompt_execution_settings():
chat_settings = VertexAIChatPromptExecutionSettings(service_id="test_service", temperature=1.0)
new_settings = VertexAIPromptExecutionSettings(service_id="test_2", temperature=0.0)
chat_settings.update_from_prompt_execution_settings(new_settings)
assert chat_settings.service_id == "test_2"
assert chat_settings.temperature == 0.0
def test_vertex_ai_prompt_execution_settings_from_custom_completion_config():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"stop_sequences": ["world"],
"response_mime_type": "text/plain",
"candidate_count": 1,
"max_output_tokens": 128,
"temperature": 0.5,
"top_p": 0.5,
"top_k": 10,
},
)
chat_settings = VertexAIChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.stop_sequences == ["world"]
assert chat_settings.response_mime_type == "text/plain"
assert chat_settings.candidate_count == 1
assert chat_settings.max_output_tokens == 128
assert chat_settings.temperature == 0.5
assert chat_settings.top_p == 0.5
assert chat_settings.top_k == 10
def test_vertex_ai_chat_prompt_execution_settings_from_custom_completion_config_with_functions():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"tools": [],
},
)
chat_settings = VertexAIChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.tools == []
def test_create_options():
settings = VertexAIChatPromptExecutionSettings(
service_id="test_service",
extension_data={
"stop_sequences": ["world"],
"response_mime_type": "text/plain",
"candidate_count": 1,
"max_output_tokens": 128,
"temperature": 0.5,
"top_p": 0.5,
"top_k": 10,
},
)
options = settings.prepare_settings_dict()
assert options["stop_sequences"] == ["world"]
assert options["response_mime_type"] == "text/plain"
assert options["candidate_count"] == 1
assert options["max_output_tokens"] == 128
assert options["temperature"] == 0.5
assert options["top_p"] == 0.5
assert options["top_k"] == 10
assert "tools" not in options
assert "tool_config" not in options
def test_default_vertex_ai_embedding_prompt_execution_settings():
settings = VertexAIEmbeddingPromptExecutionSettings()
assert settings.output_dimensionality is None
assert settings.auto_truncate is None
@@ -0,0 +1,219 @@
# Copyright (c) Microsoft. All rights reserved.
from threading import Thread
from unittest.mock import MagicMock, Mock, patch
import pytest
from transformers import AutoTokenizer, TextIteratorStreamer
from semantic_kernel.connectors.ai.hugging_face.services.hf_text_completion import HuggingFaceTextCompletion
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
from semantic_kernel.exceptions import KernelInvokeException, ServiceResponseException
from semantic_kernel.functions.kernel_arguments import KernelArguments
from semantic_kernel.kernel import Kernel
from semantic_kernel.prompt_template.prompt_template_config import PromptTemplateConfig
@pytest.mark.parametrize(
("model_name", "task", "input_str"),
[
(
"patrickvonplaten/t5-tiny-random",
"text2text-generation",
"translate English to Dutch: Hello, how are you?",
),
(
"Falconsai/text_summarization",
"summarization",
"""
Summarize: Whales are fully aquatic, open-ocean animals:
they can feed, mate, give birth, suckle and raise their young at sea.
Whales range in size from the 2.6 metres (8.5 ft) and 135 kilograms (298 lb)
dwarf sperm whale to the 29.9 metres (98 ft) and 190 tonnes (210 short tons) blue whale,
which is the largest known animal that has ever lived. The sperm whale is the largest
toothed predator on Earth. Several whale species exhibit sexual dimorphism,
in that the females are larger than males.
""",
),
("HuggingFaceM4/tiny-random-LlamaForCausalLM", "text-generation", "Hello, I like sleeping and "),
],
ids=["text2text-generation", "summarization", "text-generation"],
)
async def test_text_completion(model_name, task, input_str):
kernel = Kernel()
ret = {"summary_text": "test"} if task == "summarization" else {"generated_text": "test"}
mock_pipeline = Mock(return_value=ret)
# Configure LLM service
with patch("semantic_kernel.connectors.ai.hugging_face.services.hf_text_completion.pipeline") as patched_pipeline:
patched_pipeline.return_value = mock_pipeline
service = HuggingFaceTextCompletion(service_id=model_name, ai_model_id=model_name, task=task)
kernel.add_service(
service=service,
)
exec_settings = PromptExecutionSettings(service_id=model_name, extension_data={"max_new_tokens": 25})
# Define semantic function using SK prompt template language
prompt = "{{$input}}"
prompt_template_config = PromptTemplateConfig(template=prompt, execution_settings=exec_settings)
kernel.add_function(
prompt_template_config=prompt_template_config,
function_name="TestFunction",
plugin_name="TestPlugin",
prompt_execution_settings=exec_settings,
)
arguments = KernelArguments(input=input_str)
await kernel.invoke(function_name="TestFunction", plugin_name="TestPlugin", arguments=arguments)
assert mock_pipeline.call_args.args[0] == input_str
async def test_text_completion_throws():
kernel = Kernel()
model_name = "patrickvonplaten/t5-tiny-random"
task = "text2text-generation"
input_str = "translate English to Dutch: Hello, how are you?"
with patch("semantic_kernel.connectors.ai.hugging_face.services.hf_text_completion.pipeline") as patched_pipeline:
mock_generator = Mock()
mock_generator.side_effect = Exception("Test exception")
patched_pipeline.return_value = mock_generator
service = HuggingFaceTextCompletion(service_id=model_name, ai_model_id=model_name, task=task)
kernel.add_service(service=service)
exec_settings = PromptExecutionSettings(service_id=model_name, extension_data={"max_new_tokens": 25})
prompt = "{{$input}}"
prompt_template_config = PromptTemplateConfig(template=prompt, execution_settings=exec_settings)
kernel.add_function(
prompt_template_config=prompt_template_config,
function_name="TestFunction",
plugin_name="TestPlugin",
prompt_execution_settings=exec_settings,
)
arguments = KernelArguments(input=input_str)
with pytest.raises(
KernelInvokeException, match="Error occurred while invoking function: 'TestPlugin-TestFunction'"
):
await kernel.invoke(function_name="TestFunction", plugin_name="TestPlugin", arguments=arguments)
@pytest.mark.parametrize(
("model_name", "task", "input_str"),
[
(
"patrickvonplaten/t5-tiny-random",
"text2text-generation",
"translate English to Dutch: Hello, how are you?",
),
("HuggingFaceM4/tiny-random-LlamaForCausalLM", "text-generation", "Hello, I like sleeping and "),
],
ids=["text2text-generation", "text-generation"],
)
async def test_text_completion_streaming(model_name, task, input_str):
ret = {"summary_text": "test"} if task == "summarization" else {"generated_text": "test"}
mock_pipeline = Mock(return_value=ret)
mock_streamer = MagicMock(spec=TextIteratorStreamer)
mock_streamer.__iter__.return_value = iter(["mocked_text"])
with (
patch(
"semantic_kernel.connectors.ai.hugging_face.services.hf_text_completion.pipeline",
return_value=mock_pipeline,
),
patch(
"semantic_kernel.connectors.ai.hugging_face.services.hf_text_completion.Thread",
side_effect=Mock(spec=Thread),
),
patch(
"semantic_kernel.connectors.ai.hugging_face.services.hf_text_completion.AutoTokenizer",
side_effect=Mock(spec=AutoTokenizer),
),
patch(
"semantic_kernel.connectors.ai.hugging_face.services.hf_text_completion.TextIteratorStreamer",
return_value=mock_streamer,
) as mock_stream,
):
mock_stream.return_value = mock_streamer
service = HuggingFaceTextCompletion(service_id=model_name, ai_model_id=model_name, task=task)
prompt = "test prompt"
exec_settings = PromptExecutionSettings(service_id=model_name, extension_data={"max_new_tokens": 25})
result = []
async for content in service.get_streaming_text_contents(prompt, exec_settings):
result.append(content)
assert len(result) == 1
assert result[0][0].inner_content == "mocked_text"
@pytest.mark.parametrize(
("model_name", "task", "input_str"),
[
(
"patrickvonplaten/t5-tiny-random",
"text2text-generation",
"translate English to Dutch: Hello, how are you?",
),
("HuggingFaceM4/tiny-random-LlamaForCausalLM", "text-generation", "Hello, I like sleeping and "),
],
ids=["text2text-generation", "text-generation"],
)
async def test_text_completion_streaming_throws(model_name, task, input_str):
ret = {"summary_text": "test"} if task == "summarization" else {"generated_text": "test"}
mock_pipeline = Mock(return_value=ret)
mock_streamer = MagicMock(spec=TextIteratorStreamer)
mock_streamer.__iter__.return_value = Exception()
with (
patch(
"semantic_kernel.connectors.ai.hugging_face.services.hf_text_completion.pipeline",
return_value=mock_pipeline,
),
patch(
"semantic_kernel.connectors.ai.hugging_face.services.hf_text_completion.Thread",
side_effect=Exception(),
),
patch(
"semantic_kernel.connectors.ai.hugging_face.services.hf_text_completion.TextIteratorStreamer",
return_value=mock_streamer,
) as mock_stream,
):
mock_stream.return_value = mock_streamer
service = HuggingFaceTextCompletion(service_id=model_name, ai_model_id=model_name, task=task)
prompt = "test prompt"
exec_settings = PromptExecutionSettings(service_id=model_name, extension_data={"max_new_tokens": 25})
with pytest.raises(ServiceResponseException, match=("Hugging Face completion failed")):
async for _ in service.get_streaming_text_contents(prompt, exec_settings):
pass
def test_hugging_face_text_completion_init():
with (
patch("semantic_kernel.connectors.ai.hugging_face.services.hf_text_completion.pipeline") as patched_pipeline,
patch(
"semantic_kernel.connectors.ai.hugging_face.services.hf_text_completion.torch.cuda.is_available"
) as mock_torch_cuda_is_available,
):
patched_pipeline.return_value = patched_pipeline
mock_torch_cuda_is_available.return_value = False
ai_model_id = "test-model"
task = "summarization"
device = -1
service = HuggingFaceTextCompletion(service_id="test", ai_model_id=ai_model_id, task=task, device=device)
assert service is not None
@@ -0,0 +1,56 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import patch
import pytest
from numpy import array, ndarray
from semantic_kernel.connectors.ai.hugging_face.services.hf_text_embedding import HuggingFaceTextEmbedding
from semantic_kernel.exceptions import ServiceResponseException
def test_huggingface_text_embedding_initialization():
model_name = "sentence-transformers/all-MiniLM-L6-v2"
device = -1
with patch("sentence_transformers.SentenceTransformer") as mock_transformer:
mock_instance = mock_transformer.return_value
service = HuggingFaceTextEmbedding(service_id="test", ai_model_id=model_name, device=device)
assert service.ai_model_id == model_name
assert service.device == "cpu"
assert service.generator == mock_instance
mock_transformer.assert_called_once_with(model_name_or_path=model_name, device="cpu")
async def test_generate_embeddings_success():
model_name = "sentence-transformers/all-MiniLM-L6-v2"
device = -1
texts = ["Hello world!", "How are you?"]
mock_embeddings = array([[0.1, 0.2], [0.3, 0.4]])
with patch("sentence_transformers.SentenceTransformer") as mock_transformer:
mock_instance = mock_transformer.return_value
mock_instance.encode.return_value = mock_embeddings
service = HuggingFaceTextEmbedding(service_id="test", ai_model_id=model_name, device=device)
embeddings = await service.generate_embeddings(texts)
assert isinstance(embeddings, ndarray)
assert embeddings.shape == (2, 2)
assert (embeddings == mock_embeddings).all()
async def test_generate_embeddings_throws():
model_name = "sentence-transformers/all-MiniLM-L6-v2"
device = -1
texts = ["Hello world!", "How are you?"]
with patch("sentence_transformers.SentenceTransformer") as mock_transformer:
mock_instance = mock_transformer.return_value
mock_instance.encode.side_effect = Exception("Test exception")
service = HuggingFaceTextEmbedding(service_id="test", ai_model_id=model_name, device=device)
with pytest.raises(ServiceResponseException, match="Hugging Face embeddings failed"):
await service.generate_embeddings(texts)
@@ -0,0 +1,616 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from mistralai import CompletionEvent, Mistral
from mistralai.models import (
AssistantMessage,
ChatCompletionChoice,
ChatCompletionResponse,
CompletionChunk,
CompletionResponseStreamChoice,
DeltaMessage,
UsageInfo,
)
from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase
from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior, FunctionChoiceType
from semantic_kernel.connectors.ai.mistral_ai.prompt_execution_settings.mistral_ai_prompt_execution_settings import (
MistralAIChatPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.mistral_ai.services.mistral_ai_chat_completion import MistralAIChatCompletion
from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_prompt_execution_settings import (
OpenAIChatPromptExecutionSettings,
)
from semantic_kernel.contents import FunctionCallContent, TextContent
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
from semantic_kernel.exceptions import (
ServiceInitializationError,
ServiceInvalidExecutionSettingsError,
ServiceResponseException,
)
from semantic_kernel.functions.kernel_arguments import KernelArguments
from semantic_kernel.functions.kernel_function import KernelFunction
from semantic_kernel.functions.kernel_function_decorator import kernel_function
from semantic_kernel.kernel import Kernel
@pytest.fixture
def mock_settings() -> MistralAIChatPromptExecutionSettings:
return MistralAIChatPromptExecutionSettings()
@pytest.fixture
def mock_mistral_ai_client_completion() -> Mistral:
client = MagicMock(spec=Mistral)
client.chat = MagicMock()
# Use proper ChatCompletionResponse type so isinstance checks pass
chat_completion_response = ChatCompletionResponse(
id="test_id",
object="object",
created=12345,
usage=UsageInfo(prompt_tokens=10, completion_tokens=5, total_tokens=15),
model="test_model_id",
choices=[
ChatCompletionChoice(
index=0,
message=AssistantMessage(role="assistant", content="Test"),
finish_reason="stop",
)
],
)
client.chat.complete_async = AsyncMock(return_value=chat_completion_response)
return client
@pytest.fixture
def mock_mistral_ai_client_completion_stream() -> Mistral:
client = MagicMock(spec=Mistral)
client.chat = MagicMock()
# Use proper CompletionEvent/CompletionChunk types so isinstance checks pass
mock_chunk = CompletionEvent(
data=CompletionChunk(
id="test_chunk",
created=12345,
model="test_model_id",
choices=[
CompletionResponseStreamChoice(
index=0,
delta=DeltaMessage(role="assistant", content="Test"),
finish_reason="stop",
)
],
)
)
async def mock_stream_async(**kwargs):
yield mock_chunk
client.chat.stream_async = AsyncMock(return_value=mock_stream_async())
return client
async def test_complete_chat_contents(
kernel: Kernel,
mock_settings: MistralAIChatPromptExecutionSettings,
mock_mistral_ai_client_completion: Mistral,
):
chat_history = MagicMock()
arguments = KernelArguments()
chat_completion_base = MistralAIChatCompletion(
ai_model_id="test_model_id", service_id="test", api_key="", async_client=mock_mistral_ai_client_completion
)
content: list[ChatMessageContent] = await chat_completion_base.get_chat_message_contents(
chat_history=chat_history, settings=mock_settings, kernel=kernel, arguments=arguments
)
assert content is not None
mock_message_text_content = ChatMessageContent(role=AuthorRole.ASSISTANT, items=[TextContent(text="test")])
mock_message_function_call = ChatMessageContent(
role=AuthorRole.ASSISTANT,
items=[
FunctionCallContent(
name="test",
arguments={"key": "test"},
)
],
)
@pytest.mark.parametrize(
"function_choice_behavior,model_responses,expected_result",
[
pytest.param(
FunctionChoiceBehavior.Auto(),
[[mock_message_function_call], [mock_message_text_content]],
TextContent,
id="auto",
),
pytest.param(
FunctionChoiceBehavior.Auto(auto_invoke=False),
[[mock_message_function_call]],
FunctionCallContent,
id="auto_none_invoke",
),
pytest.param(
FunctionChoiceBehavior.Required(auto_invoke=False),
[[mock_message_function_call]],
FunctionCallContent,
id="required_none_invoke",
),
pytest.param(FunctionChoiceBehavior.NoneInvoke(), [[mock_message_text_content]], TextContent, id="none"),
],
)
async def test_complete_chat_contents_function_call_behavior_tool_call(
kernel: Kernel,
mock_settings: MistralAIChatPromptExecutionSettings,
function_choice_behavior: FunctionChoiceBehavior,
model_responses,
expected_result,
):
kernel.add_function("test", kernel_function(lambda key: "test", name="test"))
mock_settings.function_choice_behavior = function_choice_behavior
arguments = KernelArguments()
chat_completion_base = MistralAIChatCompletion(ai_model_id="test_model_id", service_id="test", api_key="")
with (
patch.object(chat_completion_base, "_inner_get_chat_message_contents", side_effect=model_responses),
):
response: list[ChatMessageContent] = await chat_completion_base.get_chat_message_contents(
chat_history=ChatHistory(system_message="Test"), settings=mock_settings, kernel=kernel, arguments=arguments
)
assert all(isinstance(content, expected_result) for content in response[0].items)
async def test_complete_chat_contents_function_call_behavior_without_kernel(
mock_settings: MistralAIChatPromptExecutionSettings,
mock_mistral_ai_client_completion: Mistral,
):
chat_history = MagicMock()
chat_completion_base = MistralAIChatCompletion(
ai_model_id="test_model_id", service_id="test", api_key="", async_client=mock_mistral_ai_client_completion
)
mock_settings.function_choice_behavior = FunctionChoiceBehavior.Auto()
with pytest.raises(ServiceInvalidExecutionSettingsError):
await chat_completion_base.get_chat_message_contents(chat_history=chat_history, settings=mock_settings)
async def test_complete_chat_stream_contents(
kernel: Kernel,
mock_settings: MistralAIChatPromptExecutionSettings,
mock_mistral_ai_client_completion_stream: Mistral,
):
chat_history = MagicMock()
arguments = KernelArguments()
chat_completion_base = MistralAIChatCompletion(
ai_model_id="test_model_id",
service_id="test",
api_key="",
async_client=mock_mistral_ai_client_completion_stream,
)
async for content in chat_completion_base.get_streaming_chat_message_contents(
chat_history, mock_settings, kernel=kernel, arguments=arguments
):
assert content is not None
mock_message_function_call = StreamingChatMessageContent(
role=AuthorRole.ASSISTANT, items=[FunctionCallContent(name="test")], choice_index="0"
)
mock_message_text_content = StreamingChatMessageContent(
role=AuthorRole.ASSISTANT, items=[TextContent(text="test")], choice_index="0"
)
@pytest.mark.parametrize(
"function_choice_behavior,model_responses,expected_result",
[
pytest.param(
FunctionChoiceBehavior.Auto(),
[[mock_message_function_call], [mock_message_text_content]],
TextContent,
id="auto",
),
pytest.param(
FunctionChoiceBehavior.Auto(auto_invoke=False),
[[mock_message_function_call]],
FunctionCallContent,
id="auto_none_invoke",
),
pytest.param(
FunctionChoiceBehavior.Required(auto_invoke=False),
[[mock_message_function_call]],
FunctionCallContent,
id="required_none_invoke",
),
pytest.param(FunctionChoiceBehavior.NoneInvoke(), [[mock_message_text_content]], TextContent, id="none"),
],
)
async def test_complete_chat_contents_streaming_function_call_behavior_tool_call(
kernel: Kernel,
mock_settings: MistralAIChatPromptExecutionSettings,
function_choice_behavior: FunctionChoiceBehavior,
model_responses,
expected_result,
):
mock_settings.function_choice_behavior = function_choice_behavior
# Mock sequence of model responses
generator_mocks = []
for mock_message in model_responses:
generator_mock = MagicMock()
generator_mock.__aiter__.return_value = [mock_message]
generator_mocks.append(generator_mock)
arguments = KernelArguments()
chat_completion_base = MistralAIChatCompletion(ai_model_id="test_model_id", service_id="test", api_key="")
with patch.object(chat_completion_base, "_inner_get_streaming_chat_message_contents", side_effect=generator_mocks):
messages = []
async for chunk in chat_completion_base.get_streaming_chat_message_contents(
chat_history=ChatHistory(system_message="Test"), settings=mock_settings, kernel=kernel, arguments=arguments
):
messages.append(chunk)
response = messages[-1]
assert all(isinstance(content, expected_result) for content in response[0].items)
async def test_mistral_ai_sdk_exception(kernel: Kernel, mock_settings: MistralAIChatPromptExecutionSettings):
chat_history = MagicMock()
arguments = KernelArguments()
client = MagicMock(spec=Mistral)
client.chat = MagicMock()
client.chat.complete_async.side_effect = Exception("Test Exception")
chat_completion_base = MistralAIChatCompletion(
ai_model_id="test_model_id", service_id="test", api_key="", async_client=client
)
with pytest.raises(ServiceResponseException):
await chat_completion_base.get_chat_message_contents(
chat_history=chat_history, settings=mock_settings, kernel=kernel, arguments=arguments
)
async def test_mistral_ai_sdk_exception_streaming(kernel: Kernel, mock_settings: MistralAIChatPromptExecutionSettings):
chat_history = MagicMock()
arguments = KernelArguments()
client = MagicMock(spec=Mistral)
client.chat = MagicMock()
client.chat.chat_stream.side_effect = Exception("Test Exception")
chat_completion_base = MistralAIChatCompletion(
ai_model_id="test_model_id", service_id="test", api_key="", async_client=client
)
with pytest.raises(ServiceResponseException):
async for content in chat_completion_base.get_streaming_chat_message_contents(
chat_history, mock_settings, kernel=kernel, arguments=arguments
):
assert content is not None
def test_mistral_ai_chat_completion_init(mistralai_unit_test_env) -> None:
# Test successful initialization
mistral_ai_chat_completion = MistralAIChatCompletion()
assert mistral_ai_chat_completion.ai_model_id == mistralai_unit_test_env["MISTRALAI_CHAT_MODEL_ID"]
api_key = mistralai_unit_test_env["MISTRALAI_API_KEY"]
assert mistral_ai_chat_completion.async_client.sdk_configuration.security.api_key == api_key
assert isinstance(mistral_ai_chat_completion, ChatCompletionClientBase)
@pytest.mark.parametrize("exclude_list", [["MISTRALAI_API_KEY", "MISTRALAI_CHAT_MODEL_ID"]], indirect=True)
def test_mistral_ai_chat_completion_init_constructor(mistralai_unit_test_env) -> None:
# Test successful initialization
mistral_ai_chat_completion = MistralAIChatCompletion(
api_key="overwrite_api_key",
ai_model_id="overwrite_model_id",
env_file_path="test.env",
)
assert mistral_ai_chat_completion.ai_model_id == "overwrite_model_id"
assert mistral_ai_chat_completion.async_client.sdk_configuration.security.api_key == "overwrite_api_key"
assert isinstance(mistral_ai_chat_completion, ChatCompletionClientBase)
@pytest.mark.parametrize("exclude_list", [["MISTRALAI_API_KEY", "MISTRALAI_CHAT_MODEL_ID"]], indirect=True)
def test_mistral_ai_chat_completion_init_constructor_missing_model(mistralai_unit_test_env) -> None:
# Test successful initialization
with pytest.raises(ServiceInitializationError):
MistralAIChatCompletion(api_key="overwrite_api_key", env_file_path="test.env")
@pytest.mark.parametrize("exclude_list", [["MISTRALAI_API_KEY", "MISTRALAI_CHAT_MODEL_ID"]], indirect=True)
def test_mistral_ai_chat_completion_init_constructor_missing_api_key(mistralai_unit_test_env) -> None:
# Test successful initialization
with pytest.raises(ServiceInitializationError):
MistralAIChatCompletion(ai_model_id="overwrite_model_id", env_file_path="test.env")
def test_mistral_ai_chat_completion_init_hybrid(mistralai_unit_test_env) -> None:
mistral_ai_chat_completion = MistralAIChatCompletion(
ai_model_id="overwrite_model_id",
env_file_path="test.env",
)
assert mistral_ai_chat_completion.ai_model_id == "overwrite_model_id"
assert mistral_ai_chat_completion.async_client.sdk_configuration.security.api_key == "test_api_key"
@pytest.mark.parametrize("exclude_list", [["MISTRALAI_CHAT_MODEL_ID"]], indirect=True)
def test_mistral_ai_chat_completion_init_with_empty_model_id(mistralai_unit_test_env) -> None:
with pytest.raises(ServiceInitializationError):
MistralAIChatCompletion(
env_file_path="test.env",
)
def test_prompt_execution_settings_class(mistralai_unit_test_env):
mistral_ai_chat_completion = MistralAIChatCompletion()
prompt_execution_settings = mistral_ai_chat_completion.get_prompt_execution_settings_class()
assert prompt_execution_settings == MistralAIChatPromptExecutionSettings
async def test_with_different_execution_settings(kernel: Kernel, mock_mistral_ai_client_completion: MagicMock):
chat_history = MagicMock()
settings = OpenAIChatPromptExecutionSettings(temperature=0.2, seed=2)
arguments = KernelArguments()
chat_completion_base = MistralAIChatCompletion(
ai_model_id="test_model_id", service_id="test", api_key="", async_client=mock_mistral_ai_client_completion
)
await chat_completion_base.get_chat_message_contents(
chat_history=chat_history, settings=settings, kernel=kernel, arguments=arguments
)
assert mock_mistral_ai_client_completion.chat.complete_async.call_args.kwargs["temperature"] == 0.2
assert mock_mistral_ai_client_completion.chat.complete_async.call_args.kwargs["seed"] == 2
async def test_with_different_execution_settings_stream(
kernel: Kernel, mock_mistral_ai_client_completion_stream: MagicMock
):
chat_history = MagicMock()
settings = OpenAIChatPromptExecutionSettings(temperature=0.2, seed=2)
arguments = KernelArguments()
chat_completion_base = MistralAIChatCompletion(
ai_model_id="test_model_id",
service_id="test",
api_key="",
async_client=mock_mistral_ai_client_completion_stream,
)
async for chunk in chat_completion_base.get_streaming_chat_message_contents(
chat_history, settings, kernel=kernel, arguments=arguments
):
continue
assert mock_mistral_ai_client_completion_stream.chat.stream_async.call_args.kwargs["temperature"] == 0.2
assert mock_mistral_ai_client_completion_stream.chat.stream_async.call_args.kwargs["seed"] == 2
async def test_mistral_ai_chat_completion_get_chat_message_contents_success():
"""Test get_chat_message_contents with a successful ChatCompletionResponse."""
# Mock the response from the Mistral chat complete_async.
mock_response = ChatCompletionResponse(
id="some_id",
object="object",
created=12345,
usage=UsageInfo(prompt_tokens=10, completion_tokens=20, total_tokens=30),
model="test-model",
choices=[
ChatCompletionChoice(
index=0,
message=AssistantMessage(role="assistant", content="Hello!"),
finish_reason="stop",
)
],
)
async_mock_client = MagicMock(spec=Mistral)
async_mock_client.chat = MagicMock()
async_mock_client.chat.complete_async = AsyncMock(return_value=mock_response)
chat_completion = MistralAIChatCompletion(
ai_model_id="test-model",
api_key="test_key",
async_client=async_mock_client,
)
# We create a ChatHistory.
chat_history = ChatHistory()
settings = MistralAIChatPromptExecutionSettings()
results = await chat_completion.get_chat_message_contents(chat_history, settings)
# We should have exactly one ChatMessageContent.
assert len(results) == 1
assert results[0].role.value == "assistant"
assert results[0].finish_reason is not None
assert results[0].finish_reason.value == "stop"
assert "Hello!" in results[0].content
async_mock_client.chat.complete_async.assert_awaited_once()
async def test_mistral_ai_chat_completion_get_chat_message_contents_failure():
"""Test get_chat_message_contents should raise ServiceResponseException if Mistral call fails."""
async_mock_client = MagicMock(spec=Mistral)
async_mock_client.chat = MagicMock()
async_mock_client.chat.complete_async = AsyncMock(side_effect=Exception("API error"))
chat_completion = MistralAIChatCompletion(
ai_model_id="test-model",
api_key="test_key",
async_client=async_mock_client,
)
chat_history = ChatHistory()
settings = MistralAIChatPromptExecutionSettings()
with pytest.raises(ServiceResponseException) as exc:
await chat_completion.get_chat_message_contents(chat_history, settings)
assert "service failed to complete the prompt" in str(exc.value)
async def test_mistral_ai_chat_completion_get_streaming_chat_message_contents_success():
"""Test get_streaming_chat_message_contents when streaming successfully."""
# We'll yield multiple chunks to simulate streaming.
mock_chunk1 = CompletionEvent(
data=CompletionChunk(
id="chunk1",
created=1,
model="test-model",
choices=[
CompletionResponseStreamChoice(
index=0,
delta=DeltaMessage(role="assistant", content="Hello "),
finish_reason=None,
)
],
)
)
mock_chunk2 = CompletionEvent(
data=CompletionChunk(
id="chunk1",
created=1,
model="test-model",
choices=[
CompletionResponseStreamChoice(
index=0,
delta=DeltaMessage(content="World!"),
finish_reason="stop",
)
],
)
)
async def mock_stream_async(**kwargs):
yield mock_chunk1
yield mock_chunk2
async_mock_client = MagicMock(spec=Mistral)
async_mock_client.chat = MagicMock()
async_mock_client.chat.stream_async = AsyncMock(return_value=mock_stream_async())
chat_completion = MistralAIChatCompletion(
ai_model_id="test-model",
api_key="test_key",
async_client=async_mock_client,
)
chat_history = ChatHistory()
settings = MistralAIChatPromptExecutionSettings()
collected_chunks = []
async for chunk_list in chat_completion.get_streaming_chat_message_contents(chat_history, settings):
collected_chunks.append(chunk_list)
# We expect two sets of chunk_list yields.
assert len(collected_chunks) == 2
assert len(collected_chunks[0]) == 1
assert len(collected_chunks[1]) == 1
# First chunk contains "Hello ", second chunk "World!".
assert collected_chunks[0][0].items[0].text == "Hello "
assert collected_chunks[1][0].items[0].text == "World!"
async def test_mistral_ai_chat_completion_get_streaming_chat_message_contents_failure():
"""Test get_streaming_chat_message_contents raising a ServiceResponseException on failure."""
async_mock_client = MagicMock(spec=Mistral)
async_mock_client.chat = MagicMock()
async_mock_client.chat.stream_async = AsyncMock(side_effect=Exception("Streaming error"))
chat_completion = MistralAIChatCompletion(
ai_model_id="test-model",
api_key="test_key",
async_client=async_mock_client,
)
chat_history = ChatHistory()
settings = MistralAIChatPromptExecutionSettings()
with pytest.raises(ServiceResponseException) as exc:
async for _ in chat_completion.get_streaming_chat_message_contents(chat_history, settings):
pass
assert "service failed to complete the prompt" in str(exc.value)
def test_mistral_ai_chat_completion_update_settings_from_function_call_configuration_mistral():
"""Test update_settings_from_function_call_configuration_mistral sets tools etc."""
chat_completion = MistralAIChatCompletion(
ai_model_id="test-model",
api_key="test_key",
)
# Create a mock settings object.
settings = MistralAIChatPromptExecutionSettings()
# Create a function choice config with some available functions.
config = FunctionCallChoiceConfiguration()
mock_func = MagicMock(
spec=KernelFunction,
)
mock_func.name = "my_func"
mock_func.description = "some desc"
mock_func.fully_qualified_name = "mod.my_func"
mock_func.parameters = []
config.available_functions = [mock_func]
# Call the update_settings_from_function_call_configuration_mistral with type=ANY.
chat_completion.update_settings_from_function_call_configuration_mistral(
function_choice_configuration=config,
settings=settings,
type=FunctionChoiceType.AUTO,
)
assert settings.tool_choice == FunctionChoiceType.AUTO.value
assert settings.tools is not None
assert len(settings.tools) == 1
assert settings.tools[0]["function"]["name"] == "mod.my_func"
def test_mistral_ai_chat_completion_reset_function_choice_settings():
"""Test that _reset_function_choice_settings resets specific attributes."""
chat_completion = MistralAIChatCompletion(
ai_model_id="test-model",
api_key="test_key",
)
settings = MistralAIChatPromptExecutionSettings(tool_choice="any", tools=[{"name": "func1"}])
chat_completion._reset_function_choice_settings(settings)
assert settings.tool_choice is None
assert settings.tools is None
def test_mistral_ai_chat_completion_service_url():
"""Test that service_url attempts to use _endpoint from the async_client."""
async_mock_client = MagicMock(spec=Mistral)
async_mock_client._endpoint = "mistral"
chat_completion = MistralAIChatCompletion(
ai_model_id="test-model",
api_key="test_key",
async_client=async_mock_client,
)
url = chat_completion.service_url()
assert url == "mistral"
@@ -0,0 +1,114 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, MagicMock
import pytest
from mistralai import Mistral
from mistralai.models import EmbeddingResponse
from semantic_kernel.connectors.ai.mistral_ai.services.mistral_ai_text_embedding import MistralAITextEmbedding
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError, ServiceResponseException
def test_embedding_with_env_variables(mistralai_unit_test_env):
text_embedding = MistralAITextEmbedding()
assert text_embedding.ai_model_id == "test_embedding_model_id"
assert text_embedding.async_client.sdk_configuration.security.api_key == "test_api_key"
@pytest.mark.parametrize("exclude_list", [["MISTRALAI_API_KEY", "MISTRALAI_EMBEDDING_MODEL_ID"]], indirect=True)
def test_embedding_with_constructor(mistralai_unit_test_env):
text_embedding = MistralAITextEmbedding(
api_key="overwrite-api-key",
ai_model_id="overwrite-model",
)
assert text_embedding.ai_model_id == "overwrite-model"
assert text_embedding.async_client.sdk_configuration.security.api_key == "overwrite-api-key"
def test_embedding_with_client(mistralai_unit_test_env):
client = MagicMock(spec=Mistral)
text_embedding = MistralAITextEmbedding(async_client=client)
assert text_embedding.async_client == client
assert text_embedding.ai_model_id == "test_embedding_model_id"
def test_embedding_with_api_key(mistralai_unit_test_env):
text_embedding = MistralAITextEmbedding(api_key="overwrite-api-key")
assert text_embedding.async_client.sdk_configuration.security.api_key == "overwrite-api-key"
assert text_embedding.ai_model_id == "test_embedding_model_id"
def test_embedding_with_model(mistralai_unit_test_env):
text_embedding = MistralAITextEmbedding(ai_model_id="overwrite-model")
assert text_embedding.ai_model_id == "overwrite-model"
assert text_embedding.async_client.sdk_configuration.security.api_key == "test_api_key"
@pytest.mark.parametrize("exclude_list", [["MISTRALAI_EMBEDDING_MODEL_ID"]], indirect=True)
def test_embedding_with_model_without_env(mistralai_unit_test_env):
text_embedding = MistralAITextEmbedding(ai_model_id="overwrite-model")
assert text_embedding.ai_model_id == "overwrite-model"
assert text_embedding.async_client.sdk_configuration.security.api_key == "test_api_key"
@pytest.mark.parametrize("exclude_list", [["MISTRALAI_EMBEDDING_MODEL_ID"]], indirect=True)
def test_embedding_missing_model(mistralai_unit_test_env):
with pytest.raises(ServiceInitializationError):
MistralAITextEmbedding(
env_file_path="test.env",
)
@pytest.mark.parametrize("exclude_list", [["MISTRALAI_API_KEY"]], indirect=True)
def test_embedding_missing_api_key(mistralai_unit_test_env):
with pytest.raises(ServiceInitializationError):
MistralAITextEmbedding(
env_file_path="test.env",
)
@pytest.mark.parametrize("exclude_list", [["MISTRALAI_API_KEY", "MISTRALAI_EMBEDDING_MODEL_ID"]], indirect=True)
def test_embedding_missing_api_key_constructor(mistralai_unit_test_env):
with pytest.raises(ServiceInitializationError):
MistralAITextEmbedding(
env_file_path="test.env",
)
@pytest.mark.parametrize("exclude_list", [["MISTRALAI_API_KEY", "MISTRALAI_EMBEDDING_MODEL_ID"]], indirect=True)
def test_embedding_missing_model_constructor(mistralai_unit_test_env):
with pytest.raises(ServiceInitializationError):
MistralAITextEmbedding(
api_key="test_api_key",
env_file_path="test.env",
)
async def test_embedding_generate_raw_embedding(mistralai_unit_test_env):
mock_client = AsyncMock(spec=Mistral)
mock_client.embeddings = AsyncMock()
mock_embedding_response = MagicMock(spec=EmbeddingResponse, data=[MagicMock(embedding=[1, 2, 3, 4, 5])])
mock_client.embeddings.create_async.return_value = mock_embedding_response
text_embedding = MistralAITextEmbedding(async_client=mock_client)
embedding = await text_embedding.generate_raw_embeddings(["test"])
assert embedding == [[1, 2, 3, 4, 5]]
async def test_embedding_generate_embedding(mistralai_unit_test_env):
mock_client = AsyncMock(spec=Mistral)
mock_client.embeddings = AsyncMock()
mock_embedding_response = MagicMock(spec=EmbeddingResponse, data=[MagicMock(embedding=[1, 2, 3, 4, 5])])
mock_client.embeddings.create_async.return_value = mock_embedding_response
text_embedding = MistralAITextEmbedding(async_client=mock_client)
embedding = await text_embedding.generate_embeddings(["test"])
assert embedding.tolist() == [[1, 2, 3, 4, 5]]
async def test_embedding_generate_embedding_exception(mistralai_unit_test_env):
mock_client = AsyncMock(spec=Mistral)
mock_client.embeddings = AsyncMock()
mock_client.embeddings.create_async.side_effect = Exception("Test Exception")
text_embedding = MistralAITextEmbedding(async_client=mock_client)
with pytest.raises(ServiceResponseException):
await text_embedding.generate_embeddings(["test"])
@@ -0,0 +1,125 @@
# Copyright (c) Microsoft. All rights reserved.
from semantic_kernel.connectors.ai.mistral_ai.prompt_execution_settings.mistral_ai_prompt_execution_settings import (
MistralAIChatPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
def test_default_mistralai_chat_prompt_execution_settings():
settings = MistralAIChatPromptExecutionSettings()
assert settings.temperature is None
assert settings.top_p is None
assert settings.max_tokens is None
assert settings.messages is None
def test_custom_mistralai_chat_prompt_execution_settings():
settings = MistralAIChatPromptExecutionSettings(
temperature=0.5,
top_p=0.5,
max_tokens=128,
messages=[{"role": "system", "content": "Hello"}],
)
assert settings.temperature == 0.5
assert settings.top_p == 0.5
assert settings.max_tokens == 128
assert settings.messages == [{"role": "system", "content": "Hello"}]
def test_mistralai_chat_prompt_execution_settings_from_default_completion_config():
settings = PromptExecutionSettings(service_id="test_service")
chat_settings = MistralAIChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.service_id == "test_service"
assert chat_settings.temperature is None
assert chat_settings.top_p is None
assert chat_settings.max_tokens is None
def test_mistral_chat_prompt_execution_settings_from_openai_prompt_execution_settings():
chat_settings = MistralAIChatPromptExecutionSettings(service_id="test_service", temperature=1.0)
new_settings = MistralAIChatPromptExecutionSettings(service_id="test_2", temperature=0.0)
chat_settings.update_from_prompt_execution_settings(new_settings)
assert chat_settings.service_id == "test_2"
assert chat_settings.temperature == 0.0
def test_mistral_chat_prompt_execution_settings_from_custom_completion_config():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"temperature": 0.5,
"top_p": 0.5,
"max_tokens": 128,
"messages": [{"role": "system", "content": "Hello"}],
},
)
chat_settings = MistralAIChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.temperature == 0.5
assert chat_settings.top_p == 0.5
assert chat_settings.max_tokens == 128
def test_openai_chat_prompt_execution_settings_from_custom_completion_config_with_none():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"temperature": 0.5,
"top_p": 0.5,
"max_tokens": 128,
"messages": [{"role": "system", "content": "Hello"}],
},
)
chat_settings = MistralAIChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.temperature == 0.5
assert chat_settings.top_p == 0.5
assert chat_settings.max_tokens == 128
def test_openai_chat_prompt_execution_settings_from_custom_completion_config_with_functions():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"temperature": 0.5,
"top_p": 0.5,
"max_tokens": 128,
"tools": [{}],
"messages": [{"role": "system", "content": "Hello"}],
},
)
chat_settings = MistralAIChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.temperature == 0.5
assert chat_settings.top_p == 0.5
assert chat_settings.max_tokens == 128
def test_create_options():
settings = MistralAIChatPromptExecutionSettings(
service_id="test_service",
extension_data={
"temperature": 0.5,
"top_p": 0.5,
"max_tokens": 128,
"tools": [{}],
"messages": [{"role": "system", "content": "Hello"}],
},
)
options = settings.prepare_settings_dict()
assert options["temperature"] == 0.5
assert options["top_p"] == 0.5
assert options["max_tokens"] == 128
def test_create_options_with_function_choice_behavior():
settings = MistralAIChatPromptExecutionSettings(
service_id="test_service",
function_choice_behavior="auto",
extension_data={
"temperature": 0.5,
"top_p": 0.5,
"max_tokens": 128,
"tools": [{}],
"messages": [{"role": "system", "content": "Hello"}],
},
)
assert settings.function_choice_behavior
@@ -0,0 +1,101 @@
# Copyright (c) Microsoft. All rights reserved.
import pytest
from pydantic import BaseModel, ValidationError
from semantic_kernel.connectors.ai.nvidia.prompt_execution_settings.nvidia_prompt_execution_settings import (
NvidiaChatPromptExecutionSettings,
NvidiaEmbeddingPromptExecutionSettings,
NvidiaPromptExecutionSettings,
)
class TestNvidiaPromptExecutionSettings:
"""Test cases for NvidiaPromptExecutionSettings."""
def test_init_with_defaults(self):
"""Test initialization with default values."""
settings = NvidiaPromptExecutionSettings()
assert settings.format is None
assert settings.options is None
def test_init_with_values(self):
"""Test initialization with specific values."""
settings = NvidiaPromptExecutionSettings(
format="json",
options={"key": "value"},
)
assert settings.format == "json"
assert settings.options == {"key": "value"}
def test_validation_format_values(self):
"""Test format validation values."""
# Valid values
settings = NvidiaPromptExecutionSettings(format="json")
assert settings.format == "json"
class TestNvidiaChatPromptExecutionSettings:
"""Test cases for NvidiaChatPromptExecutionSettings."""
def test_init_with_defaults(self):
"""Test initialization with default values."""
settings = NvidiaChatPromptExecutionSettings()
assert settings.messages is None
assert settings.response_format is None
def test_response_format_with_pydantic_model(self):
"""Test response_format with Pydantic model."""
class TestModel(BaseModel):
name: str
value: int
settings = NvidiaChatPromptExecutionSettings(response_format=TestModel)
assert settings.response_format == TestModel
def test_response_format_with_dict(self):
"""Test response_format with dictionary."""
settings = NvidiaChatPromptExecutionSettings(response_format={"type": "json_object"})
assert settings.response_format == {"type": "json_object"}
class TestNvidiaEmbeddingPromptExecutionSettings:
"""Test cases for NvidiaEmbeddingPromptExecutionSettings."""
def test_init_with_defaults(self):
"""Test initialization with default values."""
settings = NvidiaEmbeddingPromptExecutionSettings()
assert settings.input is None
assert settings.encoding_format == "float"
assert settings.input_type == "query"
assert settings.truncate == "NONE"
def test_init_with_values(self):
"""Test initialization with specific values."""
settings = NvidiaEmbeddingPromptExecutionSettings(
input=["hello", "world"],
encoding_format="base64",
input_type="passage",
truncate="START",
)
assert settings.input == ["hello", "world"]
assert settings.encoding_format == "base64"
assert settings.input_type == "passage"
assert settings.truncate == "START"
def test_validation_encoding_format(self):
"""Test encoding_format validation."""
# Valid values
settings = NvidiaEmbeddingPromptExecutionSettings(encoding_format="float")
assert settings.encoding_format == "float"
settings = NvidiaEmbeddingPromptExecutionSettings(encoding_format="base64")
assert settings.encoding_format == "base64"
# Invalid values
with pytest.raises(ValidationError):
NvidiaEmbeddingPromptExecutionSettings(encoding_format="invalid")
@@ -0,0 +1,151 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, patch
import pytest
from openai.resources.chat.completions import AsyncCompletions
from openai.types.chat import ChatCompletion, ChatCompletionMessage
from openai.types.chat.chat_completion import Choice
from openai.types.completion_usage import CompletionUsage
from pydantic import BaseModel
from semantic_kernel.connectors.ai.nvidia import NvidiaChatCompletion
from semantic_kernel.connectors.ai.nvidia.prompt_execution_settings.nvidia_prompt_execution_settings import (
NvidiaChatPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.nvidia.services.nvidia_chat_completion import DEFAULT_NVIDIA_CHAT_MODEL
from semantic_kernel.contents import ChatHistory
from semantic_kernel.exceptions import ServiceInitializationError, ServiceResponseException
@pytest.fixture
def nvidia_unit_test_env(monkeypatch, exclude_list, override_env_param_dict):
"""Fixture to set environment variables for NvidiaChatCompletion."""
if exclude_list is None:
exclude_list = []
if override_env_param_dict is None:
override_env_param_dict = {}
env_vars = {"NVIDIA_API_KEY": "test_api_key", "NVIDIA_CHAT_MODEL_ID": "meta/llama-3.1-8b-instruct"}
env_vars.update(override_env_param_dict)
for key, value in env_vars.items():
if key not in exclude_list:
monkeypatch.setenv(key, value)
else:
monkeypatch.delenv(key, raising=False)
return env_vars
def _create_mock_chat_completion(content: str = "Hello!") -> ChatCompletion:
"""Helper function to create a mock ChatCompletion response."""
message = ChatCompletionMessage(role="assistant", content=content)
choice = Choice(
finish_reason="stop",
index=0,
message=message,
)
usage = CompletionUsage(completion_tokens=20, prompt_tokens=10, total_tokens=30)
return ChatCompletion(
id="test-id",
choices=[choice],
created=1234567890,
model="meta/llama-3.1-8b-instruct",
object="chat.completion",
usage=usage,
)
class TestNvidiaChatCompletion:
"""Test cases for NvidiaChatCompletion."""
def test_init_with_defaults(self, nvidia_unit_test_env):
"""Test initialization with default values."""
service = NvidiaChatCompletion()
assert service.ai_model_id == nvidia_unit_test_env["NVIDIA_CHAT_MODEL_ID"]
def test_get_prompt_execution_settings_class(self, nvidia_unit_test_env):
"""Test getting the prompt execution settings class."""
service = NvidiaChatCompletion()
from semantic_kernel.connectors.ai.nvidia.prompt_execution_settings.nvidia_prompt_execution_settings import (
NvidiaChatPromptExecutionSettings,
)
assert service.get_prompt_execution_settings_class() == NvidiaChatPromptExecutionSettings
@pytest.mark.parametrize("exclude_list", [["NVIDIA_API_KEY"]], indirect=True)
def test_init_with_empty_api_key(self, nvidia_unit_test_env):
"""Test initialization fails with empty API key."""
with pytest.raises(ServiceInitializationError):
NvidiaChatCompletion()
@pytest.mark.parametrize("exclude_list", [["NVIDIA_CHAT_MODEL_ID"]], indirect=True)
def test_init_with_empty_model_id(self, nvidia_unit_test_env):
"""Test initialization with empty model ID uses default."""
service = NvidiaChatCompletion()
assert service.ai_model_id == DEFAULT_NVIDIA_CHAT_MODEL
def test_init_with_custom_model_id(self, nvidia_unit_test_env):
"""Test initialization with custom model ID."""
custom_model = "custom/nvidia-model"
service = NvidiaChatCompletion(ai_model_id=custom_model)
assert service.ai_model_id == custom_model
@pytest.mark.asyncio
@patch.object(AsyncCompletions, "create", new_callable=AsyncMock)
async def test_get_chat_message_contents(self, mock_create, nvidia_unit_test_env):
"""Test basic chat completion."""
mock_create.return_value = _create_mock_chat_completion("Hello!")
service = NvidiaChatCompletion()
chat_history = ChatHistory()
chat_history.add_user_message("Hello")
settings = NvidiaChatPromptExecutionSettings()
result = await service.get_chat_message_contents(chat_history, settings)
assert len(result) == 1
assert result[0].content == "Hello!"
@pytest.mark.asyncio
@patch.object(AsyncCompletions, "create", new_callable=AsyncMock)
async def test_structured_output_with_pydantic_model(self, mock_create, nvidia_unit_test_env):
"""Test structured output with Pydantic model."""
# Define test model
class TestModel(BaseModel):
name: str
value: int
mock_create.return_value = _create_mock_chat_completion('{"name": "test", "value": 42}')
service = NvidiaChatCompletion()
chat_history = ChatHistory()
chat_history.add_user_message("Give me structured data")
settings = NvidiaChatPromptExecutionSettings()
settings.response_format = TestModel
await service.get_chat_message_contents(chat_history, settings)
# Verify nvext was passed
call_args = mock_create.call_args[1]
assert "extra_body" in call_args
assert "nvext" in call_args["extra_body"]
assert "guided_json" in call_args["extra_body"]["nvext"]
@pytest.mark.asyncio
@patch.object(AsyncCompletions, "create", new_callable=AsyncMock)
async def test_error_handling(self, mock_create, nvidia_unit_test_env):
"""Test error handling."""
mock_create.side_effect = Exception("API Error")
service = NvidiaChatCompletion()
chat_history = ChatHistory()
chat_history.add_user_message("Hello")
settings = NvidiaChatPromptExecutionSettings()
with pytest.raises(ServiceResponseException):
await service.get_chat_message_contents(chat_history, settings)
@@ -0,0 +1,152 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, MagicMock
import pytest
from openai import AsyncOpenAI
from semantic_kernel.connectors.ai.nvidia.prompt_execution_settings.nvidia_prompt_execution_settings import (
NvidiaChatPromptExecutionSettings,
NvidiaEmbeddingPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.nvidia.services.nvidia_handler import NvidiaHandler
from semantic_kernel.connectors.ai.nvidia.services.nvidia_model_types import NvidiaModelTypes
@pytest.fixture
def mock_openai_client():
"""Create a mock OpenAI client."""
return AsyncMock(spec=AsyncOpenAI)
@pytest.fixture
def nvidia_handler(mock_openai_client):
"""Create a NvidiaHandler instance with mocked client."""
return NvidiaHandler(
client=mock_openai_client,
ai_model_type=NvidiaModelTypes.CHAT,
ai_model_id="test-model",
api_key="test-key",
)
class TestNvidiaHandler:
"""Test cases for NvidiaHandler."""
def test_init(self, mock_openai_client):
"""Test initialization."""
handler = NvidiaHandler(
client=mock_openai_client,
ai_model_type=NvidiaModelTypes.CHAT,
)
assert handler.client == mock_openai_client
assert handler.ai_model_type == NvidiaModelTypes.CHAT
assert handler.MODEL_PROVIDER_NAME == "nvidia"
@pytest.mark.asyncio
async def test_send_chat_completion_request(self, nvidia_handler, mock_openai_client):
"""Test sending chat completion request."""
# Mock the response
mock_response = MagicMock()
mock_response.choices = [
MagicMock(
message=MagicMock(role="assistant", content="Hello!"),
finish_reason="stop",
)
]
mock_response.usage = MagicMock(prompt_tokens=10, completion_tokens=20, total_tokens=30)
mock_openai_client.chat.completions.create = AsyncMock(return_value=mock_response)
# Create settings
settings = NvidiaChatPromptExecutionSettings(
messages=[{"role": "user", "content": "Hello"}],
model="test-model",
)
# Test the method
result = await nvidia_handler._send_chat_completion_request(settings)
assert result == mock_response
# Verify usage was stored
assert nvidia_handler.prompt_tokens == 10
assert nvidia_handler.completion_tokens == 20
assert nvidia_handler.total_tokens == 30
@pytest.mark.asyncio
async def test_send_chat_completion_request_with_nvext(self, nvidia_handler, mock_openai_client):
"""Test sending chat completion request with nvext parameter."""
# Mock the response
mock_response = MagicMock()
mock_response.choices = [
MagicMock(
message=MagicMock(role="assistant", content='{"result": "success"}'),
finish_reason="stop",
)
]
mock_response.usage = MagicMock(prompt_tokens=10, completion_tokens=20, total_tokens=30)
mock_openai_client.chat.completions.create = AsyncMock(return_value=mock_response)
# Create settings with nvext
settings = NvidiaChatPromptExecutionSettings(
messages=[{"role": "user", "content": "Give me JSON"}],
model="test-model",
extra_body={"nvext": {"guided_json": {"type": "object"}}},
)
# Test the method
result = await nvidia_handler._send_chat_completion_request(settings)
assert result == mock_response
# Verify the client was called with nvext in extra_body
call_args = mock_openai_client.chat.completions.create.call_args[1]
assert "extra_body" in call_args
assert "nvext" in call_args["extra_body"]
assert call_args["extra_body"]["nvext"] == {"guided_json": {"type": "object"}}
@pytest.mark.asyncio
async def test_send_embedding_request(self, mock_openai_client):
"""Test sending embedding request."""
handler = NvidiaHandler(
client=mock_openai_client,
ai_model_type=NvidiaModelTypes.EMBEDDING,
ai_model_id="test-model",
)
# Mock the response
mock_response = MagicMock()
mock_response.data = [
MagicMock(embedding=[0.1, 0.2, 0.3]),
MagicMock(embedding=[0.4, 0.5, 0.6]),
]
mock_response.usage = MagicMock(prompt_tokens=10, total_tokens=10)
mock_openai_client.embeddings.create = AsyncMock(return_value=mock_response)
# Create settings
settings = NvidiaEmbeddingPromptExecutionSettings(
input=["hello", "world"],
model="test-model",
)
# Test the method
result = await handler._send_embedding_request(settings)
assert result == [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]]
@pytest.mark.asyncio
async def test_send_request_unsupported_model_type(self, mock_openai_client):
"""Test send_request with unsupported model type."""
# Create a handler with invalid model type by bypassing validation
handler = NvidiaHandler(
client=mock_openai_client,
ai_model_type=NvidiaModelTypes.CHAT,
)
# Manually set the attribute to bypass Pydantic validation
object.__setattr__(handler, "ai_model_type", "UNSUPPORTED")
settings = NvidiaChatPromptExecutionSettings(
messages=[{"role": "user", "content": "Hello"}],
model="test-model",
)
with pytest.raises(NotImplementedError, match="Model type UNSUPPORTED is not supported"):
await handler._send_request(settings)
@@ -0,0 +1,134 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import AsyncMock, patch
import pytest
from openai import AsyncClient
from openai.resources.embeddings import AsyncEmbeddings
from semantic_kernel.connectors.ai.nvidia.prompt_execution_settings.nvidia_prompt_execution_settings import (
NvidiaEmbeddingPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.nvidia.services.nvidia_text_embedding import NvidiaTextEmbedding
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError, ServiceResponseException
@pytest.fixture
def nvidia_unit_test_env(monkeypatch, exclude_list, override_env_param_dict):
"""Fixture to set environment variables for NvidiaTextEmbedding."""
if exclude_list is None:
exclude_list = []
if override_env_param_dict is None:
override_env_param_dict = {}
env_vars = {"NVIDIA_API_KEY": "test_api_key", "NVIDIA_EMBEDDING_MODEL_ID": "test_embedding_model_id"}
env_vars.update(override_env_param_dict)
for key, value in env_vars.items():
if key not in exclude_list:
monkeypatch.setenv(key, value)
else:
monkeypatch.delenv(key, raising=False)
return env_vars
def test_init(nvidia_unit_test_env):
nvidia_text_embedding = NvidiaTextEmbedding()
assert nvidia_text_embedding.client is not None
assert isinstance(nvidia_text_embedding.client, AsyncClient)
assert nvidia_text_embedding.ai_model_id == nvidia_unit_test_env["NVIDIA_EMBEDDING_MODEL_ID"]
assert nvidia_text_embedding.get_prompt_execution_settings_class() == NvidiaEmbeddingPromptExecutionSettings
def test_init_validation_fail() -> None:
with pytest.raises(ServiceInitializationError):
NvidiaTextEmbedding(api_key="34523", ai_model_id={"test": "dict"})
def test_init_to_from_dict(nvidia_unit_test_env):
default_headers = {"X-Unit-Test": "test-guid"}
settings = {
"ai_model_id": nvidia_unit_test_env["NVIDIA_EMBEDDING_MODEL_ID"],
"api_key": nvidia_unit_test_env["NVIDIA_API_KEY"],
"default_headers": default_headers,
}
text_embedding = NvidiaTextEmbedding.from_dict(settings)
dumped_settings = text_embedding.to_dict()
assert dumped_settings["ai_model_id"] == settings["ai_model_id"]
assert dumped_settings["api_key"] == settings["api_key"]
@patch.object(AsyncEmbeddings, "create", new_callable=AsyncMock)
async def test_embedding_calls_with_parameters(mock_create, nvidia_unit_test_env) -> None:
ai_model_id = "NV-Embed-QA"
texts = ["hello world", "goodbye world"]
embedding_dimensions = 1536
nvidia_text_embedding = NvidiaTextEmbedding(
ai_model_id=ai_model_id,
)
await nvidia_text_embedding.generate_embeddings(texts, dimensions=embedding_dimensions)
mock_create.assert_awaited_once_with(
input=texts,
model=ai_model_id,
dimensions=embedding_dimensions,
encoding_format="float",
extra_body={"input_type": "query", "truncate": "NONE"},
)
@patch.object(AsyncEmbeddings, "create", new_callable=AsyncMock)
async def test_embedding_calls_with_settings(mock_create, nvidia_unit_test_env) -> None:
ai_model_id = "test_model_id"
texts = ["hello world", "goodbye world"]
settings = NvidiaEmbeddingPromptExecutionSettings(service_id="default")
nvidia_text_embedding = NvidiaTextEmbedding(service_id="default", ai_model_id=ai_model_id)
await nvidia_text_embedding.generate_embeddings(texts, settings=settings)
mock_create.assert_awaited_once_with(
input=texts,
model=ai_model_id,
encoding_format="float",
extra_body={"input_type": "query", "truncate": "NONE"},
)
@patch.object(AsyncEmbeddings, "create", new_callable=AsyncMock, side_effect=Exception)
async def test_embedding_fail(mock_create, nvidia_unit_test_env) -> None:
ai_model_id = "test_model_id"
texts = ["hello world", "goodbye world"]
nvidia_text_embedding = NvidiaTextEmbedding(
ai_model_id=ai_model_id,
)
with pytest.raises(ServiceResponseException):
await nvidia_text_embedding.generate_embeddings(texts)
@patch.object(AsyncEmbeddings, "create", new_callable=AsyncMock)
async def test_embedding_pes(mock_create, nvidia_unit_test_env) -> None:
ai_model_id = "test_model_id"
texts = ["hello world", "goodbye world"]
pes = PromptExecutionSettings(service_id="x", ai_model_id=ai_model_id)
nvidia_text_embedding = NvidiaTextEmbedding(ai_model_id=ai_model_id)
await nvidia_text_embedding.generate_raw_embeddings(texts, pes)
mock_create.assert_awaited_once_with(
input=texts,
model=ai_model_id,
encoding_format="float",
extra_body={"input_type": "query", "truncate": "NONE"},
)
@@ -0,0 +1,56 @@
# Copyright (c) Microsoft. All rights reserved.
from semantic_kernel.connectors.ai.nvidia.settings.nvidia_settings import NvidiaSettings
class TestNvidiaSettings:
"""Test cases for NvidiaSettings."""
def test_init_with_defaults(self):
"""Test initialization with default values."""
settings = NvidiaSettings()
assert settings.api_key is None
assert settings.base_url == "https://integrate.api.nvidia.com/v1"
assert settings.embedding_model_id is None
assert settings.chat_model_id is None
def test_init_with_values(self):
"""Test initialization with specific values."""
settings = NvidiaSettings(
api_key="test-api-key",
base_url="https://custom.nvidia.com/v1",
embedding_model_id="test-embedding-model",
chat_model_id="test-chat-model",
)
assert settings.api_key.get_secret_value() == "test-api-key"
assert settings.base_url == "https://custom.nvidia.com/v1"
assert settings.embedding_model_id == "test-embedding-model"
assert settings.chat_model_id == "test-chat-model"
def test_env_prefix(self):
"""Test environment variable prefix."""
assert NvidiaSettings.env_prefix == "NVIDIA_"
def test_api_key_secret_str(self):
"""Test that api_key is properly handled as SecretStr."""
settings = NvidiaSettings(api_key="secret-key")
# Should be SecretStr type
assert hasattr(settings.api_key, "get_secret_value")
assert settings.api_key.get_secret_value() == "secret-key"
# Should not expose the secret in string representation
str_repr = str(settings)
assert "secret-key" not in str_repr
def test_environment_variables(self, monkeypatch):
"""Test that environment variables override defaults."""
monkeypatch.setenv("NVIDIA_API_KEY", "env-key")
monkeypatch.setenv("NVIDIA_CHAT_MODEL_ID", "env-chat")
settings = NvidiaSettings()
assert settings.api_key.get_secret_value() == "env-key"
assert settings.chat_model_id == "env-chat"
@@ -0,0 +1,84 @@
# Copyright (c) Microsoft. All rights reserved.
from collections.abc import AsyncGenerator, AsyncIterator
from unittest.mock import MagicMock
import pytest
from ollama import AsyncClient
from semantic_kernel.contents.chat_history import ChatHistory
@pytest.fixture()
def model_id() -> str:
return "test_model_id"
@pytest.fixture()
def service_id() -> str:
return "test_service_id"
@pytest.fixture()
def host() -> str:
return "http://localhost:5000"
@pytest.fixture()
def custom_client() -> AsyncClient:
return AsyncClient("http://localhost:5001")
@pytest.fixture()
def chat_history() -> ChatHistory:
chat_history = ChatHistory()
chat_history.add_user_message("test_prompt")
return chat_history
@pytest.fixture()
def prompt() -> str:
return "test_prompt"
@pytest.fixture()
def default_options() -> dict:
return {"test": "test"}
@pytest.fixture()
def ollama_unit_test_env(monkeypatch, host, exclude_list):
"""Fixture to set environment variables for OllamaSettings."""
if exclude_list is None:
exclude_list = []
env_vars = {
"OLLAMA_CHAT_MODEL_ID": "test_chat_model_id",
"OLLAMA_TEXT_MODEL_ID": "test_text_model_id",
"OLLAMA_EMBEDDING_MODEL_ID": "test_embedding_model_id",
"OLLAMA_HOST": host,
}
for key, value in env_vars.items():
if key not in exclude_list:
monkeypatch.setenv(key, value)
else:
monkeypatch.delenv(key, raising=False)
return env_vars
@pytest.fixture()
def mock_streaming_text_response() -> AsyncIterator:
streaming_text_response = MagicMock(spec=AsyncGenerator)
streaming_text_response.__aiter__.return_value = [{"response": "test_response"}]
return streaming_text_response
@pytest.fixture()
def mock_streaming_chat_response() -> AsyncIterator:
streaming_chat_response = MagicMock(spec=AsyncGenerator)
streaming_chat_response.__aiter__.return_value = [{"message": {"content": "test_response"}}]
return streaming_chat_response
@@ -0,0 +1,453 @@
# Copyright (c) Microsoft. All rights reserved.
from collections.abc import AsyncGenerator
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
from ollama import AsyncClient
import semantic_kernel.connectors.ai.ollama.services.ollama_chat_completion as occ_module
from semantic_kernel.connectors.ai.completion_usage import CompletionUsage
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior
from semantic_kernel.connectors.ai.ollama.ollama_prompt_execution_settings import OllamaChatPromptExecutionSettings
from semantic_kernel.connectors.ai.ollama.services.ollama_chat_completion import OllamaChatCompletion
from semantic_kernel.contents.chat_history import ChatHistory
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.function_call_content import FunctionCallContent
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
from semantic_kernel.contents.streaming_text_content import StreamingTextContent
from semantic_kernel.exceptions.service_exceptions import (
ServiceInitializationError,
ServiceInvalidExecutionSettingsError,
ServiceInvalidResponseError,
)
def test_settings(model_id):
"""Test that the settings class is correct."""
ollama = OllamaChatCompletion(ai_model_id=model_id)
settings = ollama.get_prompt_execution_settings_class()
assert settings == OllamaChatPromptExecutionSettings
def test_init_empty_service_id(model_id):
"""Test that the service initializes correctly with an empty service id."""
ollama = OllamaChatCompletion(ai_model_id=model_id)
assert ollama.service_id == model_id
def test_init_empty_string_ai_model_id():
"""Test that the service initializes with a error if there is no ai_model_id."""
with pytest.raises(ServiceInitializationError):
_ = OllamaChatCompletion(ai_model_id="")
def test_custom_client(model_id, custom_client):
"""Test that the service initializes correctly with a custom client."""
ollama = OllamaChatCompletion(ai_model_id=model_id, client=custom_client)
assert ollama.client == custom_client
def test_invalid_ollama_settings():
"""Test that the service initializes incorrectly with invalid settings."""
with pytest.raises(ServiceInitializationError):
_ = OllamaChatCompletion(ai_model_id=123)
@pytest.mark.parametrize("exclude_list", [["OLLAMA_CHAT_MODEL_ID"]], indirect=True)
def test_init_empty_model_id_in_env(ollama_unit_test_env):
"""Test that the service initializes incorrectly with an empty model id."""
with pytest.raises(ServiceInitializationError):
_ = OllamaChatCompletion(env_file_path="fake_env_file_path.env")
def test_function_choice_settings(ollama_unit_test_env):
"""Test that REQUIRED and NONE function choice settings are unsupported."""
ollama = OllamaChatCompletion()
with pytest.raises(ServiceInvalidExecutionSettingsError):
ollama._verify_function_choice_settings(
OllamaChatPromptExecutionSettings(function_choice_behavior=FunctionChoiceBehavior.Required())
)
with pytest.raises(ServiceInvalidExecutionSettingsError):
ollama._verify_function_choice_settings(
OllamaChatPromptExecutionSettings(function_choice_behavior=FunctionChoiceBehavior.NoneInvoke())
)
def test_service_url(ollama_unit_test_env):
"""Test that the service URL is correct."""
ollama = OllamaChatCompletion()
assert ollama.service_url() == ollama_unit_test_env["OLLAMA_HOST"]
@patch("ollama.AsyncClient.__init__", return_value=None) # mock_client
@patch("ollama.AsyncClient.chat") # mock_chat_client
async def test_custom_host(
mock_chat_client,
mock_client,
model_id,
service_id,
host,
chat_history,
prompt,
default_options,
):
"""Test that the service initializes and generates content correctly with a custom host."""
mock_chat_client.return_value = {"message": {"content": "test_response"}}
ollama = OllamaChatCompletion(ai_model_id=model_id, host=host)
chat_responses = await ollama.get_chat_message_contents(
chat_history,
OllamaChatPromptExecutionSettings(service_id=service_id, options=default_options),
)
# Check that the client was initialized once with the correct host
assert mock_client.call_count == 1
mock_client.assert_called_with(host=host)
# Check that the chat client was called once and the responses are correct
assert mock_chat_client.call_count == 1
assert len(chat_responses) == 1
assert chat_responses[0].content == "test_response"
@patch("ollama.AsyncClient.__init__", return_value=None) # mock_client
@patch("ollama.AsyncClient.chat") # mock_chat_client
async def test_custom_host_streaming(
mock_chat_client,
mock_client,
mock_streaming_chat_response,
model_id,
service_id,
host,
chat_history,
prompt,
default_options,
):
"""Test that the service initializes and generates streaming content correctly with a custom host."""
mock_chat_client.return_value = mock_streaming_chat_response
ollama = OllamaChatCompletion(ai_model_id=model_id, host=host)
async for messages in ollama.get_streaming_chat_message_contents(
chat_history,
OllamaChatPromptExecutionSettings(service_id=service_id, options=default_options),
):
assert len(messages) == 1
assert messages[0].role == "assistant"
assert messages[0].content == "test_response"
# Check that the client was initialized once with the correct host
assert mock_client.call_count == 1
mock_client.assert_called_with(host=host)
# Check that the chat client was called once
assert mock_chat_client.call_count == 1
@patch("ollama.AsyncClient.chat")
async def test_chat_completion(mock_chat_client, model_id, service_id, chat_history, default_options):
"""Test that the chat completion service completes correctly."""
mock_chat_client.return_value = {"message": {"content": "test_response"}}
ollama = OllamaChatCompletion(ai_model_id=model_id)
response = await ollama.get_chat_message_contents(
chat_history=chat_history,
settings=OllamaChatPromptExecutionSettings(service_id=service_id, options=default_options),
)
assert len(response) == 1
assert response[0].content == "test_response"
mock_chat_client.assert_called_once_with(
model=model_id,
messages=ollama._prepare_chat_history_for_request(chat_history),
options=default_options,
stream=False,
)
@patch("ollama.AsyncClient.chat")
async def test_chat_completion_wrong_return_type(
mock_chat_client,
mock_streaming_chat_response,
model_id,
service_id,
chat_history,
default_options,
):
"""Test that the chat completion service fails when the return type is incorrect."""
mock_chat_client.return_value = mock_streaming_chat_response # should not be a streaming response
ollama = OllamaChatCompletion(ai_model_id=model_id)
with pytest.raises(ServiceInvalidResponseError):
await ollama.get_chat_message_contents(
chat_history,
OllamaChatPromptExecutionSettings(service_id=service_id, options=default_options),
)
@patch("ollama.AsyncClient.chat")
async def test_streaming_chat_completion(
mock_chat_client,
mock_streaming_chat_response,
model_id,
service_id,
chat_history,
default_options,
):
"""Test that the streaming chat completion service completes correctly."""
mock_chat_client.return_value = mock_streaming_chat_response
ollama = OllamaChatCompletion(ai_model_id=model_id)
response = ollama.get_streaming_chat_message_contents(
chat_history,
OllamaChatPromptExecutionSettings(service_id=service_id, options=default_options),
)
responses = []
async for line in response:
if line:
assert line[0].content == "test_response"
responses.append(line[0].content)
assert len(responses) == 1
mock_chat_client.assert_called_once_with(
model=model_id,
messages=ollama._prepare_chat_history_for_request(chat_history),
options=default_options,
stream=True,
)
@patch("ollama.AsyncClient.chat")
async def test_streaming_chat_completion_wrong_return_type(
mock_chat_client,
model_id,
service_id,
chat_history,
default_options,
):
"""Test that the chat completion streaming service fails when the return type is incorrect."""
mock_chat_client.return_value = {"message": {"content": "test_response"}} # should not be a non-streaming response
ollama = OllamaChatCompletion(ai_model_id=model_id)
with pytest.raises(ServiceInvalidResponseError):
async for _ in ollama.get_streaming_chat_message_contents(
chat_history,
OllamaChatPromptExecutionSettings(service_id=service_id, options=default_options),
):
pass
@pytest.fixture
async def setup_ollama_chat_completion():
async_client_mock = AsyncMock(spec=AsyncClient)
async_client_mock.chat = AsyncMock()
ollama_chat_completion = OllamaChatCompletion(
service_id="test_service", ai_model_id="test_model_id", client=async_client_mock
)
return ollama_chat_completion, async_client_mock
async def test_service_url_new(setup_ollama_chat_completion):
ollama_chat_completion, async_client_mock = setup_ollama_chat_completion
# Mock the client's internal structure
async_client_mock._client = AsyncMock(spec=httpx.AsyncClient)
async_client_mock._client.base_url = "http://mocked_base_url"
service_url = ollama_chat_completion.service_url()
assert service_url == "http://mocked_base_url"
async def test_prepare_chat_history_for_request(setup_ollama_chat_completion):
ollama_chat_completion, _ = setup_ollama_chat_completion
chat_history = MagicMock(spec=ChatHistory)
chat_history.messages = []
prepared_history = ollama_chat_completion._prepare_chat_history_for_request(chat_history)
assert prepared_history == []
async def test_service_url_with_httpx_client(model_id: str) -> None:
"""
Test that service_url returns the base_url of the underlying httpx.AsyncClient.
"""
# Initialize an AsyncClient and manually set its _client attribute to an httpx.AsyncClient
client = AsyncClient(host="unused")
base = httpx.AsyncClient(base_url="http://example.com:8000")
client._client = base # simulate underlying httpx client
ollama = OllamaChatCompletion(ai_model_id=model_id, client=client)
# service_url should reflect the base_url of the httpx client
assert ollama.service_url() == "http://example.com:8000"
@patch("ollama.AsyncClient.chat", new_callable=AsyncMock)
async def test_chat_response_branch(
mock_chat: AsyncMock,
model_id: str,
service_id: str,
default_options: dict,
chat_history,
monkeypatch,
) -> None:
"""
Test get_chat_message_contents when AsyncClient.chat returns a ChatResponse instance.
"""
class DummyFunction:
def __init__(self, name, arguments):
self.name = name
self.arguments = arguments
class DummyToolCall:
def __init__(self, function):
self.function = function
class DummyMessage:
def __init__(self, content: str, tool_calls=None) -> None:
self.content = content
self.tool_calls = tool_calls or []
class DummyChatResponse:
def __init__(
self,
content: str,
model: str,
prompt_eval_count: int,
eval_count: int,
tool_calls=None,
) -> None:
function_calls = [
DummyToolCall(DummyFunction(tc["function"]["name"], tc["function"]["arguments"])) for tc in tool_calls
]
self.message = DummyMessage(content, function_calls)
self.model = model
self.prompt_eval_count = prompt_eval_count
self.eval_count = eval_count
# Monkeypatch the ChatResponse type in the module so isinstance works
monkeypatch.setattr(occ_module, "ChatResponse", DummyChatResponse)
# Prepare a dummy ChatResponse return value
dummy_resp = DummyChatResponse(
content="resp_text",
model="mdl",
prompt_eval_count=2,
eval_count=3,
tool_calls=[{"function": {"name": "fn", "arguments": {"x": 1}}}],
)
mock_chat.return_value = dummy_resp
ollama = OllamaChatCompletion(ai_model_id=model_id)
settings = OllamaChatPromptExecutionSettings(service_id=service_id, options=default_options)
results = await ollama.get_chat_message_contents(chat_history, settings)
# Only one response expected
assert len(results) == 1
msg = results[0]
# Assert it's a ChatMessageContent
assert isinstance(msg, ChatMessageContent)
# The content property should return the response text
assert msg.content == "resp_text"
# The second item should be a FunctionCallContent
func_item = msg.items[1]
assert isinstance(func_item, FunctionCallContent)
# Validate function call details
assert func_item.name == "fn"
assert func_item.arguments == {"x": 1}
# Check metadata
assert "model" in msg.metadata and msg.metadata["model"] == "mdl"
# Access usage directly, key should exist
usage = msg.metadata["usage"]
assert isinstance(usage, CompletionUsage)
assert usage.prompt_tokens == 2 and usage.completion_tokens == 3
@patch("ollama.AsyncClient.chat", new_callable=AsyncMock)
async def test_streaming_chat_response_branch(
mock_chat: AsyncMock,
model_id: str,
service_id: str,
default_options: dict,
chat_history,
monkeypatch,
) -> None:
"""
Test get_streaming_chat_message_contents when AsyncClient.chat yields ChatResponse instances.
"""
class DummyFunction:
def __init__(self, name, arguments):
self.name = name
self.arguments = arguments
class DummyToolCall:
def __init__(self, function):
self.function = function
class DummyMessage:
def __init__(self, content: str, tool_calls=None) -> None:
self.content = content
self.tool_calls = tool_calls or []
class DummyChatResponse:
def __init__(
self,
content: str,
model: str,
prompt_eval_count: int,
eval_count: int,
tool_calls=None,
) -> None:
function_calls = [
DummyToolCall(DummyFunction(tc["function"]["name"], tc["function"]["arguments"])) for tc in tool_calls
]
self.message = DummyMessage(content, function_calls)
self.model = model
self.prompt_eval_count = prompt_eval_count
self.eval_count = eval_count
# Monkeypatch ChatResponse type
monkeypatch.setattr(occ_module, "ChatResponse", DummyChatResponse)
# Prepare an async generator yielding DummyChatResponse
async def fake_stream() -> AsyncGenerator[DummyChatResponse, None]:
yield DummyChatResponse(
content="stream_text",
model="m2",
prompt_eval_count=1,
eval_count=1,
tool_calls=[{"function": {"name": "f2", "arguments": {}}}],
)
mock_chat.return_value = fake_stream()
ollama = OllamaChatCompletion(ai_model_id=model_id)
settings = OllamaChatPromptExecutionSettings(service_id=service_id, options=default_options)
collected = []
# Iterate over streamed batches
async for batch in ollama.get_streaming_chat_message_contents(chat_history, settings):
# We expect a list with a single StreamingChatMessageContent
assert len(batch) == 1
sc = batch[0]
assert isinstance(sc, StreamingChatMessageContent)
# First item should be text content
text_item = sc.items[0]
assert isinstance(text_item, StreamingTextContent)
assert text_item.text == "stream_text"
# Next item should be a FunctionCallContent
func_item = sc.items[1]
assert isinstance(func_item, FunctionCallContent)
assert func_item.name == "f2"
collected.append(sc)
# Only one batch should be collected
assert len(collected) == 1
@@ -0,0 +1,178 @@
# Copyright (c) Microsoft. All rights reserved.
from collections.abc import AsyncGenerator
from unittest.mock import MagicMock, patch
import pytest
from semantic_kernel.connectors.ai.ollama.ollama_prompt_execution_settings import OllamaTextPromptExecutionSettings
from semantic_kernel.connectors.ai.ollama.services.ollama_text_completion import OllamaTextCompletion
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError, ServiceInvalidResponseError
def test_settings(model_id):
"""Test that the settings class is correct"""
ollama = OllamaTextCompletion(ai_model_id=model_id)
settings = ollama.get_prompt_execution_settings_class()
assert settings == OllamaTextPromptExecutionSettings
def test_init_empty_service_id(model_id):
"""Test that the service initializes correctly with an empty service_id"""
ollama = OllamaTextCompletion(ai_model_id=model_id)
assert ollama.service_id == model_id
def test_custom_client(model_id, custom_client):
"""Test that the service initializes correctly with a custom client."""
ollama = OllamaTextCompletion(ai_model_id=model_id, client=custom_client)
assert ollama.client == custom_client
def test_invalid_ollama_settings():
"""Test that the service initializes incorrectly with invalid settings."""
with pytest.raises(ServiceInitializationError):
_ = OllamaTextCompletion(ai_model_id=123)
def test_service_url(ollama_unit_test_env):
"""Test that the service URL is correct."""
ollama = OllamaTextCompletion()
assert ollama.service_url() == ollama_unit_test_env["OLLAMA_HOST"]
@pytest.mark.parametrize("exclude_list", [["OLLAMA_TEXT_MODEL_ID"]], indirect=True)
def test_init_empty_model_id(ollama_unit_test_env):
"""Test that the service initializes incorrectly with an empty model_id"""
with pytest.raises(ServiceInitializationError):
_ = OllamaTextCompletion(env_file_path="fake_env_file_path.env")
@patch("ollama.AsyncClient.__init__", return_value=None) # mock_client
@patch("ollama.AsyncClient.generate") # mock_completion_client
async def test_custom_host(
mock_completion_client, mock_client, model_id, service_id, host, chat_history, default_options
):
"""Test that the service initializes and generates content correctly with a custom host."""
mock_completion_client.return_value = {"response": "test_response"}
ollama = OllamaTextCompletion(ai_model_id=model_id, host=host)
_ = await ollama.get_text_contents(
chat_history,
OllamaTextPromptExecutionSettings(service_id=service_id, options=default_options),
)
mock_client.assert_called_once_with(host=host)
@patch("ollama.AsyncClient.__init__", return_value=None) # mock_client
@patch("ollama.AsyncClient.generate") # mock_completion_client
async def test_custom_host_streaming(
mock_completion_client, mock_client, model_id, service_id, host, chat_history, default_options
):
"""Test that the service initializes and generates streaming content correctly with a custom host."""
# Create a proper async iterator mock for streaming
streaming_response = MagicMock(spec=AsyncGenerator)
streaming_response.__aiter__.return_value = [{"response": "test_response"}]
mock_completion_client.return_value = streaming_response
ollama = OllamaTextCompletion(ai_model_id=model_id, host=host)
async for _ in ollama.get_streaming_text_contents(
chat_history,
OllamaTextPromptExecutionSettings(service_id=service_id, options=default_options),
):
pass
mock_client.assert_called_once_with(host=host)
@patch("ollama.AsyncClient.generate")
async def test_completion(mock_completion_client, model_id, service_id, prompt, default_options):
"""Test that the service generates content correctly."""
mock_completion_client.return_value = {"response": "test_response"}
ollama = OllamaTextCompletion(ai_model_id=model_id)
response = await ollama.get_text_contents(
prompt=prompt,
settings=OllamaTextPromptExecutionSettings(service_id=service_id, options=default_options),
)
assert response[0].text == "test_response"
mock_completion_client.assert_called_once_with(
model=model_id,
prompt=prompt,
options=default_options,
stream=False,
)
@patch("ollama.AsyncClient.generate")
async def test_completion_wrong_return_type(
mock_completion_client,
mock_streaming_text_response,
model_id,
service_id,
chat_history,
default_options,
):
"""Test that the completion service fails when the return type is incorrect."""
mock_completion_client.return_value = mock_streaming_text_response # should not be a streaming response
ollama = OllamaTextCompletion(ai_model_id=model_id)
with pytest.raises(ServiceInvalidResponseError):
await ollama.get_text_contents(
chat_history,
OllamaTextPromptExecutionSettings(service_id=service_id, options=default_options),
)
@patch("ollama.AsyncClient.generate")
async def test_streaming_completion(
mock_completion_client,
mock_streaming_text_response,
model_id,
service_id,
prompt,
default_options,
):
"""Test that the service generates streaming content correctly."""
mock_completion_client.return_value = mock_streaming_text_response
ollama = OllamaTextCompletion(ai_model_id=model_id)
response = ollama.get_streaming_text_contents(
prompt,
OllamaTextPromptExecutionSettings(service_id=service_id, options=default_options),
)
responses = []
async for line in response:
assert line[0].text == "test_response"
responses.append(line)
assert len(responses) == 1
mock_completion_client.assert_called_once_with(
model=model_id,
prompt=prompt,
options=default_options,
stream=True,
)
@patch("ollama.AsyncClient.generate")
async def test_streaming_completion_wrong_return_type(
mock_completion_client,
model_id,
service_id,
chat_history,
default_options,
):
"""Test that the streaming completion service fails when the return type is incorrect."""
mock_completion_client.return_value = {"response": "test_response"} # should not be a non-streaming response
ollama = OllamaTextCompletion(ai_model_id=model_id)
with pytest.raises(ServiceInvalidResponseError):
async for _ in ollama.get_streaming_text_contents(
chat_history,
OllamaTextPromptExecutionSettings(service_id=service_id, options=default_options),
):
pass
@@ -0,0 +1,122 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import patch
import numpy
import pytest
from numpy import array
from semantic_kernel.connectors.ai.ollama.ollama_prompt_execution_settings import OllamaEmbeddingPromptExecutionSettings
from semantic_kernel.connectors.ai.ollama.services.ollama_text_embedding import OllamaTextEmbedding
from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError
def test_init_empty_service_id(model_id):
"""Test that the service initializes correctly with an empty service id."""
ollama = OllamaTextEmbedding(ai_model_id=model_id)
assert ollama.service_id == model_id
def test_custom_client(model_id, custom_client):
"""Test that the service initializes correctly with a custom client."""
ollama = OllamaTextEmbedding(ai_model_id=model_id, client=custom_client)
assert ollama.client == custom_client
def test_invalid_ollama_settings():
"""Test that the service initializes incorrectly with invalid settings."""
with pytest.raises(ServiceInitializationError):
_ = OllamaTextEmbedding(ai_model_id=123)
@pytest.mark.parametrize("exclude_list", [["OLLAMA_EMBEDDING_MODEL_ID"]], indirect=True)
def test_init_empty_model_id(ollama_unit_test_env):
"""Test that the service initializes incorrectly with an empty model id."""
with pytest.raises(ServiceInitializationError):
_ = OllamaTextEmbedding(env_file_path="fake_env_file_path.env")
@patch("ollama.AsyncClient.__init__", return_value=None) # mock_client
@patch("ollama.AsyncClient.embeddings") # mock_embedding_client
async def test_custom_host(mock_embedding_client, mock_client, model_id, host, prompt):
"""Test that the service initializes and generates embeddings correctly with a custom host."""
mock_embedding_client.return_value = {"embedding": [0.1, 0.2, 0.3]}
ollama = OllamaTextEmbedding(ai_model_id=model_id, host=host)
_ = await ollama.generate_embeddings(
[prompt],
)
mock_client.assert_called_once_with(host=host)
@patch("ollama.AsyncClient.embeddings")
async def test_embedding(mock_embedding_client, model_id, prompt):
"""Test that the service initializes and generates embeddings correctly."""
mock_embedding_client.return_value = {"embedding": [0.1, 0.2, 0.3]}
settings = OllamaEmbeddingPromptExecutionSettings()
settings.options = {"test_key": "test_value"}
ollama = OllamaTextEmbedding(ai_model_id=model_id)
response = await ollama.generate_embeddings(
[prompt],
settings=settings,
)
assert response.all() == array([0.1, 0.2, 0.3]).all()
mock_embedding_client.assert_called_once_with(model=model_id, prompt=prompt, options=settings.options)
@patch("ollama.AsyncClient.embeddings")
async def test_embedding_list_input(mock_embedding_client, model_id, prompt):
"""Test that the service initializes and generates embeddings correctly with a list of prompts."""
mock_embedding_client.return_value = {"embedding": [0.1, 0.2, 0.3]}
settings = OllamaEmbeddingPromptExecutionSettings()
settings.options = {"test_key": "test_value"}
ollama = OllamaTextEmbedding(ai_model_id=model_id)
responses = await ollama.generate_embeddings(
[prompt, prompt],
settings=settings,
)
assert len(responses) == 2
assert type(responses) is numpy.ndarray
assert all(type(response) is numpy.ndarray for response in responses)
assert mock_embedding_client.call_count == 2
mock_embedding_client.assert_called_with(model=model_id, prompt=prompt, options=settings.options)
@patch("ollama.AsyncClient.embeddings")
async def test_raw_embedding(mock_embedding_client, model_id, prompt):
"""Test that the service initializes and generates embeddings correctly."""
mock_embedding_client.return_value = {"embedding": [0.1, 0.2, 0.3]}
settings = OllamaEmbeddingPromptExecutionSettings()
settings.options = {"test_key": "test_value"}
ollama = OllamaTextEmbedding(ai_model_id=model_id)
response = await ollama.generate_raw_embeddings(
[prompt],
settings=settings,
)
assert response == [[0.1, 0.2, 0.3]]
mock_embedding_client.assert_called_once_with(model=model_id, prompt=prompt, options=settings.options)
@patch("ollama.AsyncClient.embeddings")
async def test_raw_embedding_list_input(mock_embedding_client, model_id, prompt):
"""Test that the service initializes and generates embeddings correctly with a list of prompts."""
mock_embedding_client.return_value = {"embedding": [0.1, 0.2, 0.3]}
settings = OllamaEmbeddingPromptExecutionSettings()
settings.options = {"test_key": "test_value"}
ollama = OllamaTextEmbedding(ai_model_id=model_id)
responses = await ollama.generate_raw_embeddings(
[prompt, prompt],
settings=settings,
)
assert responses == [[0.1, 0.2, 0.3], [0.1, 0.2, 0.3]]
assert mock_embedding_client.call_count == 2
mock_embedding_client.assert_called_with(model=model_id, prompt=prompt, options=settings.options)
@@ -0,0 +1,240 @@
# Copyright (c) Microsoft. All rights reserved.
from unittest.mock import MagicMock, patch
import pytest
from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration
from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType
# The code under test
from semantic_kernel.connectors.ai.ollama.services.utils import (
MESSAGE_CONVERTERS,
update_settings_from_function_choice_configuration,
)
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
from semantic_kernel.contents.chat_message_content import ChatMessageContent
from semantic_kernel.contents.function_call_content import FunctionCallContent
from semantic_kernel.contents.function_result_content import FunctionResultContent
from semantic_kernel.contents.image_content import ImageContent
from semantic_kernel.contents.utils.author_role import AuthorRole
@pytest.fixture
def mock_chat_message_content() -> ChatMessageContent:
"""Fixture to create a basic ChatMessageContent object with role=USER and simple text content."""
return ChatMessageContent(
role=AuthorRole.USER,
content="Hello, I am a user message.", # The text content
)
@pytest.fixture
def mock_system_message_content() -> ChatMessageContent:
"""Fixture to create a ChatMessageContent object with role=SYSTEM."""
return ChatMessageContent(role=AuthorRole.SYSTEM, content="This is a system message.")
@pytest.fixture
def mock_assistant_message_content() -> ChatMessageContent:
"""Fixture to create a ChatMessageContent object with role=ASSISTANT."""
return ChatMessageContent(role=AuthorRole.ASSISTANT, content="This is an assistant message.")
@pytest.fixture
def mock_tool_message_content() -> ChatMessageContent:
"""Fixture to create a ChatMessageContent object with role=TOOL."""
return ChatMessageContent(role=AuthorRole.TOOL, content="This is a tool message.")
def test_message_converters_system(mock_system_message_content: ChatMessageContent) -> None:
"""Test that passing a system message returns the correct dictionary structure for 'system' role."""
# Act
converter = MESSAGE_CONVERTERS[AuthorRole.SYSTEM]
result = converter(mock_system_message_content)
# Assert
assert result["role"] == "system", "Expected role to be 'system' on the returned message."
assert result["content"] == mock_system_message_content.content, (
"Expected content to match the system message content."
)
def test_message_converters_user_no_images(mock_chat_message_content: ChatMessageContent) -> None:
"""Test that passing a user message without images returns correct dictionary structure for 'user' role."""
# Act
converter = MESSAGE_CONVERTERS[AuthorRole.USER]
result = converter(mock_chat_message_content)
# Assert
assert result["role"] == "user", "Expected role to be 'user' on the returned message."
assert result["content"] == mock_chat_message_content.content, "Expected content to match the user message content."
# Ensure that no 'images' field is added
assert "images" not in result, "No images should be present if no ImageContent is added."
def test_message_converters_user_with_images() -> None:
"""Test user message with multiple images, verifying the 'images' field is populated."""
# Arrange
img1 = ImageContent(data="some_base64_data")
img2 = ImageContent(data="other_base64_data")
content = ChatMessageContent(role=AuthorRole.USER, items=[img1, img2], content="User with images")
# Act
converter = MESSAGE_CONVERTERS[AuthorRole.USER]
result = converter(content)
# Assert
assert result["role"] == "user"
assert result["content"] == content.content
assert "images" in result, "Images field expected when ImageContent is present."
assert len(result["images"]) == 2, "Two images should be in the 'images' field."
assert result["images"] == [b"some_base64_data", b"other_base64_data"], (
"Image data should match the content from ImageContent."
)
def test_message_converters_user_with_image_missing_data() -> None:
"""Test user message with image content that has missing data, expecting ValueError."""
# Arrange
bad_image = ImageContent(data="") # empty data for image
content = ChatMessageContent(role=AuthorRole.USER, items=[bad_image])
# Act & Assert
converter = MESSAGE_CONVERTERS[AuthorRole.USER]
with pytest.raises(ValueError) as exc_info:
converter(content)
assert "Image item must contain data encoded as base64." in str(exc_info.value), (
"Should raise ValueError for missing base64 data in image."
)
def test_message_converters_assistant_basic(mock_assistant_message_content: ChatMessageContent) -> None:
"""Test assistant message without images or tool calls."""
# Act
converter = MESSAGE_CONVERTERS[AuthorRole.ASSISTANT]
result = converter(mock_assistant_message_content)
# Assert
assert result["role"] == "assistant", "Assistant role expected."
assert result["content"] == mock_assistant_message_content.content
assert "images" not in result, "No images included, so should not have an 'images' field."
assert "tool_calls" not in result, "No FunctionCallContent, so 'tool_calls' field shouldn't be present."
def test_message_converters_assistant_with_image() -> None:
"""Test assistant message containing images. Verify 'images' field is added."""
# Arrange
img = ImageContent(data="assistant_base64_data")
content = ChatMessageContent(role=AuthorRole.ASSISTANT, items=[img], content="Assistant image message")
# Act
converter = MESSAGE_CONVERTERS[AuthorRole.ASSISTANT]
result = converter(content)
# Assert
assert result["role"] == "assistant"
assert result["content"] == content.content
assert "images" in result, "Images should be included for assistant messages with ImageContent."
assert result["images"] == [b"assistant_base64_data"], "Expected matching base64 data in images."
def test_message_converters_assistant_with_tool_calls() -> None:
"""Test assistant message with FunctionCallContent should populate 'tool_calls'."""
# Arrange
tool_call_1 = FunctionCallContent(function_name="foo", arguments='{"key": "value"}')
tool_call_2 = FunctionCallContent(function_name="bar", arguments='{"another": "123"}')
content = ChatMessageContent(
role=AuthorRole.ASSISTANT, items=[tool_call_1, tool_call_2], content="Assistant with tools"
)
# Act
converter = MESSAGE_CONVERTERS[AuthorRole.ASSISTANT]
result = converter(content)
# Assert
assert result["role"] == "assistant"
assert result["content"] == content.content
assert "tool_calls" in result, "tool_calls field should be present for assistant messages with FunctionCallContent."
assert len(result["tool_calls"]) == 2, "Expected two tool calls in the result."
assert result["tool_calls"][0]["function"]["name"] == "foo", "First tool call function name mismatched."
assert result["tool_calls"][0]["function"]["arguments"] == {"key": "value"}, "Expected arguments to be JSON loaded."
assert result["tool_calls"][1]["function"]["name"] == "bar", "Second tool call function name mismatched."
assert result["tool_calls"][1]["function"]["arguments"] == {"another": "123"}, (
"Expected arguments to be JSON loaded."
)
def test_message_converters_tool_with_result() -> None:
"""Test tool message with a FunctionResultContent, verifying the message content is set."""
# Arrange
fr_content = FunctionResultContent(id="some_id", result="some result", function_name="test_func")
tool_message = ChatMessageContent(role=AuthorRole.TOOL, items=[fr_content])
# Act
converter = MESSAGE_CONVERTERS[AuthorRole.TOOL]
result = converter(tool_message)
# Assert
assert result["role"] == "tool", "Expected role to be 'tool' for a tool message."
# The code takes the first FunctionResultContent's result as the content
assert result["content"] == fr_content.result, "Expected content to match the function result."
def test_message_converters_tool_missing_function_result_content(mock_tool_message_content: ChatMessageContent) -> None:
"""Test that if no FunctionResultContent is present, ValueError is raised."""
# Arrange
mock_tool_message_content.items = [] # no FunctionResultContent in items
converter = MESSAGE_CONVERTERS[AuthorRole.TOOL]
# Act & Assert
with pytest.raises(ValueError) as exc_info:
converter(mock_tool_message_content)
assert "Tool message must have a function result content item." in str(exc_info.value)
@pytest.mark.parametrize("choice_type", [FunctionChoiceType.AUTO, FunctionChoiceType.NONE, FunctionChoiceType.REQUIRED])
def test_update_settings_from_function_choice_configuration(choice_type: FunctionChoiceType) -> None:
"""Test that update_settings_from_function_choice_configuration updates the settings with the correct tools."""
# Arrange
# We'll create a mock configuration with some available functions.
mock_config = FunctionCallChoiceConfiguration()
mock_config.available_functions = [MagicMock() for _ in range(2)]
# We also patch the kernel_function_metadata_to_function_call_format function.
# The function returns a dict object describing each function.
mock_tool_description = {"type": "function", "function": {"name": "mocked_function"}}
with patch(
"semantic_kernel.connectors.ai.ollama.services.utils.kernel_function_metadata_to_function_call_format",
return_value=mock_tool_description,
):
settings = PromptExecutionSettings()
# Act
update_settings_from_function_choice_configuration(
function_choice_configuration=mock_config,
settings=settings,
type=choice_type,
)
# Assert
# After the call, either settings.tools or settings.extension_data["tools"] should be set.
# The code tries settings.tools first and if it fails, it sets extension_data["tools"].
# We'll check both possibilities.
possible_tools = getattr(settings, "tools", None)
if possible_tools is not None:
# If settings.tools exists, ensure it got updated
assert len(possible_tools) == 2, "Should have exactly two tools set in the settings.tools attribute."
assert possible_tools[0]["function"]["name"] == "mocked_function", (
"Expected mocked function name in settings.tools."
)
else:
# Otherwise check for extension_data
assert "tools" in settings.extension_data, "Expected 'tools' in extension_data if settings.tools not present."
assert len(settings.extension_data["tools"]) == 2, "Should have exactly two tools in extension_data."
assert settings.extension_data["tools"][0]["function"]["name"] == "mocked_function", (
"Expected mocked function name in extension_data."
)
@@ -0,0 +1,95 @@
# Copyright (c) Microsoft. All rights reserved.
from semantic_kernel.connectors.ai.ollama import OllamaPromptExecutionSettings
from semantic_kernel.connectors.ai.ollama.ollama_prompt_execution_settings import (
OllamaChatPromptExecutionSettings,
OllamaTextPromptExecutionSettings,
)
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
def test_default_ollama_prompt_execution_settings():
settings = OllamaPromptExecutionSettings()
assert settings.format is None
assert settings.options is None
def test_custom_ollama_prompt_execution_settings():
settings = OllamaPromptExecutionSettings(
format="json",
options={
"key": "value",
},
)
assert settings.format == "json"
assert settings.options == {"key": "value"}
def test_ollama_prompt_execution_settings_from_default_completion_config():
settings = PromptExecutionSettings(service_id="test_service")
chat_settings = OllamaChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.service_id == "test_service"
assert chat_settings.format is None
assert chat_settings.options is None
def test_ollama_prompt_execution_settings_from_openai_prompt_execution_settings():
chat_settings = OllamaChatPromptExecutionSettings(service_id="test_service", options={"temperature": 0.5})
new_settings = OllamaPromptExecutionSettings(service_id="test_2", options={"temperature": 0.0})
chat_settings.update_from_prompt_execution_settings(new_settings)
assert chat_settings.service_id == "test_2"
assert chat_settings.options["temperature"] == 0.0
def test_ollama_prompt_execution_settings_from_custom_completion_config():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"format": "json",
"options": {
"key": "value",
},
},
)
chat_settings = OllamaChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.service_id == "test_service"
assert chat_settings.format == "json"
assert chat_settings.options == {"key": "value"}
def test_ollama_chat_prompt_execution_settings_from_custom_completion_config_with_functions():
settings = PromptExecutionSettings(
service_id="test_service",
extension_data={
"tools": [{"function": {}}],
},
)
chat_settings = OllamaChatPromptExecutionSettings.from_prompt_execution_settings(settings)
assert chat_settings.tools == [{"function": {}}]
def test_create_options():
settings = OllamaChatPromptExecutionSettings(
service_id="test_service",
extension_data={
"format": "json",
},
)
options = settings.prepare_settings_dict()
assert options["format"] == "json"
def test_default_ollama_text_prompt_execution_settings():
settings = OllamaTextPromptExecutionSettings()
assert settings.system is None
assert settings.template is None
assert settings.context is None
assert settings.raw is None
@@ -0,0 +1,29 @@
# Copyright (c) Microsoft. All rights reserved.
import json
class MockResponse:
def __init__(self, response, status=200):
self._response = response
self.status = status
async def text(self):
return self._response
async def json(self):
return self._response
def raise_for_status(self):
pass
@property
async def content(self):
yield json.dumps(self._response).encode("utf-8")
yield json.dumps({"done": True}).encode("utf-8")
async def __aexit__(self, exc_type, exc, tb):
pass
async def __aenter__(self):
return self
@@ -0,0 +1,32 @@
# Copyright (c) Microsoft. All rights reserved.
from pytest import fixture
@fixture()
def onnx_unit_test_env(monkeypatch, exclude_list, override_env_param_dict):
"""Fixture to set environment variables for OnnxGenAISettings."""
if exclude_list is None:
exclude_list = []
if override_env_param_dict is None:
override_env_param_dict = {}
env_vars = {
"ONNX_GEN_AI_CHAT_MODEL_FOLDER": "test",
"ONNX_GEN_AI_TEXT_MODEL_FOLDER": "test",
}
env_vars.update(override_env_param_dict)
for key, value in env_vars.items():
if key not in exclude_list:
monkeypatch.setenv(key, value)
else:
monkeypatch.delenv(key, raising=False)
return env_vars
gen_ai_config = {"model": {"test": "test"}}
gen_ai_config_vision = {"model": {"vision": "test"}}
@@ -0,0 +1,187 @@
# Copyright (c) Microsoft. All rights reserved.
import json
import os
from unittest.mock import MagicMock, mock_open, patch
import pytest
from semantic_kernel.connectors.ai.onnx import OnnxGenAIChatCompletion, OnnxGenAIPromptExecutionSettings, ONNXTemplate
from semantic_kernel.contents import AuthorRole, ChatHistory, ChatMessageContent, ImageContent
from semantic_kernel.exceptions import ServiceInitializationError, ServiceInvalidExecutionSettingsError
from semantic_kernel.kernel import Kernel
from tests.unit.connectors.ai.onnx.conftest import gen_ai_config, gen_ai_config_vision
@patch("builtins.open", new_callable=mock_open, read_data=json.dumps(gen_ai_config))
@patch("onnxruntime_genai.Model")
@patch("onnxruntime_genai.Tokenizer")
def test_onnx_chat_completion_with_valid_env_variable(gen_ai_config, model, tokenizer, onnx_unit_test_env):
service = OnnxGenAIChatCompletion(template=ONNXTemplate.PHI3, env_file_path="test.env")
assert not service.enable_multi_modality
@patch("builtins.open", new_callable=mock_open, read_data=json.dumps(gen_ai_config_vision))
@patch("onnxruntime_genai.Model")
@patch("onnxruntime_genai.Tokenizer")
def test_onnx_chat_completion_with_vision_valid_env_variable(
gen_ai_vision_config, model, tokenizer, onnx_unit_test_env
):
service = OnnxGenAIChatCompletion(template=ONNXTemplate.PHI3, env_file_path="test.env")
assert service.enable_multi_modality
@patch("builtins.open", new_callable=mock_open, read_data=json.dumps(gen_ai_config))
@patch("onnxruntime_genai.Model")
@patch("onnxruntime_genai.Tokenizer")
def test_onnx_chat_completion_with_valid_parameter(gen_ai_config, model, tokenizer):
assert OnnxGenAIChatCompletion(ai_model_path="/valid_path", template=ONNXTemplate.PHI3)
@patch("builtins.open", new_callable=mock_open, read_data=json.dumps(gen_ai_config))
@patch("onnxruntime_genai.Model")
@patch("onnxruntime_genai.Tokenizer")
def test_onnx_chat_completion_with_str_template(gen_ai_config, model, tokenizer):
assert OnnxGenAIChatCompletion(ai_model_path="/valid_path", template="phi3")
def test_onnx_chat_completion_with_invalid_model():
with pytest.raises(ServiceInitializationError):
OnnxGenAIChatCompletion(
ai_model_path="/invalid_path",
template=ONNXTemplate.PHI3,
)
@patch("builtins.open", new_callable=mock_open, read_data=json.dumps(gen_ai_config_vision))
def test_onnx_chat_completion_with_multimodality_without_prompt_template(gen_ai_config_vision):
with pytest.raises(ServiceInitializationError):
OnnxGenAIChatCompletion()
def test_onnx_chat_completion_with_invalid_env_variable(onnx_unit_test_env):
with pytest.raises(ServiceInitializationError):
OnnxGenAIChatCompletion(
template=ONNXTemplate.PHI3,
)
@pytest.mark.parametrize("exclude_list", [["ONNX_GEN_AI_CHAT_MODEL_FOLDER"]], indirect=True)
def test_onnx_chat_completion_with_missing_ai_path(onnx_unit_test_env):
with pytest.raises(ServiceInitializationError):
OnnxGenAIChatCompletion(template=ONNXTemplate.PHI3, env_file_path="test.env")
@patch("builtins.open", new_callable=mock_open, read_data=json.dumps(gen_ai_config))
@patch("onnxruntime_genai.Model")
@patch("onnxruntime_genai.Tokenizer")
async def test_onnx_chat_completion(gen_ai_config, model, tokenizer):
generator_mock = MagicMock()
generator_mock.__aiter__.return_value = [["H"], ["e"], ["l"], ["l"], ["o"]]
chat_completion = OnnxGenAIChatCompletion(template=ONNXTemplate.PHI3, ai_model_path="test")
history = ChatHistory()
history.add_system_message("test")
history.add_user_message("test")
with patch.object(chat_completion, "_generate_next_token_async", return_value=generator_mock):
completed_text: ChatMessageContent = await chat_completion.get_chat_message_content(
prompt="test", chat_history=history, settings=OnnxGenAIPromptExecutionSettings(), kernel=Kernel()
)
assert str(completed_text) == "Hello"
@patch("builtins.open", new_callable=mock_open, read_data=json.dumps(gen_ai_config))
@patch("onnxruntime_genai.Model")
@patch("onnxruntime_genai.Tokenizer")
async def test_onnx_chat_completion_streaming(gen_ai_config, model, tokenizer):
generator_mock = MagicMock()
generator_mock.__aiter__.return_value = [["H"], ["e"], ["l"], ["l"], ["o"]]
chat_completion = OnnxGenAIChatCompletion(template=ONNXTemplate.PHI3, ai_model_path="test")
history = ChatHistory()
history.add_system_message("test")
history.add_user_message("test")
completed_text: str = ""
with patch.object(chat_completion, "_generate_next_token_async", return_value=generator_mock):
async for chunk in chat_completion.get_streaming_chat_message_content(
prompt="test", chat_history=history, settings=OnnxGenAIPromptExecutionSettings(), kernel=Kernel()
):
completed_text += str(chunk)
assert completed_text == "Hello"
@patch("onnxruntime_genai.Model")
def test_onnx_chat_get_image_history(model):
builtin_open = open # save the unpatched version
def patch_open(*args, **kwargs):
if "genai_config.json" in str(args[0]):
# mocked open for path "genai_config.json"
return mock_open(read_data=json.dumps(gen_ai_config_vision))(*args, **kwargs)
# unpatched version for every other path
return builtin_open(*args, **kwargs)
with patch("builtins.open", patch_open):
chat_completion = OnnxGenAIChatCompletion(
template=ONNXTemplate.PHI3,
ai_model_path="test",
)
image_content = ImageContent.from_image_path(
image_path=os.path.join(os.path.dirname(__file__), "../../../../../", "assets/sample_image.jpg")
)
history = ChatHistory()
history.add_system_message("test")
history.add_user_message("test")
history.add_message(
ChatMessageContent(
role=AuthorRole.USER,
items=[image_content],
),
)
last_image = chat_completion._get_images_from_history(history)
assert last_image == [image_content]
@patch("onnxruntime_genai.Model")
@patch("onnxruntime_genai.Tokenizer")
async def test_onnx_chat_get_image_history_with_not_multimodal(model, tokenizer):
builtin_open = open # save the unpatched version
def patch_open(*args, **kwargs):
if "genai_config.json" in str(args[0]):
# mocked open for path "genai_config.json"
return mock_open(read_data=json.dumps(gen_ai_config))(*args, **kwargs)
# unpatched version for every other path
return builtin_open(*args, **kwargs)
with patch("builtins.open", patch_open):
chat_completion = OnnxGenAIChatCompletion(
template=ONNXTemplate.PHI3,
ai_model_path="test",
)
image_content = ImageContent.from_image_path(
image_path=os.path.join(os.path.dirname(__file__), "../../../../../", "assets/sample_image.jpg")
)
history = ChatHistory()
history.add_system_message("test")
history.add_user_message("test")
history.add_message(
ChatMessageContent(
role=AuthorRole.USER,
items=[image_content],
),
)
with pytest.raises(ServiceInvalidExecutionSettingsError):
_ = await chat_completion._get_images_from_history(history)

Some files were not shown because too many files have changed in this diff Show More