chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,22 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import pytest
|
||||
|
||||
import semantic_kernel
|
||||
from semantic_kernel.utils.telemetry.model_diagnostics.model_diagnostics_settings import ModelDiagnosticSettings
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def model_diagnostics_unit_test_env(monkeypatch):
|
||||
"""Fixture to set environment variables for Model Diagnostics Unit Tests."""
|
||||
env_vars = {
|
||||
"SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS": "true",
|
||||
"SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE": "true",
|
||||
}
|
||||
|
||||
for key, value in env_vars.items():
|
||||
monkeypatch.setenv(key, value)
|
||||
|
||||
# Need to reload the settings to pick up the new environment variables since the
|
||||
# settings are loaded at import time and this fixture is called after the import
|
||||
semantic_kernel.utils.telemetry.agent_diagnostics.decorators.MODEL_DIAGNOSTICS_SETTINGS = ModelDiagnosticSettings()
|
||||
@@ -0,0 +1,43 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import pytest
|
||||
|
||||
from semantic_kernel.agents.chat_completion.chat_completion_agent import ChatCompletionAgent
|
||||
from semantic_kernel.agents.open_ai.openai_assistant_agent import OpenAIAssistantAgent
|
||||
|
||||
pytestmark = pytest.mark.parametrize(
|
||||
"decorated_method, expected_attribute",
|
||||
[
|
||||
# region ChatCompletionAgent
|
||||
pytest.param(
|
||||
ChatCompletionAgent.invoke,
|
||||
"__agent_diagnostics__",
|
||||
id="ChatCompletionAgent.invoke",
|
||||
),
|
||||
pytest.param(
|
||||
ChatCompletionAgent.invoke_stream,
|
||||
"__agent_diagnostics__",
|
||||
id="ChatCompletionAgent.invoke_stream",
|
||||
),
|
||||
# endregion
|
||||
# region OpenAIAssistantAgent
|
||||
pytest.param(
|
||||
OpenAIAssistantAgent.invoke,
|
||||
"__agent_diagnostics__",
|
||||
id="OpenAIAssistantBase.invoke",
|
||||
),
|
||||
pytest.param(
|
||||
OpenAIAssistantAgent.invoke_stream,
|
||||
"__agent_diagnostics__",
|
||||
id="OpenAIAssistantBase.invoke_stream",
|
||||
),
|
||||
# endregion
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_decorated(decorated_method, expected_attribute):
|
||||
"""Test that the connectors are being decorated properly with the agent diagnostics decorators."""
|
||||
assert hasattr(decorated_method, expected_attribute) and getattr(decorated_method, expected_attribute), (
|
||||
f"{decorated_method} should be decorated with the appropriate agent diagnostics decorator."
|
||||
)
|
||||
@@ -0,0 +1,65 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
from semantic_kernel.agents.chat_completion.chat_completion_agent import ChatCompletionAgent, ChatHistoryAgentThread
|
||||
from semantic_kernel.exceptions.kernel_exceptions import KernelServiceNotFoundError
|
||||
|
||||
|
||||
@patch("semantic_kernel.utils.telemetry.agent_diagnostics.decorators.tracer")
|
||||
async def test_chat_completion_agent_get_response(
|
||||
mock_tracer,
|
||||
chat_history,
|
||||
model_diagnostics_unit_test_env,
|
||||
):
|
||||
# Arrange
|
||||
chat_completion_agent = ChatCompletionAgent()
|
||||
thread = ChatHistoryAgentThread(chat_history=chat_history)
|
||||
# Act
|
||||
with pytest.raises(KernelServiceNotFoundError):
|
||||
await chat_completion_agent.get_response(messages="test", thread=thread)
|
||||
# Assert
|
||||
mock_tracer.start_as_current_span.assert_called_once()
|
||||
args, _ = mock_tracer.start_as_current_span.call_args
|
||||
assert args[0] == f"invoke_agent {chat_completion_agent.name}"
|
||||
|
||||
|
||||
@patch("semantic_kernel.utils.telemetry.agent_diagnostics.decorators.tracer")
|
||||
async def test_chat_completion_agent_invoke(
|
||||
mock_tracer,
|
||||
chat_history,
|
||||
model_diagnostics_unit_test_env,
|
||||
):
|
||||
# Arrange
|
||||
chat_completion_agent = ChatCompletionAgent()
|
||||
thread = ChatHistoryAgentThread(chat_history=chat_history)
|
||||
# Act
|
||||
with pytest.raises(KernelServiceNotFoundError):
|
||||
async for _ in chat_completion_agent.invoke(messages="test", thread=thread):
|
||||
pass
|
||||
# Assert
|
||||
mock_tracer.start_as_current_span.assert_called_once()
|
||||
args, _ = mock_tracer.start_as_current_span.call_args
|
||||
assert args[0] == f"invoke_agent {chat_completion_agent.name}"
|
||||
|
||||
|
||||
@patch("semantic_kernel.utils.telemetry.agent_diagnostics.decorators.tracer")
|
||||
async def test_chat_completion_agent_invoke_stream(
|
||||
mock_tracer,
|
||||
chat_history,
|
||||
model_diagnostics_unit_test_env,
|
||||
):
|
||||
# Arrange
|
||||
chat_completion_agent = ChatCompletionAgent()
|
||||
thread = ChatHistoryAgentThread(chat_history=chat_history)
|
||||
# Act
|
||||
with pytest.raises(KernelServiceNotFoundError):
|
||||
async for _ in chat_completion_agent.invoke_stream(messages="test", thread=thread):
|
||||
pass
|
||||
# Assert
|
||||
mock_tracer.start_as_current_span.assert_called_once()
|
||||
args, _ = mock_tracer.start_as_current_span.call_args
|
||||
assert args[0] == f"invoke_agent {chat_completion_agent.name}"
|
||||
@@ -0,0 +1,128 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from openai import AsyncOpenAI
|
||||
from openai.types.beta.assistant import Assistant
|
||||
|
||||
from semantic_kernel.agents.open_ai.openai_assistant_agent import AssistantAgentThread, OpenAIAssistantAgent
|
||||
from semantic_kernel.contents.chat_message_content import ChatMessageContent
|
||||
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
|
||||
from semantic_kernel.contents.utils.author_role import AuthorRole
|
||||
|
||||
|
||||
@patch("semantic_kernel.utils.telemetry.agent_diagnostics.decorators.tracer")
|
||||
async def test_open_ai_assistant_agent_get_response(
|
||||
mock_tracer,
|
||||
chat_history,
|
||||
openai_unit_test_env,
|
||||
model_diagnostics_unit_test_env,
|
||||
):
|
||||
# Arrange
|
||||
client = AsyncMock(spec=AsyncOpenAI)
|
||||
definition = AsyncMock(spec=Assistant)
|
||||
definition.name = "agentName"
|
||||
definition.description = "agentDescription"
|
||||
definition.id = "agentId"
|
||||
definition.instructions = "agentInstructions"
|
||||
definition.tools = []
|
||||
definition.model = "agentModel"
|
||||
definition.temperature = 1.0
|
||||
definition.top_p = 1.0
|
||||
definition.metadata = {}
|
||||
openai_assistant_agent = OpenAIAssistantAgent(client=client, definition=definition)
|
||||
|
||||
thread = AsyncMock(spec=AssistantAgentThread)
|
||||
|
||||
async def fake_invoke(*args, **kwargs):
|
||||
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
|
||||
|
||||
# Act
|
||||
with patch(
|
||||
"semantic_kernel.agents.open_ai.assistant_thread_actions.AssistantThreadActions.invoke",
|
||||
side_effect=fake_invoke,
|
||||
):
|
||||
await openai_assistant_agent.get_response(messages="message", thread=thread)
|
||||
# Assert
|
||||
mock_tracer.start_as_current_span.assert_called_once()
|
||||
args, _ = mock_tracer.start_as_current_span.call_args
|
||||
assert args[0] == f"invoke_agent {openai_assistant_agent.name}"
|
||||
|
||||
|
||||
@patch("semantic_kernel.utils.telemetry.agent_diagnostics.decorators.tracer")
|
||||
async def test_open_ai_assistant_agent_invoke(
|
||||
mock_tracer,
|
||||
chat_history,
|
||||
openai_unit_test_env,
|
||||
model_diagnostics_unit_test_env,
|
||||
):
|
||||
# Arrange
|
||||
client = AsyncMock(spec=AsyncOpenAI)
|
||||
definition = AsyncMock(spec=Assistant)
|
||||
definition.name = "agentName"
|
||||
definition.description = "agentDescription"
|
||||
definition.id = "agentId"
|
||||
definition.instructions = "agentInstructions"
|
||||
definition.tools = []
|
||||
definition.model = "agentModel"
|
||||
definition.temperature = 1.0
|
||||
definition.top_p = 1.0
|
||||
definition.metadata = {}
|
||||
openai_assistant_agent = OpenAIAssistantAgent(client=client, definition=definition)
|
||||
|
||||
thread = AsyncMock(spec=AssistantAgentThread)
|
||||
|
||||
async def fake_invoke(*args, **kwargs):
|
||||
yield True, ChatMessageContent(role=AuthorRole.ASSISTANT, content="content")
|
||||
|
||||
# Act
|
||||
with patch(
|
||||
"semantic_kernel.agents.open_ai.assistant_thread_actions.AssistantThreadActions.invoke",
|
||||
side_effect=fake_invoke,
|
||||
):
|
||||
async for item in openai_assistant_agent.invoke(messages="message", thread=thread):
|
||||
pass
|
||||
# Assert
|
||||
mock_tracer.start_as_current_span.assert_called_once()
|
||||
args, _ = mock_tracer.start_as_current_span.call_args
|
||||
assert args[0] == f"invoke_agent {openai_assistant_agent.name}"
|
||||
|
||||
|
||||
@patch("semantic_kernel.utils.telemetry.agent_diagnostics.decorators.tracer")
|
||||
async def test_open_ai_assistant_agent_invoke_stream(
|
||||
mock_tracer,
|
||||
chat_history,
|
||||
openai_unit_test_env,
|
||||
model_diagnostics_unit_test_env,
|
||||
):
|
||||
# Arrange
|
||||
client = AsyncMock(spec=AsyncOpenAI)
|
||||
definition = AsyncMock(spec=Assistant)
|
||||
definition.name = "agentName"
|
||||
definition.description = "agentDescription"
|
||||
definition.id = "agentId"
|
||||
definition.instructions = "agentInstructions"
|
||||
definition.tools = []
|
||||
definition.model = "agentModel"
|
||||
definition.temperature = 1.0
|
||||
definition.top_p = 1.0
|
||||
definition.metadata = {}
|
||||
openai_assistant_agent = OpenAIAssistantAgent(client=client, definition=definition)
|
||||
|
||||
thread = AsyncMock(spec=AssistantAgentThread)
|
||||
|
||||
async def fake_invoke(*args, **kwargs):
|
||||
yield StreamingChatMessageContent(role=AuthorRole.ASSISTANT, choice_index=0, content="content")
|
||||
|
||||
# Act
|
||||
with patch(
|
||||
"semantic_kernel.agents.open_ai.assistant_thread_actions.AssistantThreadActions.invoke_stream",
|
||||
side_effect=fake_invoke,
|
||||
):
|
||||
async for item in openai_assistant_agent.invoke_stream(messages="message", thread=thread):
|
||||
pass
|
||||
# Assert
|
||||
mock_tracer.start_as_current_span.assert_called_once()
|
||||
args, _ = mock_tracer.start_as_current_span.call_args
|
||||
assert args[0] == f"invoke_agent {openai_assistant_agent.name}"
|
||||
@@ -0,0 +1,96 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
|
||||
import sys
|
||||
from collections.abc import AsyncGenerator
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import pytest
|
||||
|
||||
if sys.version_info >= (3, 12):
|
||||
from typing import override # pragma: no cover
|
||||
else:
|
||||
from typing_extensions import override # pragma: no cover
|
||||
|
||||
import semantic_kernel
|
||||
from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.connectors.ai.text_completion_client_base import TextCompletionClientBase
|
||||
from semantic_kernel.contents.chat_history import ChatHistory
|
||||
from semantic_kernel.contents.chat_message_content import ChatMessageContent
|
||||
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
|
||||
from semantic_kernel.contents.streaming_text_content import StreamingTextContent
|
||||
from semantic_kernel.contents.text_content import TextContent
|
||||
from semantic_kernel.utils.telemetry.model_diagnostics.model_diagnostics_settings import ModelDiagnosticSettings
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def model_diagnostics_unit_test_env(monkeypatch):
|
||||
"""Fixture to set environment variables for Model Diagnostics Unit Tests."""
|
||||
env_vars = {
|
||||
"SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS": "true",
|
||||
"SEMANTICKERNEL_EXPERIMENTAL_GENAI_ENABLE_OTEL_DIAGNOSTICS_SENSITIVE": "true",
|
||||
}
|
||||
|
||||
for key, value in env_vars.items():
|
||||
monkeypatch.setenv(key, value)
|
||||
|
||||
# Need to reload the settings to pick up the new environment variables since the
|
||||
# settings are loaded at import time and this fixture is called after the import
|
||||
semantic_kernel.utils.telemetry.model_diagnostics.decorators.MODEL_DIAGNOSTICS_SETTINGS = ModelDiagnosticSettings()
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def service_env_vars(monkeypatch, request):
|
||||
"""Fixture to set environment variables for AI Service Unit Tests."""
|
||||
for key, value in request.param.items():
|
||||
monkeypatch.setenv(key, value)
|
||||
|
||||
|
||||
class MockChatCompletion(ChatCompletionClientBase):
|
||||
MODEL_PROVIDER_NAME: ClassVar[str] = "mock"
|
||||
|
||||
@override
|
||||
async def _inner_get_chat_message_contents(
|
||||
self,
|
||||
chat_history: "ChatHistory",
|
||||
settings: "PromptExecutionSettings",
|
||||
) -> list["ChatMessageContent"]:
|
||||
return []
|
||||
|
||||
@override
|
||||
async def _inner_get_streaming_chat_message_contents(
|
||||
self,
|
||||
chat_history: "ChatHistory",
|
||||
settings: "PromptExecutionSettings",
|
||||
) -> AsyncGenerator[list["StreamingChatMessageContent"], Any]:
|
||||
yield []
|
||||
|
||||
@override
|
||||
def service_url(self) -> str | None:
|
||||
return "http://mock-service-url"
|
||||
|
||||
|
||||
class MockTextCompletion(TextCompletionClientBase):
|
||||
MODEL_PROVIDER_NAME: ClassVar[str] = "mock"
|
||||
|
||||
@override
|
||||
async def _inner_get_text_contents(
|
||||
self,
|
||||
prompt: str,
|
||||
settings: "PromptExecutionSettings",
|
||||
) -> list["TextContent"]:
|
||||
return []
|
||||
|
||||
@override
|
||||
async def _inner_get_streaming_text_contents(
|
||||
self,
|
||||
prompt: str,
|
||||
settings: "PromptExecutionSettings",
|
||||
) -> AsyncGenerator[list["StreamingTextContent"], Any]:
|
||||
yield []
|
||||
|
||||
@override
|
||||
def service_url(self) -> str | None:
|
||||
# Returning None to test the case where the service URL is not available
|
||||
return None
|
||||
@@ -0,0 +1,137 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import pytest
|
||||
|
||||
from semantic_kernel.connectors.ai.anthropic.services.anthropic_chat_completion import AnthropicChatCompletion
|
||||
from semantic_kernel.connectors.ai.google.google_ai.services.google_ai_chat_completion import GoogleAIChatCompletion
|
||||
from semantic_kernel.connectors.ai.google.google_ai.services.google_ai_text_completion import GoogleAITextCompletion
|
||||
from semantic_kernel.connectors.ai.google.vertex_ai.services.vertex_ai_chat_completion import VertexAIChatCompletion
|
||||
from semantic_kernel.connectors.ai.google.vertex_ai.services.vertex_ai_text_completion import VertexAITextCompletion
|
||||
from semantic_kernel.connectors.ai.mistral_ai.services.mistral_ai_chat_completion import MistralAIChatCompletion
|
||||
from semantic_kernel.connectors.ai.ollama.services.ollama_chat_completion import OllamaChatCompletion
|
||||
from semantic_kernel.connectors.ai.ollama.services.ollama_text_completion import OllamaTextCompletion
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion import OpenAIChatCompletion
|
||||
from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion import OpenAITextCompletion
|
||||
|
||||
pytestmark = pytest.mark.parametrize(
|
||||
"decorated_method, expected_attribute",
|
||||
[
|
||||
# OpenAIChatCompletion
|
||||
pytest.param(
|
||||
OpenAIChatCompletion._inner_get_chat_message_contents,
|
||||
"__model_diagnostics_chat_completion__",
|
||||
id="OpenAIChatCompletion._inner_get_chat_message_contents",
|
||||
),
|
||||
pytest.param(
|
||||
OpenAIChatCompletion._inner_get_streaming_chat_message_contents,
|
||||
"__model_diagnostics_streaming_chat_completion__",
|
||||
id="OpenAIChatCompletion._inner_get_streaming_chat_message_contents",
|
||||
),
|
||||
# OpenAITextCompletion
|
||||
pytest.param(
|
||||
OpenAITextCompletion._inner_get_text_contents,
|
||||
"__model_diagnostics_text_completion__",
|
||||
id="OpenAITextCompletion._inner_get_text_contents",
|
||||
),
|
||||
pytest.param(
|
||||
OpenAITextCompletion._inner_get_streaming_text_contents,
|
||||
"__model_diagnostics_streaming_text_completion__",
|
||||
id="OpenAITextCompletion._inner_get_streaming_text_contents",
|
||||
),
|
||||
# OllamaChatCompletion
|
||||
pytest.param(
|
||||
OllamaChatCompletion._inner_get_chat_message_contents,
|
||||
"__model_diagnostics_chat_completion__",
|
||||
id="OllamaChatCompletion._inner_get_chat_message_contents",
|
||||
),
|
||||
pytest.param(
|
||||
OllamaChatCompletion._inner_get_streaming_chat_message_contents,
|
||||
"__model_diagnostics_streaming_chat_completion__",
|
||||
id="OllamaChatCompletion._inner_get_streaming_chat_message_contents",
|
||||
),
|
||||
# OllamaTextCompletion
|
||||
pytest.param(
|
||||
OllamaTextCompletion._inner_get_text_contents,
|
||||
"__model_diagnostics_text_completion__",
|
||||
id="OllamaTextCompletion._inner_get_text_contents",
|
||||
),
|
||||
pytest.param(
|
||||
OllamaTextCompletion._inner_get_streaming_text_contents,
|
||||
"__model_diagnostics_streaming_text_completion__",
|
||||
id="OllamaTextCompletion._inner_get_streaming_text_contents",
|
||||
),
|
||||
# MistralAIChatCompletion
|
||||
pytest.param(
|
||||
MistralAIChatCompletion._inner_get_chat_message_contents,
|
||||
"__model_diagnostics_chat_completion__",
|
||||
id="MistralAIChatCompletion._inner_get_chat_message_contents",
|
||||
),
|
||||
pytest.param(
|
||||
MistralAIChatCompletion._inner_get_streaming_chat_message_contents,
|
||||
"__model_diagnostics_streaming_chat_completion__",
|
||||
id="MistralAIChatCompletion._inner_get_streaming_chat_message_contents",
|
||||
),
|
||||
# VertexAIChatCompletion
|
||||
pytest.param(
|
||||
VertexAIChatCompletion._inner_get_chat_message_contents,
|
||||
"__model_diagnostics_chat_completion__",
|
||||
id="VertexAIChatCompletion._inner_get_chat_message_contents",
|
||||
),
|
||||
pytest.param(
|
||||
VertexAIChatCompletion._inner_get_streaming_chat_message_contents,
|
||||
"__model_diagnostics_streaming_chat_completion__",
|
||||
id="VertexAIChatCompletion._inner_get_streaming_chat_message_contents",
|
||||
),
|
||||
# VertexAITextCompletion
|
||||
pytest.param(
|
||||
VertexAITextCompletion._inner_get_text_contents,
|
||||
"__model_diagnostics_text_completion__",
|
||||
id="VertexAITextCompletion._inner_get_text_contents",
|
||||
),
|
||||
pytest.param(
|
||||
VertexAITextCompletion._inner_get_streaming_text_contents,
|
||||
"__model_diagnostics_streaming_text_completion__",
|
||||
id="VertexAITextCompletion._inner_get_streaming_text_contents",
|
||||
),
|
||||
# GoogleAIChatCompletion
|
||||
pytest.param(
|
||||
GoogleAIChatCompletion._inner_get_chat_message_contents,
|
||||
"__model_diagnostics_chat_completion__",
|
||||
id="GoogleAIChatCompletion._inner_get_chat_message_contents",
|
||||
),
|
||||
pytest.param(
|
||||
GoogleAIChatCompletion._inner_get_streaming_chat_message_contents,
|
||||
"__model_diagnostics_streaming_chat_completion__",
|
||||
id="GoogleAIChatCompletion._inner_get_streaming_chat_message_contents",
|
||||
),
|
||||
# GoogleAITextCompletion
|
||||
pytest.param(
|
||||
GoogleAITextCompletion._inner_get_text_contents,
|
||||
"__model_diagnostics_text_completion__",
|
||||
id="GoogleAITextCompletion._inner_get_text_contents",
|
||||
),
|
||||
pytest.param(
|
||||
GoogleAITextCompletion._inner_get_streaming_text_contents,
|
||||
"__model_diagnostics_streaming_text_completion__",
|
||||
id="GoogleAITextCompletion._inner_get_streaming_text_contents",
|
||||
),
|
||||
# AnthropicChatCompletion
|
||||
pytest.param(
|
||||
AnthropicChatCompletion._inner_get_chat_message_contents,
|
||||
"__model_diagnostics_chat_completion__",
|
||||
id="AnthropicChatCompletion._inner_get_chat_message_contents",
|
||||
),
|
||||
pytest.param(
|
||||
AnthropicChatCompletion._inner_get_streaming_chat_message_contents,
|
||||
"__model_diagnostics_streaming_chat_completion__",
|
||||
id="AnthropicChatCompletion._inner_get_streaming_chat_message_contents",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_decorated(decorated_method, expected_attribute):
|
||||
"""Test that the connectors are being decorated properly with the model diagnostics decorators."""
|
||||
assert hasattr(decorated_method, expected_attribute) and getattr(decorated_method, expected_attribute), (
|
||||
f"{decorated_method} should be decorated with the appropriate model diagnostics decorator."
|
||||
)
|
||||
@@ -0,0 +1,205 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import json
|
||||
from unittest.mock import call, patch
|
||||
|
||||
import pytest
|
||||
from opentelemetry.trace import StatusCode
|
||||
|
||||
from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.contents.chat_message_content import ChatMessageContent
|
||||
from semantic_kernel.contents.utils.author_role import AuthorRole
|
||||
from semantic_kernel.contents.utils.finish_reason import FinishReason
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceResponseException
|
||||
from semantic_kernel.utils.telemetry.model_diagnostics import gen_ai_attributes
|
||||
from semantic_kernel.utils.telemetry.model_diagnostics.decorators import (
|
||||
CHAT_COMPLETION_OPERATION,
|
||||
ChatHistoryMessageTimestampFilter,
|
||||
trace_chat_completion,
|
||||
)
|
||||
from tests.unit.utils.model_diagnostics.conftest import MockChatCompletion
|
||||
|
||||
pytestmark = pytest.mark.parametrize(
|
||||
"execution_settings, mock_response",
|
||||
[
|
||||
pytest.param(
|
||||
PromptExecutionSettings(
|
||||
extension_data={
|
||||
"max_tokens": 1000,
|
||||
"temperature": 0.5,
|
||||
"top_p": 0.9,
|
||||
}
|
||||
),
|
||||
[
|
||||
ChatMessageContent(
|
||||
role=AuthorRole.ASSISTANT,
|
||||
ai_model_id="ai_model_id",
|
||||
content="Test content",
|
||||
metadata={"id": "test_id"},
|
||||
finish_reason=FinishReason.STOP,
|
||||
)
|
||||
],
|
||||
id="test_execution_settings_with_extension_data",
|
||||
),
|
||||
pytest.param(
|
||||
PromptExecutionSettings(),
|
||||
[
|
||||
ChatMessageContent(
|
||||
role=AuthorRole.ASSISTANT,
|
||||
ai_model_id="ai_model_id",
|
||||
metadata={"id": "test_id"},
|
||||
finish_reason=FinishReason.STOP,
|
||||
)
|
||||
],
|
||||
id="test_execution_settings_no_extension_data",
|
||||
),
|
||||
pytest.param(
|
||||
PromptExecutionSettings(),
|
||||
[
|
||||
ChatMessageContent(
|
||||
role=AuthorRole.ASSISTANT,
|
||||
ai_model_id="ai_model_id",
|
||||
metadata={},
|
||||
finish_reason=FinishReason.STOP,
|
||||
)
|
||||
],
|
||||
id="test_chat_message_content_no_metadata",
|
||||
),
|
||||
pytest.param(
|
||||
PromptExecutionSettings(),
|
||||
[
|
||||
ChatMessageContent(
|
||||
role=AuthorRole.ASSISTANT,
|
||||
ai_model_id="ai_model_id",
|
||||
metadata={"id": "test_id"},
|
||||
)
|
||||
],
|
||||
id="test_chat_message_content_no_finish_reason",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.logger")
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.tracer")
|
||||
async def test_trace_chat_completion(
|
||||
mock_tracer,
|
||||
mock_logger,
|
||||
execution_settings,
|
||||
mock_response,
|
||||
chat_history,
|
||||
model_diagnostics_unit_test_env,
|
||||
):
|
||||
mock_span = mock_tracer.start_span.return_value
|
||||
# Setup
|
||||
chat_completion: ChatCompletionClientBase = MockChatCompletion(ai_model_id="ai_model_id")
|
||||
|
||||
with patch.object(MockChatCompletion, "_inner_get_chat_message_contents", return_value=mock_response):
|
||||
# We need to reapply the decorator to the method since the mock will not have the decorator applied
|
||||
MockChatCompletion._inner_get_chat_message_contents = trace_chat_completion(
|
||||
MockChatCompletion.MODEL_PROVIDER_NAME
|
||||
)(chat_completion._inner_get_chat_message_contents)
|
||||
|
||||
results: list[ChatMessageContent] = await chat_completion.get_chat_message_contents(
|
||||
chat_history, execution_settings
|
||||
)
|
||||
|
||||
assert results == mock_response
|
||||
|
||||
# Before the call to the model
|
||||
mock_span.set_attributes.assert_called_with({
|
||||
gen_ai_attributes.OPERATION: CHAT_COMPLETION_OPERATION,
|
||||
gen_ai_attributes.SYSTEM: MockChatCompletion.MODEL_PROVIDER_NAME,
|
||||
gen_ai_attributes.MODEL: chat_completion.ai_model_id,
|
||||
})
|
||||
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.ADDRESS, chat_completion.service_url())
|
||||
|
||||
# No all connectors take the same parameters
|
||||
if execution_settings.extension_data.get("max_tokens") is not None:
|
||||
mock_span.set_attribute.assert_any_call(
|
||||
gen_ai_attributes.MAX_TOKENS, execution_settings.extension_data["max_tokens"]
|
||||
)
|
||||
if execution_settings.extension_data.get("temperature") is not None:
|
||||
mock_span.set_attribute.assert_any_call(
|
||||
gen_ai_attributes.TEMPERATURE, execution_settings.extension_data["temperature"]
|
||||
)
|
||||
if execution_settings.extension_data.get("top_p") is not None:
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.TOP_P, execution_settings.extension_data["top_p"])
|
||||
|
||||
mock_logger.info.assert_has_calls(
|
||||
[
|
||||
call(
|
||||
json.dumps(message.to_dict()),
|
||||
extra={
|
||||
gen_ai_attributes.EVENT_NAME: gen_ai_attributes.ROLE_EVENT_MAP.get(message.role),
|
||||
gen_ai_attributes.SYSTEM: MockChatCompletion.MODEL_PROVIDER_NAME,
|
||||
ChatHistoryMessageTimestampFilter.INDEX_KEY: idx,
|
||||
},
|
||||
)
|
||||
]
|
||||
for idx, message in enumerate(chat_history)
|
||||
)
|
||||
|
||||
# After the call to the model
|
||||
# Not all connectors return the same metadata
|
||||
if mock_response[0].metadata.get("id") is not None:
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.RESPONSE_ID, mock_response[0].metadata["id"])
|
||||
if any(completion.finish_reason is not None for completion in mock_response):
|
||||
mock_span.set_attribute.assert_any_call(
|
||||
gen_ai_attributes.FINISH_REASON,
|
||||
",".join([str(completion.finish_reason) for completion in mock_response]),
|
||||
)
|
||||
|
||||
mock_logger.info.assert_any_call(
|
||||
json.dumps({"message": results[0].to_dict(), "finish_reason": results[0].finish_reason}),
|
||||
extra={
|
||||
gen_ai_attributes.EVENT_NAME: gen_ai_attributes.CHOICE,
|
||||
gen_ai_attributes.SYSTEM: MockChatCompletion.MODEL_PROVIDER_NAME,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.logger")
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.tracer")
|
||||
async def test_trace_chat_completion_exception(
|
||||
mock_tracer,
|
||||
mock_logger,
|
||||
execution_settings,
|
||||
mock_response,
|
||||
chat_history,
|
||||
model_diagnostics_unit_test_env,
|
||||
):
|
||||
mock_span = mock_tracer.start_span.return_value
|
||||
# Setup
|
||||
chat_completion: ChatCompletionClientBase = MockChatCompletion(ai_model_id="ai_model_id")
|
||||
|
||||
with patch.object(MockChatCompletion, "_inner_get_chat_message_contents", side_effect=ServiceResponseException()):
|
||||
# We need to reapply the decorator to the method since the mock will not have the decorator applied
|
||||
MockChatCompletion._inner_get_chat_message_contents = trace_chat_completion(
|
||||
MockChatCompletion.MODEL_PROVIDER_NAME
|
||||
)(chat_completion._inner_get_chat_message_contents)
|
||||
|
||||
with pytest.raises(ServiceResponseException):
|
||||
await chat_completion.get_chat_message_contents(chat_history, execution_settings)
|
||||
|
||||
exception = ServiceResponseException()
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.ERROR_TYPE, str(type(exception)))
|
||||
mock_span.set_status.assert_any_call(StatusCode.ERROR, repr(exception))
|
||||
|
||||
mock_span.end.assert_any_call()
|
||||
|
||||
mock_logger.info.assert_has_calls(
|
||||
[
|
||||
call(
|
||||
json.dumps(message.to_dict()),
|
||||
extra={
|
||||
gen_ai_attributes.EVENT_NAME: gen_ai_attributes.ROLE_EVENT_MAP.get(message.role),
|
||||
gen_ai_attributes.SYSTEM: MockChatCompletion.MODEL_PROVIDER_NAME,
|
||||
ChatHistoryMessageTimestampFilter.INDEX_KEY: idx,
|
||||
},
|
||||
)
|
||||
]
|
||||
for idx, message in enumerate(chat_history)
|
||||
)
|
||||
@@ -0,0 +1,225 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from functools import reduce
|
||||
from unittest.mock import MagicMock, call, patch
|
||||
|
||||
import pytest
|
||||
from opentelemetry.trace import StatusCode
|
||||
|
||||
from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent
|
||||
from semantic_kernel.contents.utils.author_role import AuthorRole
|
||||
from semantic_kernel.contents.utils.finish_reason import FinishReason
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceResponseException
|
||||
from semantic_kernel.utils.telemetry.model_diagnostics import gen_ai_attributes
|
||||
from semantic_kernel.utils.telemetry.model_diagnostics.decorators import (
|
||||
CHAT_COMPLETION_OPERATION,
|
||||
ChatHistoryMessageTimestampFilter,
|
||||
trace_streaming_chat_completion,
|
||||
)
|
||||
from tests.unit.utils.model_diagnostics.conftest import MockChatCompletion
|
||||
|
||||
pytestmark = pytest.mark.parametrize(
|
||||
"execution_settings, mock_response",
|
||||
[
|
||||
pytest.param(
|
||||
PromptExecutionSettings(
|
||||
extension_data={
|
||||
"max_tokens": 1000,
|
||||
"temperature": 0.5,
|
||||
"top_p": 0.9,
|
||||
}
|
||||
),
|
||||
[
|
||||
StreamingChatMessageContent(
|
||||
role=AuthorRole.ASSISTANT,
|
||||
choice_index=0,
|
||||
ai_model_id="ai_model_id",
|
||||
content="Test content",
|
||||
metadata={"id": "test_id"},
|
||||
finish_reason=FinishReason.STOP,
|
||||
)
|
||||
],
|
||||
id="test_execution_settings_with_extension_data",
|
||||
),
|
||||
pytest.param(
|
||||
PromptExecutionSettings(),
|
||||
[
|
||||
StreamingChatMessageContent(
|
||||
role=AuthorRole.ASSISTANT,
|
||||
choice_index=0,
|
||||
ai_model_id="ai_model_id",
|
||||
metadata={"id": "test_id"},
|
||||
finish_reason=FinishReason.STOP,
|
||||
)
|
||||
],
|
||||
id="test_execution_settings_no_extension_data",
|
||||
),
|
||||
pytest.param(
|
||||
PromptExecutionSettings(),
|
||||
[
|
||||
StreamingChatMessageContent(
|
||||
role=AuthorRole.ASSISTANT,
|
||||
choice_index=0,
|
||||
ai_model_id="ai_model_id",
|
||||
metadata={},
|
||||
finish_reason=FinishReason.STOP,
|
||||
)
|
||||
],
|
||||
id="test_chat_message_content_no_metadata",
|
||||
),
|
||||
pytest.param(
|
||||
PromptExecutionSettings(),
|
||||
[
|
||||
StreamingChatMessageContent(
|
||||
role=AuthorRole.ASSISTANT,
|
||||
choice_index=0,
|
||||
ai_model_id="ai_model_id",
|
||||
metadata={"id": "test_id"},
|
||||
)
|
||||
],
|
||||
id="test_chat_message_content_no_finish_reason",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.logger")
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.tracer")
|
||||
async def test_trace_streaming_chat_completion(
|
||||
mock_tracer,
|
||||
mock_logger,
|
||||
execution_settings,
|
||||
mock_response,
|
||||
chat_history,
|
||||
model_diagnostics_unit_test_env,
|
||||
):
|
||||
# Setup
|
||||
mock_span = mock_tracer.start_span.return_value
|
||||
chat_completion: ChatCompletionClientBase = MockChatCompletion(ai_model_id="ai_model_id")
|
||||
iterable = MagicMock(spec=AsyncGenerator)
|
||||
iterable.__aiter__.return_value = [mock_response]
|
||||
|
||||
with patch.object(MockChatCompletion, "_inner_get_streaming_chat_message_contents", return_value=iterable):
|
||||
# We need to reapply the decorator to the method since the mock will not have the decorator applied
|
||||
MockChatCompletion._inner_get_streaming_chat_message_contents = trace_streaming_chat_completion(
|
||||
MockChatCompletion.MODEL_PROVIDER_NAME
|
||||
)(chat_completion._inner_get_streaming_chat_message_contents)
|
||||
|
||||
updates = []
|
||||
async for update in chat_completion._inner_get_streaming_chat_message_contents(
|
||||
chat_history, execution_settings
|
||||
):
|
||||
updates.append(update)
|
||||
updates_flatten = [reduce(lambda x, y: x + y, messages) for messages in updates]
|
||||
|
||||
assert updates_flatten == mock_response
|
||||
|
||||
# Before the call to the model
|
||||
mock_span.set_attributes.assert_called_with({
|
||||
gen_ai_attributes.OPERATION: CHAT_COMPLETION_OPERATION,
|
||||
gen_ai_attributes.SYSTEM: MockChatCompletion.MODEL_PROVIDER_NAME,
|
||||
gen_ai_attributes.MODEL: chat_completion.ai_model_id,
|
||||
})
|
||||
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.ADDRESS, chat_completion.service_url())
|
||||
|
||||
# No all connectors take the same parameters
|
||||
if execution_settings.extension_data.get("max_tokens") is not None:
|
||||
mock_span.set_attribute.assert_any_call(
|
||||
gen_ai_attributes.MAX_TOKENS, execution_settings.extension_data["max_tokens"]
|
||||
)
|
||||
if execution_settings.extension_data.get("temperature") is not None:
|
||||
mock_span.set_attribute.assert_any_call(
|
||||
gen_ai_attributes.TEMPERATURE, execution_settings.extension_data["temperature"]
|
||||
)
|
||||
if execution_settings.extension_data.get("top_p") is not None:
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.TOP_P, execution_settings.extension_data["top_p"])
|
||||
|
||||
mock_logger.info.assert_has_calls(
|
||||
[
|
||||
call(
|
||||
json.dumps(message.to_dict()),
|
||||
extra={
|
||||
gen_ai_attributes.EVENT_NAME: gen_ai_attributes.ROLE_EVENT_MAP.get(message.role),
|
||||
gen_ai_attributes.SYSTEM: MockChatCompletion.MODEL_PROVIDER_NAME,
|
||||
ChatHistoryMessageTimestampFilter.INDEX_KEY: idx,
|
||||
},
|
||||
)
|
||||
]
|
||||
for idx, message in enumerate(chat_history)
|
||||
)
|
||||
|
||||
# After the call to the model
|
||||
# Not all connectors return the same metadata
|
||||
if mock_response[0].metadata.get("id") is not None:
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.RESPONSE_ID, mock_response[0].metadata["id"])
|
||||
if any(completion.finish_reason is not None for completion in mock_response):
|
||||
mock_span.set_attribute.assert_any_call(
|
||||
gen_ai_attributes.FINISH_REASON,
|
||||
",".join([str(completion.finish_reason) for completion in mock_response]),
|
||||
)
|
||||
|
||||
mock_logger.info.assert_any_call(
|
||||
json.dumps({
|
||||
"message": updates_flatten[0].to_dict(),
|
||||
"finish_reason": updates_flatten[0].finish_reason,
|
||||
"index": 0,
|
||||
}),
|
||||
extra={
|
||||
gen_ai_attributes.EVENT_NAME: gen_ai_attributes.CHOICE,
|
||||
gen_ai_attributes.SYSTEM: MockChatCompletion.MODEL_PROVIDER_NAME,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.logger")
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.tracer")
|
||||
async def test_trace_streaming_chat_completion_exception(
|
||||
mock_tracer,
|
||||
mock_logger,
|
||||
execution_settings,
|
||||
mock_response,
|
||||
chat_history,
|
||||
model_diagnostics_unit_test_env,
|
||||
):
|
||||
# Setup
|
||||
mock_span = mock_tracer.start_span.return_value
|
||||
chat_completion: ChatCompletionClientBase = MockChatCompletion(ai_model_id="ai_model_id")
|
||||
|
||||
with patch.object(
|
||||
MockChatCompletion, "_inner_get_streaming_chat_message_contents", side_effect=ServiceResponseException()
|
||||
):
|
||||
# We need to reapply the decorator to the method since the mock will not have the decorator applied
|
||||
MockChatCompletion._inner_get_streaming_chat_message_contents = trace_streaming_chat_completion(
|
||||
MockChatCompletion.MODEL_PROVIDER_NAME
|
||||
)(chat_completion._inner_get_streaming_chat_message_contents)
|
||||
|
||||
with pytest.raises(ServiceResponseException):
|
||||
async for update in chat_completion._inner_get_streaming_chat_message_contents(
|
||||
chat_history, execution_settings
|
||||
):
|
||||
pass
|
||||
|
||||
exception = ServiceResponseException()
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.ERROR_TYPE, str(type(exception)))
|
||||
mock_span.set_status.assert_any_call(StatusCode.ERROR, repr(exception))
|
||||
|
||||
mock_span.end.assert_any_call()
|
||||
|
||||
mock_logger.info.assert_has_calls(
|
||||
[
|
||||
call(
|
||||
json.dumps(message.to_dict()),
|
||||
extra={
|
||||
gen_ai_attributes.EVENT_NAME: gen_ai_attributes.ROLE_EVENT_MAP.get(message.role),
|
||||
gen_ai_attributes.SYSTEM: MockChatCompletion.MODEL_PROVIDER_NAME,
|
||||
ChatHistoryMessageTimestampFilter.INDEX_KEY: idx,
|
||||
},
|
||||
)
|
||||
]
|
||||
for idx, message in enumerate(chat_history)
|
||||
)
|
||||
@@ -0,0 +1,183 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator
|
||||
from functools import reduce
|
||||
from unittest.mock import ANY, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from opentelemetry.trace import StatusCode
|
||||
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.connectors.ai.text_completion_client_base import TextCompletionClientBase
|
||||
from semantic_kernel.contents.streaming_text_content import StreamingTextContent
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceResponseException
|
||||
from semantic_kernel.utils.telemetry.model_diagnostics import gen_ai_attributes
|
||||
from semantic_kernel.utils.telemetry.model_diagnostics.decorators import (
|
||||
TEXT_COMPLETION_OPERATION,
|
||||
trace_streaming_text_completion,
|
||||
)
|
||||
from tests.unit.utils.model_diagnostics.conftest import MockTextCompletion
|
||||
|
||||
pytestmark = pytest.mark.parametrize(
|
||||
"execution_settings, mock_response",
|
||||
[
|
||||
pytest.param(
|
||||
PromptExecutionSettings(
|
||||
extension_data={
|
||||
"max_tokens": 1000,
|
||||
"temperature": 0.5,
|
||||
"top_p": 0.9,
|
||||
}
|
||||
),
|
||||
[
|
||||
StreamingTextContent(
|
||||
choice_index=0,
|
||||
ai_model_id="ai_model_id",
|
||||
text="Test content",
|
||||
metadata={"id": "test_id"},
|
||||
)
|
||||
],
|
||||
id="test_execution_settings_with_extension_data",
|
||||
),
|
||||
pytest.param(
|
||||
PromptExecutionSettings(),
|
||||
[
|
||||
StreamingTextContent(
|
||||
choice_index=0,
|
||||
ai_model_id="ai_model_id",
|
||||
text="Test content",
|
||||
metadata={"id": "test_id"},
|
||||
)
|
||||
],
|
||||
id="test_execution_settings_no_extension_data",
|
||||
),
|
||||
pytest.param(
|
||||
PromptExecutionSettings(),
|
||||
[
|
||||
StreamingTextContent(
|
||||
choice_index=0,
|
||||
ai_model_id="ai_model_id",
|
||||
text="Test content",
|
||||
metadata={},
|
||||
)
|
||||
],
|
||||
id="test_text_content_no_metadata",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.logger")
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.tracer")
|
||||
async def test_trace_streaming_text_completion(
|
||||
mock_tracer,
|
||||
mock_logger,
|
||||
execution_settings,
|
||||
mock_response,
|
||||
prompt,
|
||||
model_diagnostics_unit_test_env,
|
||||
):
|
||||
mock_span = mock_tracer.start_span.return_value
|
||||
# Setup
|
||||
mock_span = mock_tracer.start_span.return_value
|
||||
text_completion: TextCompletionClientBase = MockTextCompletion(ai_model_id="ai_model_id")
|
||||
iterable = MagicMock(spec=AsyncGenerator)
|
||||
iterable.__aiter__.return_value = [mock_response]
|
||||
|
||||
with patch.object(MockTextCompletion, "_inner_get_streaming_text_contents", return_value=iterable):
|
||||
# We need to reapply the decorator to the method since the mock will not have the decorator applied
|
||||
MockTextCompletion._inner_get_streaming_text_contents = trace_streaming_text_completion(
|
||||
MockTextCompletion.MODEL_PROVIDER_NAME
|
||||
)(text_completion._inner_get_streaming_text_contents)
|
||||
|
||||
updates = []
|
||||
async for update in text_completion.get_streaming_text_contents(prompt=prompt, settings=execution_settings):
|
||||
updates.append(update)
|
||||
updates_flatten = [reduce(lambda x, y: x + y, update) for update in updates]
|
||||
|
||||
assert updates_flatten == mock_response
|
||||
|
||||
# Before the call to the model
|
||||
mock_span.set_attributes.assert_called_with({
|
||||
gen_ai_attributes.OPERATION: TEXT_COMPLETION_OPERATION,
|
||||
gen_ai_attributes.SYSTEM: MockTextCompletion.MODEL_PROVIDER_NAME,
|
||||
gen_ai_attributes.MODEL: text_completion.ai_model_id,
|
||||
})
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
# The service_url attribute is not set for text completion
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.ADDRESS, ANY)
|
||||
|
||||
# No all connectors take the same parameters
|
||||
if execution_settings.extension_data.get("max_tokens") is not None:
|
||||
mock_span.set_attribute.assert_any_call(
|
||||
gen_ai_attributes.MAX_TOKENS, execution_settings.extension_data["max_tokens"]
|
||||
)
|
||||
if execution_settings.extension_data.get("temperature") is not None:
|
||||
mock_span.set_attribute.assert_any_call(
|
||||
gen_ai_attributes.TEMPERATURE, execution_settings.extension_data["temperature"]
|
||||
)
|
||||
if execution_settings.extension_data.get("top_p") is not None:
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.TOP_P, execution_settings.extension_data["top_p"])
|
||||
|
||||
mock_logger.info.assert_any_call(
|
||||
prompt,
|
||||
extra={
|
||||
gen_ai_attributes.EVENT_NAME: gen_ai_attributes.PROMPT,
|
||||
gen_ai_attributes.SYSTEM: MockTextCompletion.MODEL_PROVIDER_NAME,
|
||||
},
|
||||
)
|
||||
|
||||
# After the call to the model
|
||||
# Not all connectors return the same metadata
|
||||
if mock_response[0].metadata.get("id") is not None:
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.RESPONSE_ID, mock_response[0].metadata["id"])
|
||||
|
||||
mock_logger.info.assert_any_call(
|
||||
json.dumps({"message": updates_flatten[0].to_dict(), "index": 0}),
|
||||
extra={
|
||||
gen_ai_attributes.EVENT_NAME: gen_ai_attributes.CHOICE,
|
||||
gen_ai_attributes.SYSTEM: MockTextCompletion.MODEL_PROVIDER_NAME,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.logger")
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.tracer")
|
||||
async def test_trace_streaming_text_completion_exception(
|
||||
mock_tracer,
|
||||
mock_logger,
|
||||
execution_settings,
|
||||
mock_response,
|
||||
prompt,
|
||||
model_diagnostics_unit_test_env,
|
||||
):
|
||||
mock_span = mock_tracer.start_span.return_value
|
||||
# Setup
|
||||
mock_span = mock_tracer.start_span.return_value
|
||||
text_completion: TextCompletionClientBase = MockTextCompletion(ai_model_id="ai_model_id")
|
||||
|
||||
with patch.object(MockTextCompletion, "_inner_get_streaming_text_contents", side_effect=ServiceResponseException()):
|
||||
# We need to reapply the decorator to the method since the mock will not have the decorator applied
|
||||
MockTextCompletion._inner_get_streaming_text_contents = trace_streaming_text_completion(
|
||||
MockTextCompletion.MODEL_PROVIDER_NAME
|
||||
)(text_completion._inner_get_streaming_text_contents)
|
||||
|
||||
with pytest.raises(ServiceResponseException):
|
||||
async for update in text_completion.get_streaming_text_contents(prompt=prompt, settings=execution_settings):
|
||||
pass
|
||||
|
||||
exception = ServiceResponseException()
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.ERROR_TYPE, str(type(exception)))
|
||||
mock_span.set_status.assert_any_call(StatusCode.ERROR, repr(exception))
|
||||
|
||||
mock_span.end.assert_any_call()
|
||||
|
||||
mock_logger.info.assert_any_call(
|
||||
prompt,
|
||||
extra={
|
||||
gen_ai_attributes.EVENT_NAME: gen_ai_attributes.PROMPT,
|
||||
gen_ai_attributes.SYSTEM: MockTextCompletion.MODEL_PROVIDER_NAME,
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,175 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import json
|
||||
from unittest.mock import ANY, patch
|
||||
|
||||
import pytest
|
||||
from opentelemetry.trace import StatusCode
|
||||
|
||||
from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings
|
||||
from semantic_kernel.connectors.ai.text_completion_client_base import TextCompletionClientBase
|
||||
from semantic_kernel.contents.chat_message_content import ChatMessageContent
|
||||
from semantic_kernel.contents.text_content import TextContent
|
||||
from semantic_kernel.exceptions.service_exceptions import ServiceResponseException
|
||||
from semantic_kernel.utils.telemetry.model_diagnostics import gen_ai_attributes
|
||||
from semantic_kernel.utils.telemetry.model_diagnostics.decorators import (
|
||||
TEXT_COMPLETION_OPERATION,
|
||||
trace_text_completion,
|
||||
)
|
||||
from tests.unit.utils.model_diagnostics.conftest import MockTextCompletion
|
||||
|
||||
pytestmark = pytest.mark.parametrize(
|
||||
"execution_settings, mock_response",
|
||||
[
|
||||
pytest.param(
|
||||
PromptExecutionSettings(
|
||||
extension_data={
|
||||
"max_tokens": 1000,
|
||||
"temperature": 0.5,
|
||||
"top_p": 0.9,
|
||||
}
|
||||
),
|
||||
[
|
||||
TextContent(
|
||||
ai_model_id="ai_model_id",
|
||||
text="Test content",
|
||||
metadata={"id": "test_id"},
|
||||
)
|
||||
],
|
||||
id="test_execution_settings_with_extension_data",
|
||||
),
|
||||
pytest.param(
|
||||
PromptExecutionSettings(),
|
||||
[
|
||||
TextContent(
|
||||
ai_model_id="ai_model_id",
|
||||
text="Test content",
|
||||
metadata={"id": "test_id"},
|
||||
)
|
||||
],
|
||||
id="test_execution_settings_no_extension_data",
|
||||
),
|
||||
pytest.param(
|
||||
PromptExecutionSettings(),
|
||||
[
|
||||
TextContent(
|
||||
ai_model_id="ai_model_id",
|
||||
text="Test content",
|
||||
metadata={},
|
||||
)
|
||||
],
|
||||
id="test_text_content_no_metadata",
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.logger")
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.tracer")
|
||||
async def test_trace_text_completion(
|
||||
mock_tracer,
|
||||
mock_logger,
|
||||
execution_settings,
|
||||
mock_response,
|
||||
prompt,
|
||||
model_diagnostics_unit_test_env,
|
||||
):
|
||||
mock_span = mock_tracer.start_span.return_value
|
||||
# Setup
|
||||
mock_span = mock_tracer.start_span.return_value
|
||||
text_completion: TextCompletionClientBase = MockTextCompletion(ai_model_id="ai_model_id")
|
||||
|
||||
with patch.object(MockTextCompletion, "_inner_get_text_contents", return_value=mock_response):
|
||||
# We need to reapply the decorator to the method since the mock will not have the decorator applied
|
||||
MockTextCompletion._inner_get_text_contents = trace_text_completion(MockTextCompletion.MODEL_PROVIDER_NAME)(
|
||||
text_completion._inner_get_text_contents
|
||||
)
|
||||
|
||||
results: list[ChatMessageContent] = await text_completion.get_text_contents(
|
||||
prompt=prompt, settings=execution_settings
|
||||
)
|
||||
|
||||
assert results == mock_response
|
||||
|
||||
# Before the call to the model
|
||||
mock_span.set_attributes.assert_called_with({
|
||||
gen_ai_attributes.OPERATION: TEXT_COMPLETION_OPERATION,
|
||||
gen_ai_attributes.SYSTEM: MockTextCompletion.MODEL_PROVIDER_NAME,
|
||||
gen_ai_attributes.MODEL: text_completion.ai_model_id,
|
||||
})
|
||||
|
||||
with pytest.raises(AssertionError):
|
||||
# The service_url attribute is not set for text completion
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.ADDRESS, ANY)
|
||||
|
||||
# No all connectors take the same parameters
|
||||
if execution_settings.extension_data.get("max_tokens") is not None:
|
||||
mock_span.set_attribute.assert_any_call(
|
||||
gen_ai_attributes.MAX_TOKENS, execution_settings.extension_data["max_tokens"]
|
||||
)
|
||||
if execution_settings.extension_data.get("temperature") is not None:
|
||||
mock_span.set_attribute.assert_any_call(
|
||||
gen_ai_attributes.TEMPERATURE, execution_settings.extension_data["temperature"]
|
||||
)
|
||||
if execution_settings.extension_data.get("top_p") is not None:
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.TOP_P, execution_settings.extension_data["top_p"])
|
||||
|
||||
mock_logger.info.assert_any_call(
|
||||
prompt,
|
||||
extra={
|
||||
gen_ai_attributes.EVENT_NAME: gen_ai_attributes.PROMPT,
|
||||
gen_ai_attributes.SYSTEM: MockTextCompletion.MODEL_PROVIDER_NAME,
|
||||
},
|
||||
)
|
||||
|
||||
# After the call to the model
|
||||
# Not all connectors return the same metadata
|
||||
if mock_response[0].metadata.get("id") is not None:
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.RESPONSE_ID, mock_response[0].metadata["id"])
|
||||
|
||||
mock_logger.info.assert_any_call(
|
||||
json.dumps({"message": results[0].to_dict()}),
|
||||
extra={
|
||||
gen_ai_attributes.EVENT_NAME: gen_ai_attributes.CHOICE,
|
||||
gen_ai_attributes.SYSTEM: MockTextCompletion.MODEL_PROVIDER_NAME,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.logger")
|
||||
@patch("semantic_kernel.utils.telemetry.model_diagnostics.decorators.tracer")
|
||||
async def test_trace_text_completion_exception(
|
||||
mock_tracer,
|
||||
mock_logger,
|
||||
execution_settings,
|
||||
mock_response,
|
||||
prompt,
|
||||
model_diagnostics_unit_test_env,
|
||||
):
|
||||
mock_span = mock_tracer.start_span.return_value
|
||||
# Setup
|
||||
mock_span = mock_tracer.start_span.return_value
|
||||
text_completion: TextCompletionClientBase = MockTextCompletion(ai_model_id="ai_model_id")
|
||||
|
||||
with patch.object(MockTextCompletion, "_inner_get_text_contents", side_effect=ServiceResponseException()):
|
||||
# We need to reapply the decorator to the method since the mock will not have the decorator applied
|
||||
MockTextCompletion._inner_get_text_contents = trace_text_completion(MockTextCompletion.MODEL_PROVIDER_NAME)(
|
||||
text_completion._inner_get_text_contents
|
||||
)
|
||||
|
||||
with pytest.raises(ServiceResponseException):
|
||||
await text_completion.get_text_contents(prompt=prompt, settings=execution_settings)
|
||||
|
||||
exception = ServiceResponseException()
|
||||
mock_span.set_attribute.assert_any_call(gen_ai_attributes.ERROR_TYPE, str(type(exception)))
|
||||
mock_span.set_status.assert_any_call(StatusCode.ERROR, repr(exception))
|
||||
|
||||
mock_span.end.assert_any_call()
|
||||
|
||||
mock_logger.info.assert_any_call(
|
||||
prompt,
|
||||
extra={
|
||||
gen_ai_attributes.EVENT_NAME: gen_ai_attributes.PROMPT,
|
||||
gen_ai_attributes.SYSTEM: MockTextCompletion.MODEL_PROVIDER_NAME,
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,21 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from unittest.mock import Mock
|
||||
|
||||
from semantic_kernel.utils.chat import store_results
|
||||
|
||||
|
||||
def test_store_results():
|
||||
chat_history_mock = Mock()
|
||||
chat_history_mock.add_message = Mock()
|
||||
|
||||
chat_message_content_mock = Mock()
|
||||
results = [chat_message_content_mock, chat_message_content_mock]
|
||||
|
||||
updated_chat_history = store_results(chat_history_mock, results)
|
||||
|
||||
assert chat_history_mock.add_message.call_count == len(results)
|
||||
for message in results:
|
||||
chat_history_mock.add_message.assert_any_call(message=message)
|
||||
|
||||
assert updated_chat_history == chat_history_mock
|
||||
@@ -0,0 +1,124 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from semantic_kernel.utils.feature_stage_decorator import experimental, release_candidate
|
||||
|
||||
|
||||
@experimental
|
||||
def my_function() -> None:
|
||||
"""This is a sample function docstring."""
|
||||
pass
|
||||
|
||||
|
||||
@release_candidate
|
||||
def my_function_release_candidate() -> None:
|
||||
"""This is a sample function docstring."""
|
||||
pass
|
||||
|
||||
|
||||
@release_candidate
|
||||
def my_function_release_candidate_no_doc_string() -> None:
|
||||
pass
|
||||
|
||||
|
||||
@release_candidate(version="1.0.0-rc2")
|
||||
def my_function_release_candidate_with_version() -> None:
|
||||
"""This is a sample function docstring."""
|
||||
pass
|
||||
|
||||
|
||||
@experimental
|
||||
def my_function_no_doc_string() -> None:
|
||||
pass
|
||||
|
||||
|
||||
@experimental
|
||||
class MyExperimentalClass:
|
||||
"""A class that is still evolving rapidly."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@release_candidate
|
||||
class MyRCClass:
|
||||
"""A class that is nearly final, but still in release-candidate stage."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
@release_candidate(version="1.0.0-rc2")
|
||||
class MyRCClassTwo:
|
||||
"""A class that is nearly final, but still in release-candidate stage."""
|
||||
|
||||
pass
|
||||
|
||||
|
||||
def test_function_experimental_decorator():
|
||||
assert (
|
||||
my_function.__doc__
|
||||
== "This is a sample function docstring.\n\nNote: This function is marked as 'experimental' and may change in the future." # noqa: E501
|
||||
)
|
||||
assert hasattr(my_function, "is_experimental")
|
||||
assert my_function.is_experimental is True
|
||||
|
||||
|
||||
def test_function_experimental_decorator_with_no_doc_string():
|
||||
assert (
|
||||
my_function_no_doc_string.__doc__
|
||||
== "Note: This function is marked as 'experimental' and may change in the future."
|
||||
)
|
||||
assert hasattr(my_function_no_doc_string, "is_experimental")
|
||||
assert my_function_no_doc_string.is_experimental is True
|
||||
|
||||
|
||||
def test_function_release_candidate_decorator():
|
||||
assert (
|
||||
"Features marked with this status are nearing completion and are considered"
|
||||
in my_function_release_candidate_no_doc_string.__doc__
|
||||
)
|
||||
assert hasattr(my_function_release_candidate, "is_release_candidate")
|
||||
assert my_function_release_candidate.is_release_candidate is True
|
||||
assert "Version:" in my_function_release_candidate_no_doc_string.__doc__
|
||||
|
||||
|
||||
def test_function_release_candidate_decorator_and_version():
|
||||
assert (
|
||||
"Features marked with this status are nearing completion and are considered"
|
||||
in my_function_release_candidate_with_version.__doc__
|
||||
)
|
||||
assert hasattr(my_function_release_candidate, "is_release_candidate")
|
||||
assert my_function_release_candidate.is_release_candidate is True
|
||||
assert "Version:" in my_function_release_candidate_with_version.__doc__
|
||||
|
||||
|
||||
def test_function_release_candidate_decorator_with_no_doc_string():
|
||||
assert (
|
||||
"Features marked with this status are nearing completion" in my_function_release_candidate_no_doc_string.__doc__
|
||||
)
|
||||
assert hasattr(my_function_release_candidate_no_doc_string, "is_release_candidate")
|
||||
assert my_function_release_candidate_no_doc_string.is_release_candidate is True
|
||||
assert "Version:" in my_function_release_candidate_no_doc_string.__doc__
|
||||
|
||||
|
||||
def test_class_experimental_decorator():
|
||||
assert MyExperimentalClass.__doc__ == (
|
||||
"A class that is still evolving rapidly.\n\nNote: This class is marked as "
|
||||
"'experimental' and may change in the future."
|
||||
)
|
||||
assert hasattr(MyExperimentalClass, "is_experimental")
|
||||
assert MyExperimentalClass.is_experimental is True
|
||||
|
||||
|
||||
def test_class_release_candidate_decorator():
|
||||
assert "Features marked with this status are nearing completion" in MyRCClass.__doc__
|
||||
assert hasattr(MyRCClass, "is_release_candidate")
|
||||
assert MyRCClass.is_release_candidate is True
|
||||
assert "Version:" in MyRCClass.__doc__
|
||||
|
||||
|
||||
def test_class_release_candidate_decorator_with_version():
|
||||
assert "Features marked with this status are nearing completion" in MyRCClassTwo.__doc__
|
||||
expected_version = "1.0.0-rc2"
|
||||
assert expected_version in MyRCClassTwo.__doc__
|
||||
assert hasattr(MyRCClassTwo, "is_release_candidate")
|
||||
assert MyRCClassTwo.is_release_candidate is True
|
||||
assert "Version:" in MyRCClassTwo.__doc__
|
||||
@@ -0,0 +1,14 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import logging
|
||||
|
||||
from semantic_kernel.utils.logging import setup_logging
|
||||
|
||||
|
||||
def test_setup_logging():
|
||||
"""Test that the logging is setup correctly."""
|
||||
setup_logging()
|
||||
|
||||
root_logger = logging.getLogger()
|
||||
assert root_logger.handlers
|
||||
assert any(isinstance(handler, logging.StreamHandler) for handler in root_logger.handlers)
|
||||
Reference in New Issue
Block a user