chore: import upstream snapshot with attribution
This commit is contained in:
@@ -0,0 +1,86 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def database_name():
|
||||
"""Fixture for the database name."""
|
||||
return "test_database"
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def collection_name():
|
||||
"""Fixture for the collection name."""
|
||||
return "test_collection"
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def url():
|
||||
"""Fixture for the url."""
|
||||
return "https://test.cosmos.azure.com/"
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def key():
|
||||
"""Fixture for the key."""
|
||||
return "test_key"
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def azure_cosmos_db_mongo_db_unit_test_env(monkeypatch, url, key, database_name, exclude_list, override_env_param_dict):
|
||||
"""Fixture to set environment variables for Azure Cosmos DB NoSQL unit tests."""
|
||||
if exclude_list is None:
|
||||
exclude_list = []
|
||||
|
||||
if override_env_param_dict is None:
|
||||
override_env_param_dict = {}
|
||||
|
||||
env_vars = {
|
||||
"AZURE_COSMOS_DB_MONGODB_CONNECTION_STRING": url,
|
||||
"AZURE_COSMOS_DB_MONGODB_DATABASE_NAME": database_name,
|
||||
}
|
||||
|
||||
env_vars.update(override_env_param_dict)
|
||||
|
||||
for key, value in env_vars.items():
|
||||
if key not in exclude_list:
|
||||
monkeypatch.setenv(key, value)
|
||||
else:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
return env_vars
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def azure_cosmos_db_no_sql_unit_test_env(monkeypatch, url, key, database_name, exclude_list, override_env_param_dict):
|
||||
"""Fixture to set environment variables for Azure Cosmos DB NoSQL unit tests."""
|
||||
if exclude_list is None:
|
||||
exclude_list = []
|
||||
|
||||
if override_env_param_dict is None:
|
||||
override_env_param_dict = {}
|
||||
|
||||
env_vars = {
|
||||
"AZURE_COSMOS_DB_NO_SQL_URL": url,
|
||||
"AZURE_COSMOS_DB_NO_SQL_KEY": key,
|
||||
"AZURE_COSMOS_DB_NO_SQL_DATABASE_NAME": database_name,
|
||||
}
|
||||
|
||||
env_vars.update(override_env_param_dict)
|
||||
|
||||
for key, value in env_vars.items():
|
||||
if key not in exclude_list:
|
||||
monkeypatch.setenv(key, value)
|
||||
else:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
return env_vars
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def clear_azure_cosmos_db_no_sql_env(monkeypatch):
|
||||
"""Fixture to clear the environment variables for Weaviate unit tests."""
|
||||
monkeypatch.delenv("AZURE_COSMOS_DB_NO_SQL_URL", raising=False)
|
||||
monkeypatch.delenv("AZURE_COSMOS_DB_NO_SQL_KEY", raising=False)
|
||||
monkeypatch.delenv("AZURE_COSMOS_DB_NO_SQL_DATABASE_NAME", raising=False)
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from pymongo import AsyncMongoClient
|
||||
|
||||
from semantic_kernel.connectors.azure_cosmos_db import CosmosMongoCollection
|
||||
from semantic_kernel.data.vector import VectorStoreCollectionDefinition, VectorStoreField
|
||||
from semantic_kernel.exceptions import VectorStoreInitializationException
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_model() -> VectorStoreCollectionDefinition:
|
||||
return VectorStoreCollectionDefinition(
|
||||
fields=[
|
||||
VectorStoreField("key", name="id"),
|
||||
VectorStoreField("data", name="content"),
|
||||
VectorStoreField("vector", name="vector", dimensions=5),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
async def test_constructor_with_mongo_client_provided(mock_model) -> None:
|
||||
"""
|
||||
Test the constructor of AzureCosmosDBforMongoDBCollection when a mongo_client
|
||||
is directly provided. Expect that the class is successfully initialized
|
||||
and doesn't attempt to manage the client.
|
||||
"""
|
||||
mock_client = AsyncMock(spec=AsyncMongoClient)
|
||||
collection_name = "test_collection"
|
||||
|
||||
collection = CosmosMongoCollection(
|
||||
collection_name=collection_name,
|
||||
record_type=dict,
|
||||
mongo_client=mock_client,
|
||||
definition=mock_model,
|
||||
)
|
||||
|
||||
assert collection.mongo_client == mock_client
|
||||
assert collection.collection_name == collection_name
|
||||
assert not collection.managed_client, "Should not be managing client when provided"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exclude_list", [["AZURE_COSMOS_DB_MONGODB_CONNECTION_STRING"]], indirect=True)
|
||||
async def test_constructor_raises_exception_on_validation_error(
|
||||
azure_cosmos_db_mongo_db_unit_test_env, definition
|
||||
) -> None:
|
||||
"""
|
||||
Test that the constructor raises VectorStoreInitializationException when
|
||||
AzureCosmosDBforMongoDBSettings fails with ValidationError.
|
||||
"""
|
||||
with pytest.raises(VectorStoreInitializationException) as exc_info:
|
||||
CosmosMongoCollection(
|
||||
collection_name="test_collection",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
database_name="",
|
||||
env_file_path=".no.env",
|
||||
)
|
||||
assert "The Azure CosmosDB for MongoDB connection string is required." in str(exc_info.value)
|
||||
|
||||
|
||||
async def test_ensure_collection_exists_calls_database_methods(definition) -> None:
|
||||
"""
|
||||
Test ensure_collection_exists to verify that it first creates a collection, then
|
||||
calls the appropriate command to create a vector index.
|
||||
"""
|
||||
# Setup
|
||||
mock_database = AsyncMock()
|
||||
mock_database.create_collection = AsyncMock()
|
||||
mock_database.command = AsyncMock()
|
||||
|
||||
mock_client = AsyncMock(spec=AsyncMongoClient)
|
||||
mock_client.get_database = MagicMock(return_value=mock_database)
|
||||
|
||||
# Instantiate
|
||||
collection = CosmosMongoCollection(
|
||||
collection_name="test_collection",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
mongo_client=mock_client,
|
||||
database_name="test_db",
|
||||
)
|
||||
|
||||
# Act
|
||||
await collection.ensure_collection_exists(customArg="customValue")
|
||||
|
||||
# Assert
|
||||
mock_database.create_collection.assert_awaited_once_with("test_collection", customArg="customValue")
|
||||
mock_database.command.assert_awaited()
|
||||
command_args = mock_database.command.call_args.kwargs["command"]
|
||||
|
||||
assert command_args["createIndexes"] == "test_collection"
|
||||
assert len(command_args["indexes"]) == 2, "One for the data field, one for the vector field"
|
||||
# Check the data field index
|
||||
assert command_args["indexes"][0]["name"] == "content_"
|
||||
# Check the vector field index creation
|
||||
assert command_args["indexes"][1]["name"] == "vector_"
|
||||
assert command_args["indexes"][1]["key"] == {"vector": "cosmosSearch"}
|
||||
assert command_args["indexes"][1]["cosmosSearchOptions"]["kind"] == "COS"
|
||||
assert command_args["indexes"][1]["cosmosSearchOptions"]["similarity"] is not None
|
||||
assert command_args["indexes"][1]["cosmosSearchOptions"]["dimensions"] == 5
|
||||
|
||||
|
||||
async def test_context_manager_calls_aconnect_and_close_when_managed(mock_model) -> None:
|
||||
"""
|
||||
Test that the context manager in AzureCosmosDBforMongoDBCollection calls 'aconnect' and
|
||||
'close' when the client is managed (i.e., created internally).
|
||||
"""
|
||||
mock_client = AsyncMock(spec=AsyncMongoClient)
|
||||
|
||||
with patch(
|
||||
"semantic_kernel.connectors.azure_cosmos_db.AsyncMongoClient",
|
||||
return_value=mock_client,
|
||||
):
|
||||
collection = CosmosMongoCollection(
|
||||
collection_name="test_collection",
|
||||
record_type=dict,
|
||||
connection_string="mongodb://fake",
|
||||
definition=mock_model,
|
||||
)
|
||||
|
||||
# "__aenter__" should call 'aconnect'
|
||||
async with collection as c:
|
||||
mock_client.aconnect.assert_awaited_once()
|
||||
assert c is collection
|
||||
|
||||
# "__aexit__" should call 'close' if managed
|
||||
mock_client.close.assert_awaited_once()
|
||||
|
||||
|
||||
async def test_context_manager_does_not_close_when_not_managed(mock_model) -> None:
|
||||
"""
|
||||
Test that the context manager in AzureCosmosDBforMongoDBCollection does not call 'close'
|
||||
when the client is not managed (i.e., provided externally).
|
||||
"""
|
||||
|
||||
external_client = AsyncMock(spec=AsyncMongoClient, name="external_client", value=None)
|
||||
external_client.aconnect = AsyncMock(name="aconnect")
|
||||
external_client.close = AsyncMock(name="close")
|
||||
|
||||
collection = CosmosMongoCollection(
|
||||
collection_name="test_collection",
|
||||
record_type=dict,
|
||||
mongo_client=external_client,
|
||||
definition=mock_model,
|
||||
)
|
||||
|
||||
# "__aenter__" scenario
|
||||
async with collection as c:
|
||||
external_client.aconnect.assert_awaited()
|
||||
assert c is collection
|
||||
|
||||
# "__aexit__" should NOT call "close" when not managed
|
||||
external_client.close.assert_not_awaited()
|
||||
+554
@@ -0,0 +1,554 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from collections.abc import AsyncGenerator
|
||||
from unittest.mock import ANY, AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from azure.core.credentials_async import AsyncTokenCredential
|
||||
from azure.cosmos.aio import CosmosClient
|
||||
from azure.cosmos.exceptions import CosmosHttpResponseError, CosmosResourceNotFoundError
|
||||
|
||||
from semantic_kernel.connectors.azure_cosmos_db import (
|
||||
COSMOS_ITEM_ID_PROPERTY_NAME,
|
||||
CosmosNoSqlCollection,
|
||||
_create_default_indexing_policy_nosql,
|
||||
_create_default_vector_embedding_policy,
|
||||
)
|
||||
from semantic_kernel.data._shared import default_dynamic_filter_function
|
||||
from semantic_kernel.exceptions import (
|
||||
VectorStoreInitializationException,
|
||||
VectorStoreModelException,
|
||||
VectorStoreOperationException,
|
||||
)
|
||||
from semantic_kernel.functions import KernelParameterMetadata
|
||||
|
||||
|
||||
def test_azure_cosmos_db_no_sql_collection_init(
|
||||
clear_azure_cosmos_db_no_sql_env,
|
||||
record_type,
|
||||
database_name: str,
|
||||
collection_name: str,
|
||||
url: str,
|
||||
key: str,
|
||||
) -> None:
|
||||
"""Test the initialization of an AzureCosmosDBNoSQLCollection object."""
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
database_name=database_name,
|
||||
url=url,
|
||||
key=key,
|
||||
)
|
||||
|
||||
assert vector_collection is not None
|
||||
assert vector_collection.database_name == database_name
|
||||
assert vector_collection.collection_name == collection_name
|
||||
assert vector_collection.cosmos_client is not None
|
||||
assert vector_collection.partition_key.path == f"/{vector_collection.definition.key_name}"
|
||||
assert vector_collection.create_database is False
|
||||
|
||||
|
||||
def test_azure_cosmos_db_no_sql_collection_init_env(
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test the initialization of an AzureCosmosDBNoSQLCollection object with environment variables."""
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
assert vector_collection is not None
|
||||
assert (
|
||||
vector_collection.database_name == azure_cosmos_db_no_sql_unit_test_env["AZURE_COSMOS_DB_NO_SQL_DATABASE_NAME"]
|
||||
)
|
||||
assert vector_collection.collection_name == collection_name
|
||||
assert vector_collection.partition_key.path == f"/{vector_collection.definition.key_name}"
|
||||
assert vector_collection.create_database is False
|
||||
|
||||
|
||||
def test_azure_cosmos_db_no_sql_build_filter_escapes_apostrophes(
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test Cosmos DB filter building escapes apostrophes in string literals."""
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
filter_string = vector_collection._build_filter('lambda x: x.content == "O\'Reilly"')
|
||||
|
||||
assert filter_string == "c.content = 'O''Reilly'"
|
||||
|
||||
|
||||
def test_azure_cosmos_db_no_sql_build_filter_escapes_injection_payload(
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test Cosmos DB filter building keeps injection-shaped strings inside the literal."""
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
filter_string = vector_collection._build_filter("lambda x: x.content == \"test' OR '1'='1\"")
|
||||
|
||||
assert filter_string == "c.content = 'test'' OR ''1''=''1'"
|
||||
|
||||
|
||||
def test_azure_cosmos_db_no_sql_dynamic_filter_injection_payload_stays_literal(
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test default_dynamic_filter_function does not let user values alter Cosmos filter syntax."""
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
generated_filter = default_dynamic_filter_function(
|
||||
filter=None,
|
||||
parameters=[
|
||||
KernelParameterMetadata(
|
||||
name="content",
|
||||
description="Content filter",
|
||||
type="str",
|
||||
is_required=False,
|
||||
type_object=str,
|
||||
)
|
||||
],
|
||||
content="test' OR '1'='1",
|
||||
)
|
||||
|
||||
assert isinstance(generated_filter, str)
|
||||
assert vector_collection._build_filter(generated_filter) == "c.content = 'test'' OR ''1''=''1'"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exclude_list", [["AZURE_COSMOS_DB_NO_SQL_URL"]], indirect=True)
|
||||
def test_azure_cosmos_db_no_sql_collection_init_no_url(
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test the initialization of an AzureCosmosDBNoSQLCollection object with missing URL."""
|
||||
with pytest.raises(VectorStoreInitializationException):
|
||||
CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
env_file_path="fake_path",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exclude_list", [["AZURE_COSMOS_DB_NO_SQL_DATABASE_NAME"]], indirect=True)
|
||||
def test_azure_cosmos_db_no_sql_collection_init_no_database_name(
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test the initialization of an AzureCosmosDBNoSQLCollection object with missing database name."""
|
||||
with pytest.raises(
|
||||
VectorStoreInitializationException, match="The name of the Azure Cosmos DB NoSQL database is missing."
|
||||
):
|
||||
CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
env_file_path="fake_path",
|
||||
)
|
||||
|
||||
|
||||
def test_azure_cosmos_db_no_sql_collection_invalid_settings(
|
||||
clear_azure_cosmos_db_no_sql_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test the initialization of an AzureCosmosDBNoSQLCollection object with invalid settings."""
|
||||
with pytest.raises(VectorStoreInitializationException):
|
||||
CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
url="invalid_url",
|
||||
)
|
||||
|
||||
|
||||
@patch.object(CosmosClient, "__init__", return_value=None)
|
||||
def test_azure_cosmos_db_no_sql_get_cosmos_client(
|
||||
mock_cosmos_client_init,
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test the creation of a cosmos client."""
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
assert vector_collection.cosmos_client is not None
|
||||
mock_cosmos_client_init.assert_called_once_with(
|
||||
str(azure_cosmos_db_no_sql_unit_test_env["AZURE_COSMOS_DB_NO_SQL_URL"]),
|
||||
credential=azure_cosmos_db_no_sql_unit_test_env["AZURE_COSMOS_DB_NO_SQL_KEY"],
|
||||
)
|
||||
|
||||
|
||||
@patch.object(CosmosClient, "__init__", return_value=None)
|
||||
def test_azure_cosmos_db_no_sql_get_cosmos_client_without_key(
|
||||
mock_cosmos_client_init,
|
||||
clear_azure_cosmos_db_no_sql_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
database_name: str,
|
||||
url: str,
|
||||
) -> None:
|
||||
"""Test the creation of a cosmos client."""
|
||||
credential = AsyncMock(spec=AsyncTokenCredential)
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
database_name=database_name,
|
||||
url=url,
|
||||
credential=credential,
|
||||
)
|
||||
|
||||
assert vector_collection.cosmos_client is not None
|
||||
mock_cosmos_client_init.assert_called_once_with(url, credential=ANY)
|
||||
|
||||
|
||||
@patch("azure.cosmos.aio.CosmosClient", spec=True)
|
||||
async def test_azure_cosmos_db_no_sql_collection_create_database_if_not_exists(
|
||||
mock_cosmos_client,
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test the creation of a cosmos DB NoSQL database if it does not exist when create_database=True."""
|
||||
mock_cosmos_client.get_database_client.side_effect = CosmosResourceNotFoundError
|
||||
mock_cosmos_client.create_database = AsyncMock()
|
||||
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
cosmos_client=mock_cosmos_client,
|
||||
create_database=True,
|
||||
)
|
||||
|
||||
assert vector_collection.create_database is True
|
||||
|
||||
await vector_collection._get_database_proxy()
|
||||
|
||||
mock_cosmos_client.get_database_client.assert_called_once_with(
|
||||
azure_cosmos_db_no_sql_unit_test_env["AZURE_COSMOS_DB_NO_SQL_DATABASE_NAME"]
|
||||
)
|
||||
mock_cosmos_client.create_database.assert_called_once_with(
|
||||
azure_cosmos_db_no_sql_unit_test_env["AZURE_COSMOS_DB_NO_SQL_DATABASE_NAME"]
|
||||
)
|
||||
|
||||
|
||||
@patch("azure.cosmos.aio.CosmosClient", spec=True)
|
||||
async def test_azure_cosmos_db_no_sql_collection_create_database_raise_if_database_not_exists(
|
||||
mock_cosmos_client,
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test _get_database_proxy raises an error if the database does not exist when create_database=False."""
|
||||
mock_cosmos_client.get_database_client.side_effect = CosmosResourceNotFoundError
|
||||
mock_cosmos_client.create_database = AsyncMock()
|
||||
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
cosmos_client=mock_cosmos_client,
|
||||
create_database=False,
|
||||
)
|
||||
|
||||
assert vector_collection.create_database is False
|
||||
|
||||
with pytest.raises(VectorStoreOperationException):
|
||||
await vector_collection._get_database_proxy()
|
||||
|
||||
|
||||
@patch("azure.cosmos.aio.CosmosClient")
|
||||
@patch("azure.cosmos.aio.DatabaseProxy")
|
||||
@pytest.mark.parametrize("index_kind, distance_function", [("flat", "cosine_similarity")])
|
||||
async def test_azure_cosmos_db_no_sql_collection_ensure_collection_exists(
|
||||
mock_database_proxy,
|
||||
mock_cosmos_client,
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
):
|
||||
"""Test the creation of a cosmos DB NoSQL collection."""
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
vector_collection._get_database_proxy = AsyncMock(return_value=mock_database_proxy)
|
||||
|
||||
mock_database_proxy.create_container_if_not_exists = AsyncMock(return_value=None)
|
||||
|
||||
await vector_collection.ensure_collection_exists()
|
||||
|
||||
mock_database_proxy.create_container_if_not_exists.assert_called_once_with(
|
||||
id=collection_name,
|
||||
partition_key=vector_collection.partition_key,
|
||||
indexing_policy=_create_default_indexing_policy_nosql(vector_collection.definition),
|
||||
vector_embedding_policy=_create_default_vector_embedding_policy(vector_collection.definition),
|
||||
)
|
||||
|
||||
|
||||
@patch("azure.cosmos.aio.CosmosClient")
|
||||
@patch("azure.cosmos.aio.DatabaseProxy")
|
||||
@pytest.mark.parametrize("index_kind, distance_function", [("flat", "cosine_similarity")])
|
||||
async def test_azure_cosmos_db_no_sql_collection_ensure_collection_exists_allow_custom_indexing_policy(
|
||||
mock_database_proxy,
|
||||
mock_cosmos_client,
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
):
|
||||
"""Test the creation of a cosmos DB NoSQL collection with a custom indexing policy."""
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
vector_collection._get_database_proxy = AsyncMock(return_value=mock_database_proxy)
|
||||
|
||||
mock_database_proxy.create_container_if_not_exists = AsyncMock(return_value=None)
|
||||
|
||||
await vector_collection.ensure_collection_exists(indexing_policy={"automatic": False})
|
||||
|
||||
mock_database_proxy.create_container_if_not_exists.assert_called_once_with(
|
||||
id=collection_name,
|
||||
partition_key=vector_collection.partition_key,
|
||||
indexing_policy={"automatic": False},
|
||||
vector_embedding_policy=_create_default_vector_embedding_policy(vector_collection.definition),
|
||||
)
|
||||
|
||||
|
||||
@patch("azure.cosmos.aio.CosmosClient")
|
||||
@patch("azure.cosmos.aio.DatabaseProxy")
|
||||
@pytest.mark.parametrize("index_kind, distance_function", [("flat", "cosine_similarity")])
|
||||
async def test_azure_cosmos_db_no_sql_collection_ensure_collection_exists_allow_custom_vector_embedding_policy(
|
||||
mock_database_proxy,
|
||||
mock_cosmos_client,
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
):
|
||||
"""Test the creation of a cosmos DB NoSQL collection with a custom vector embedding policy."""
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
vector_collection._get_database_proxy = AsyncMock(return_value=mock_database_proxy)
|
||||
|
||||
mock_database_proxy.create_container_if_not_exists = AsyncMock(return_value=None)
|
||||
|
||||
await vector_collection.ensure_collection_exists(vector_embedding_policy={"vectorEmbeddings": []})
|
||||
|
||||
mock_database_proxy.create_container_if_not_exists.assert_called_once_with(
|
||||
id=collection_name,
|
||||
partition_key=vector_collection.partition_key,
|
||||
indexing_policy=_create_default_indexing_policy_nosql(vector_collection.definition),
|
||||
vector_embedding_policy={"vectorEmbeddings": []},
|
||||
)
|
||||
|
||||
|
||||
@patch("azure.cosmos.aio.CosmosClient")
|
||||
@patch("azure.cosmos.aio.DatabaseProxy")
|
||||
@pytest.mark.parametrize(
|
||||
"index_kind, distance_function, vector_property_type",
|
||||
[
|
||||
("hnsw", "cosine_similarity", "float"), # unsupported index kind
|
||||
("flat", "hamming", "float"), # unsupported distance function
|
||||
],
|
||||
)
|
||||
async def test_azure_cosmos_db_no_sql_collection_ensure_collection_exists_unsupported_vector_field_property(
|
||||
mock_database_proxy,
|
||||
mock_cosmos_client,
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
):
|
||||
"""Test the creation of a cosmos DB NoSQL collection with an unsupported index kind."""
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
vector_collection._get_database_proxy = AsyncMock(return_value=mock_database_proxy)
|
||||
|
||||
mock_database_proxy.create_container_if_not_exists = AsyncMock(return_value=None)
|
||||
|
||||
with pytest.raises(VectorStoreModelException):
|
||||
await vector_collection.ensure_collection_exists()
|
||||
|
||||
|
||||
@patch("azure.cosmos.aio.DatabaseProxy")
|
||||
async def test_azure_cosmos_db_no_sql_collection_ensure_collection_deleted(
|
||||
mock_database_proxy,
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test the deletion of a cosmos DB NoSQL collection."""
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
vector_collection._get_database_proxy = AsyncMock(return_value=mock_database_proxy)
|
||||
|
||||
mock_database_proxy.delete_container = AsyncMock()
|
||||
|
||||
await vector_collection.ensure_collection_deleted()
|
||||
|
||||
mock_database_proxy.delete_container.assert_called_once_with(collection_name)
|
||||
|
||||
|
||||
@patch("azure.cosmos.aio.DatabaseProxy")
|
||||
async def test_azure_cosmos_db_no_sql_collection_ensure_collection_deleted_fail(
|
||||
mock_database_proxy,
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test the deletion of a cosmos DB NoSQL collection that does not exist."""
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
vector_collection._get_database_proxy = AsyncMock(return_value=mock_database_proxy)
|
||||
mock_database_proxy.delete_container = AsyncMock(side_effect=CosmosHttpResponseError)
|
||||
|
||||
with pytest.raises(VectorStoreOperationException, match="Container could not be deleted."):
|
||||
await vector_collection.ensure_collection_deleted()
|
||||
|
||||
|
||||
@patch("azure.cosmos.aio.ContainerProxy")
|
||||
async def test_azure_cosmos_db_no_sql_upsert(
|
||||
mock_container_proxy,
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test the upsert of a document in a cosmos DB NoSQL collection."""
|
||||
item = {"content": "test_content", "vector": [1.0, 2.0, 3.0], "id": "test_id"}
|
||||
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
vector_collection._get_container_proxy = AsyncMock(return_value=mock_container_proxy)
|
||||
|
||||
mock_container_proxy.upsert_item = AsyncMock(return_value={COSMOS_ITEM_ID_PROPERTY_NAME: item["id"]})
|
||||
|
||||
result = await vector_collection.upsert(item)
|
||||
|
||||
mock_container_proxy.upsert_item.assert_called_once_with(item)
|
||||
assert result == item["id"]
|
||||
|
||||
|
||||
@patch("azure.cosmos.aio.ContainerProxy")
|
||||
async def test_azure_cosmos_db_no_sql_upsert_without_id(
|
||||
mock_container_proxy,
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type_with_key_as_key_field,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test the upsert of a document in a cosmos DB NoSQL collection where the name of the key field is 'key'."""
|
||||
item = {"content": "test_content", "vector": [1.0, 2.0, 3.0], "key": "test_key"}
|
||||
item_with_id = {"content": "test_content", "vector": [1.0, 2.0, 3.0], COSMOS_ITEM_ID_PROPERTY_NAME: "test_key"}
|
||||
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type_with_key_as_key_field,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
vector_collection._get_container_proxy = AsyncMock(return_value=mock_container_proxy)
|
||||
|
||||
mock_container_proxy.upsert_item = AsyncMock(return_value={COSMOS_ITEM_ID_PROPERTY_NAME: item["key"]})
|
||||
|
||||
result = await vector_collection.upsert(item)
|
||||
|
||||
mock_container_proxy.upsert_item.assert_called_once_with(item_with_id)
|
||||
assert result == item["key"]
|
||||
|
||||
|
||||
@patch("azure.cosmos.aio.ContainerProxy")
|
||||
async def test_azure_cosmos_db_no_sql_get(
|
||||
mock_container_proxy,
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test the retrieval of a document from a cosmos DB NoSQL collection."""
|
||||
vector_collection: CosmosNoSqlCollection[str, record_type] = CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
vector_collection._get_container_proxy = AsyncMock(return_value=mock_container_proxy)
|
||||
|
||||
get_results = MagicMock(spec=AsyncGenerator)
|
||||
get_results.__aiter__.return_value = [{"content": "test_content", "id": "test_id"}]
|
||||
mock_container_proxy.query_items.return_value = get_results
|
||||
|
||||
record = await vector_collection.get("test_id")
|
||||
assert isinstance(record, record_type)
|
||||
assert record.content == "test_content"
|
||||
assert record.id == "test_id"
|
||||
|
||||
|
||||
@patch("azure.cosmos.aio.ContainerProxy")
|
||||
async def test_azure_cosmos_db_no_sql_get_without_id(
|
||||
mock_container_proxy,
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type_with_key_as_key_field,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test the retrieval of a document from a cosmos DB NoSQL collection where the name of the key field is 'key'."""
|
||||
vector_collection = CosmosNoSqlCollection(
|
||||
record_type=record_type_with_key_as_key_field,
|
||||
collection_name=collection_name,
|
||||
)
|
||||
|
||||
vector_collection._get_container_proxy = AsyncMock(return_value=mock_container_proxy)
|
||||
|
||||
get_results = MagicMock(spec=AsyncGenerator)
|
||||
get_results.__aiter__.return_value = [
|
||||
{"content": "test_content", "vector": [1.0, 2.0, 3.0], COSMOS_ITEM_ID_PROPERTY_NAME: "test_key"}
|
||||
]
|
||||
mock_container_proxy.query_items.return_value = get_results
|
||||
|
||||
record = await vector_collection.get("test_key")
|
||||
assert isinstance(record, record_type_with_key_as_key_field)
|
||||
assert record.content == "test_content"
|
||||
assert record.vector == [1.0, 2.0, 3.0]
|
||||
assert record.key == "test_key"
|
||||
|
||||
|
||||
@patch.object(CosmosClient, "close", return_value=None)
|
||||
async def test_client_is_closed(
|
||||
mock_cosmos_client_close,
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
record_type,
|
||||
collection_name: str,
|
||||
) -> None:
|
||||
"""Test the close method of an AzureCosmosDBNoSQLCollection object."""
|
||||
async with CosmosNoSqlCollection(
|
||||
record_type=record_type,
|
||||
collection_name=collection_name,
|
||||
) as collection:
|
||||
assert collection.cosmos_client is not None
|
||||
|
||||
mock_cosmos_client_close.assert_called()
|
||||
+101
@@ -0,0 +1,101 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from azure.cosmos.aio import CosmosClient
|
||||
|
||||
from semantic_kernel.connectors.azure_cosmos_db import CosmosNoSqlCollection, CosmosNoSqlStore
|
||||
from semantic_kernel.exceptions import VectorStoreInitializationException
|
||||
|
||||
|
||||
def test_azure_cosmos_db_no_sql_store_init(
|
||||
clear_azure_cosmos_db_no_sql_env,
|
||||
database_name: str,
|
||||
url: str,
|
||||
key: str,
|
||||
) -> None:
|
||||
"""Test the initialization of an AzureCosmosDBNoSQLStore object."""
|
||||
vector_store = CosmosNoSqlStore(url=url, key=key, database_name=database_name)
|
||||
|
||||
assert vector_store is not None
|
||||
assert vector_store.database_name == database_name
|
||||
assert vector_store.cosmos_client is not None
|
||||
assert vector_store.create_database is False
|
||||
|
||||
|
||||
def test_azure_cosmos_db_no_sql_store_init_env(azure_cosmos_db_no_sql_unit_test_env) -> None:
|
||||
"""Test the initialization of an AzureCosmosDBNoSQLStore object with environment variables."""
|
||||
vector_store = CosmosNoSqlStore()
|
||||
|
||||
assert vector_store is not None
|
||||
assert vector_store.database_name == azure_cosmos_db_no_sql_unit_test_env["AZURE_COSMOS_DB_NO_SQL_DATABASE_NAME"]
|
||||
assert vector_store.cosmos_client is not None
|
||||
assert vector_store.create_database is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exclude_list", [["AZURE_COSMOS_DB_NO_SQL_URL"]], indirect=True)
|
||||
def test_azure_cosmos_db_no_sql_store_init_no_url(
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
) -> None:
|
||||
"""Test the initialization of an AzureCosmosDBNoSQLStore object with missing URL."""
|
||||
with pytest.raises(VectorStoreInitializationException):
|
||||
CosmosNoSqlStore(env_file_path="fake_path")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("exclude_list", [["AZURE_COSMOS_DB_NO_SQL_DATABASE_NAME"]], indirect=True)
|
||||
def test_azure_cosmos_db_no_sql_store_init_no_database_name(
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
) -> None:
|
||||
"""Test the initialization of an AzureCosmosDBNoSQLStore object with missing database name."""
|
||||
with pytest.raises(
|
||||
VectorStoreInitializationException, match="The name of the Azure Cosmos DB NoSQL database is missing."
|
||||
):
|
||||
CosmosNoSqlStore(env_file_path="fake_path")
|
||||
|
||||
|
||||
def test_azure_cosmos_db_no_sql_store_invalid_settings(
|
||||
clear_azure_cosmos_db_no_sql_env,
|
||||
) -> None:
|
||||
"""Test the initialization of an AzureCosmosDBNoSQLStore object with invalid settings."""
|
||||
with pytest.raises(VectorStoreInitializationException, match="Failed to validate Azure Cosmos DB NoSQL settings."):
|
||||
CosmosNoSqlStore(url="invalid_url")
|
||||
|
||||
|
||||
@patch.object(CosmosNoSqlCollection, "__init__", return_value=None)
|
||||
def test_azure_cosmos_db_no_sql_store_get_collection(
|
||||
mock_azure_cosmos_db_no_sql_collection_init,
|
||||
azure_cosmos_db_no_sql_unit_test_env,
|
||||
collection_name: str,
|
||||
record_type,
|
||||
) -> None:
|
||||
"""Test the get_collection method of an AzureCosmosDBNoSQLStore object."""
|
||||
vector_store = CosmosNoSqlStore()
|
||||
|
||||
collection = vector_store.get_collection(collection_name=collection_name, record_type=record_type)
|
||||
|
||||
assert collection is not None
|
||||
mock_azure_cosmos_db_no_sql_collection_init.assert_called_once_with(
|
||||
record_type=record_type,
|
||||
definition=None,
|
||||
collection_name=collection_name,
|
||||
database_name=azure_cosmos_db_no_sql_unit_test_env["AZURE_COSMOS_DB_NO_SQL_DATABASE_NAME"],
|
||||
embedding_generator=None,
|
||||
url=azure_cosmos_db_no_sql_unit_test_env["AZURE_COSMOS_DB_NO_SQL_URL"],
|
||||
key=azure_cosmos_db_no_sql_unit_test_env["AZURE_COSMOS_DB_NO_SQL_KEY"],
|
||||
cosmos_client=vector_store.cosmos_client,
|
||||
partition_key=None,
|
||||
create_database=vector_store.create_database,
|
||||
env_file_path=None,
|
||||
env_file_encoding=None,
|
||||
)
|
||||
|
||||
|
||||
@patch.object(CosmosClient, "close", return_value=None)
|
||||
async def test_client_is_closed(mock_cosmos_client_close, azure_cosmos_db_no_sql_unit_test_env) -> None:
|
||||
"""Test the close method of an AzureCosmosDBNoSQLStore object."""
|
||||
async with CosmosNoSqlStore() as vector_store:
|
||||
assert vector_store.cosmos_client is not None
|
||||
|
||||
mock_cosmos_client_close.assert_called()
|
||||
@@ -0,0 +1,308 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from _pytest.mark.structures import ParameterSet
|
||||
from pytest import fixture, param
|
||||
|
||||
from semantic_kernel.exceptions.vector_store_exceptions import VectorStoreOperationException
|
||||
|
||||
|
||||
@fixture()
|
||||
def mongodb_atlas_unit_test_env(monkeypatch, exclude_list, override_env_param_dict):
|
||||
"""Fixture to set environment variables for MongoDB Atlas Unit Tests."""
|
||||
if exclude_list is None:
|
||||
exclude_list = []
|
||||
|
||||
if override_env_param_dict is None:
|
||||
override_env_param_dict = {}
|
||||
|
||||
env_vars = {"MONGODB_ATLAS_CONNECTION_STRING": "mongodb://test", "MONGODB_ATLAS_DATABASE_NAME": "test-database"}
|
||||
|
||||
env_vars.update(override_env_param_dict)
|
||||
|
||||
for key, value in env_vars.items():
|
||||
if key not in exclude_list:
|
||||
monkeypatch.setenv(key, value)
|
||||
else:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
return env_vars
|
||||
|
||||
|
||||
@fixture
|
||||
def postgres_unit_test_env(monkeypatch, exclude_list, override_env_param_dict):
|
||||
"""Fixture to set environment variables for Postgres connector."""
|
||||
if exclude_list is None:
|
||||
exclude_list = []
|
||||
|
||||
if override_env_param_dict is None:
|
||||
override_env_param_dict = {}
|
||||
|
||||
env_vars = {"POSTGRES_CONNECTION_STRING": "host=localhost port=5432 dbname=postgres user=testuser password=example"}
|
||||
|
||||
env_vars.update(override_env_param_dict)
|
||||
|
||||
for key, value in env_vars.items():
|
||||
if key not in exclude_list:
|
||||
monkeypatch.setenv(key, value)
|
||||
else:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
return env_vars
|
||||
|
||||
|
||||
@fixture
|
||||
def qdrant_unit_test_env(monkeypatch, exclude_list, override_env_param_dict):
|
||||
"""Fixture to set environment variables for QdrantConnector."""
|
||||
if exclude_list is None:
|
||||
exclude_list = []
|
||||
|
||||
if override_env_param_dict is None:
|
||||
override_env_param_dict = {}
|
||||
|
||||
env_vars = {"QDRANT_LOCATION": "http://localhost:6333"}
|
||||
|
||||
env_vars.update(override_env_param_dict)
|
||||
|
||||
for key, value in env_vars.items():
|
||||
if key not in exclude_list:
|
||||
monkeypatch.setenv(key, value)
|
||||
else:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
return env_vars
|
||||
|
||||
|
||||
@fixture
|
||||
def redis_unit_test_env(monkeypatch, exclude_list, override_env_param_dict):
|
||||
"""Fixture to set environment variables for Redis."""
|
||||
if exclude_list is None:
|
||||
exclude_list = []
|
||||
|
||||
if override_env_param_dict is None:
|
||||
override_env_param_dict = {}
|
||||
|
||||
env_vars = {"REDIS_CONNECTION_STRING": "redis://localhost:6379"}
|
||||
|
||||
env_vars.update(override_env_param_dict)
|
||||
|
||||
for key, value in env_vars.items():
|
||||
if key not in exclude_list:
|
||||
monkeypatch.setenv(key, value)
|
||||
else:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
return env_vars
|
||||
|
||||
|
||||
@fixture
|
||||
def pinecone_unit_test_env(monkeypatch, exclude_list, override_env_param_dict):
|
||||
"""Fixture to set environment variables for Pinecone."""
|
||||
if exclude_list is None:
|
||||
exclude_list = []
|
||||
|
||||
if override_env_param_dict is None:
|
||||
override_env_param_dict = {}
|
||||
|
||||
env_vars = {"PINECONE_API_KEY": "test_key"}
|
||||
|
||||
env_vars.update(override_env_param_dict)
|
||||
|
||||
for key, value in env_vars.items():
|
||||
if key not in exclude_list:
|
||||
monkeypatch.setenv(key, value)
|
||||
else:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
return env_vars
|
||||
|
||||
|
||||
@fixture
|
||||
def sql_server_unit_test_env(monkeypatch, exclude_list, override_env_param_dict):
|
||||
"""Fixture to set environment variables for SQL Server."""
|
||||
if exclude_list is None:
|
||||
exclude_list = []
|
||||
|
||||
if override_env_param_dict is None:
|
||||
override_env_param_dict = {}
|
||||
|
||||
env_vars = {
|
||||
"SQL_SERVER_CONNECTION_STRING": "Driver={ODBC Driver 18 for SQL Server};Server=localhost;Database=testdb;User Id=testuser;Password=example;" # noqa: E501
|
||||
}
|
||||
|
||||
env_vars.update(override_env_param_dict)
|
||||
|
||||
for key, value in env_vars.items():
|
||||
if key not in exclude_list:
|
||||
monkeypatch.setenv(key, value)
|
||||
else:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
return env_vars
|
||||
|
||||
|
||||
def filter_lambda_list(store: str) -> list[ParameterSet]:
|
||||
"""Fixture to provide a list of filter lambdas for testing."""
|
||||
sets = [
|
||||
(
|
||||
lambda x: x.content == "value",
|
||||
{
|
||||
"ai_search": "content eq 'value'",
|
||||
},
|
||||
"equal with string",
|
||||
),
|
||||
(
|
||||
lambda x: x.id == 0,
|
||||
{
|
||||
"ai_search": "id eq 0",
|
||||
},
|
||||
"equal with int",
|
||||
),
|
||||
(
|
||||
lambda x: x.content != "value",
|
||||
{
|
||||
"ai_search": "content ne 'value'",
|
||||
},
|
||||
"not equal",
|
||||
),
|
||||
(
|
||||
lambda x: x.id > 0,
|
||||
{
|
||||
"ai_search": "id gt 0",
|
||||
},
|
||||
"greater than",
|
||||
),
|
||||
(
|
||||
lambda x: x.id >= 0,
|
||||
{
|
||||
"ai_search": "id ge 0",
|
||||
},
|
||||
"greater than or equal",
|
||||
),
|
||||
(
|
||||
lambda x: x.id == +0,
|
||||
{
|
||||
"ai_search": "id eq +0",
|
||||
},
|
||||
"equal with explicit positive",
|
||||
),
|
||||
(
|
||||
lambda x: x.id < 0,
|
||||
{
|
||||
"ai_search": "id lt 0",
|
||||
},
|
||||
"less than",
|
||||
),
|
||||
(
|
||||
lambda x: x.id <= 0,
|
||||
{
|
||||
"ai_search": "id le 0",
|
||||
},
|
||||
"less than or equal",
|
||||
),
|
||||
(
|
||||
lambda x: -10 <= x.id <= 0,
|
||||
{
|
||||
"ai_search": "(-10 le id and id le 0)",
|
||||
},
|
||||
"between inclusive",
|
||||
),
|
||||
(
|
||||
lambda x: -10 < x.id < 0,
|
||||
{
|
||||
"ai_search": "(-10 lt id and id lt 0)",
|
||||
},
|
||||
"between exclusive",
|
||||
),
|
||||
(
|
||||
lambda x: x.content == "value" and x.id == 0,
|
||||
{
|
||||
"ai_search": "(content eq 'value' and id eq 0)",
|
||||
},
|
||||
"and",
|
||||
),
|
||||
(
|
||||
lambda x: x.content == "value" or x.id == 0,
|
||||
{
|
||||
"ai_search": "(content eq 'value' or id eq 0)",
|
||||
},
|
||||
"or",
|
||||
),
|
||||
(
|
||||
lambda x: not x.content,
|
||||
{
|
||||
"ai_search": "not content",
|
||||
},
|
||||
"not with truthy",
|
||||
),
|
||||
(
|
||||
lambda x: not (x.content == "value"), # noqa: SIM201
|
||||
{
|
||||
"ai_search": "not content eq 'value'",
|
||||
},
|
||||
"not with equal",
|
||||
),
|
||||
(
|
||||
lambda x: not (x.content != "value"), # noqa: SIM202
|
||||
{
|
||||
"ai_search": "not content ne 'value'",
|
||||
},
|
||||
"not with not equal",
|
||||
),
|
||||
(
|
||||
lambda x: "value" in x.content,
|
||||
{
|
||||
"ai_search": "search.ismatch('value', 'content')",
|
||||
},
|
||||
"contains",
|
||||
),
|
||||
(
|
||||
lambda x: "value" not in x.content,
|
||||
{
|
||||
"ai_search": "not search.ismatch('value', 'content')",
|
||||
},
|
||||
"not contains",
|
||||
),
|
||||
(
|
||||
lambda x: (x.id > 0 and x.id < 3) or (x.id > 7 and x.id < 10),
|
||||
{
|
||||
"ai_search": "((id gt 0 and id lt 3) or (id gt 7 and id lt 10))",
|
||||
},
|
||||
"complex",
|
||||
),
|
||||
(
|
||||
lambda x: x.unknown_field == "value",
|
||||
{
|
||||
"ai_search": VectorStoreOperationException,
|
||||
},
|
||||
"fail unknown field",
|
||||
),
|
||||
(
|
||||
lambda x: any(x == "a" for x in x.content),
|
||||
{
|
||||
"ai_search": NotImplementedError,
|
||||
},
|
||||
"comprehension",
|
||||
),
|
||||
(
|
||||
lambda x: ~x.id,
|
||||
{
|
||||
"ai_search": NotImplementedError,
|
||||
},
|
||||
"invert",
|
||||
),
|
||||
(
|
||||
lambda x: constant, # noqa: F821
|
||||
{
|
||||
"ai_search": NotImplementedError,
|
||||
},
|
||||
"constant",
|
||||
),
|
||||
(
|
||||
lambda x: x.content.city == "Seattle",
|
||||
{
|
||||
"ai_search": "content/city eq 'Seattle'",
|
||||
},
|
||||
"nested property",
|
||||
),
|
||||
]
|
||||
return [param(s[0], s[1][store], id=s[2]) for s in sets if store in s[1]]
|
||||
@@ -0,0 +1,37 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from pymongo import AsyncMongoClient
|
||||
from pymongo.asynchronous.collection import AsyncCollection
|
||||
from pymongo.asynchronous.database import AsyncDatabase
|
||||
|
||||
BASE_PATH = "pymongo.asynchronous.mongo_client.AsyncMongoClient"
|
||||
DATABASE_PATH = "pymongo.asynchronous.database.AsyncDatabase"
|
||||
COLLECTION_PATH = "pymongo.asynchronous.collection.AsyncCollection"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_mongo_client():
|
||||
with patch(BASE_PATH, spec=AsyncMongoClient) as mock:
|
||||
yield mock
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_get_database(mock_mongo_client):
|
||||
with (
|
||||
patch(DATABASE_PATH, spec=AsyncDatabase) as mock_db,
|
||||
patch.object(mock_mongo_client, "get_database", new_callable=lambda: mock_db) as mock,
|
||||
):
|
||||
yield mock
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_get_collection(mock_get_database):
|
||||
with (
|
||||
patch(COLLECTION_PATH, spec=AsyncCollection) as mock_collection,
|
||||
patch.object(mock_get_database, "get_collection", new_callable=lambda: mock_collection) as mock,
|
||||
):
|
||||
yield mock
|
||||
@@ -0,0 +1,93 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
from pymongo import AsyncMongoClient
|
||||
from pymongo.asynchronous.cursor import AsyncCursor
|
||||
from pymongo.results import UpdateResult
|
||||
from pytest import mark, raises
|
||||
|
||||
from semantic_kernel.connectors.mongodb import DEFAULT_DB_NAME, DEFAULT_SEARCH_INDEX_NAME, MongoDBAtlasCollection
|
||||
from semantic_kernel.exceptions.vector_store_exceptions import VectorStoreInitializationException
|
||||
|
||||
|
||||
def test_mongodb_atlas_collection_initialization(mongodb_atlas_unit_test_env, definition, mock_mongo_client):
|
||||
collection = MongoDBAtlasCollection(
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
collection_name="test_collection",
|
||||
mongo_client=mock_mongo_client,
|
||||
)
|
||||
assert collection.mongo_client is not None
|
||||
assert isinstance(collection.mongo_client, AsyncMongoClient)
|
||||
|
||||
|
||||
@mark.parametrize("exclude_list", [["MONGODB_ATLAS_CONNECTION_STRING"]], indirect=True)
|
||||
def test_mongodb_atlas_collection_initialization_fail(mongodb_atlas_unit_test_env, definition):
|
||||
with raises(VectorStoreInitializationException):
|
||||
MongoDBAtlasCollection(
|
||||
collection_name="test_collection",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
)
|
||||
|
||||
|
||||
@mark.parametrize("exclude_list", [["MONGODB_ATLAS_DATABASE_NAME", "MONGODB_ATLAS_INDEX_NAME"]], indirect=True)
|
||||
def test_mongodb_atlas_collection_initialization_defaults(mongodb_atlas_unit_test_env, definition):
|
||||
collection = MongoDBAtlasCollection(
|
||||
collection_name="test_collection",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
)
|
||||
assert collection.database_name == DEFAULT_DB_NAME
|
||||
assert collection.index_name == DEFAULT_SEARCH_INDEX_NAME
|
||||
|
||||
|
||||
async def test_mongodb_atlas_collection_upsert(mongodb_atlas_unit_test_env, definition, mock_get_collection):
|
||||
collection = MongoDBAtlasCollection(
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
collection_name="test_collection",
|
||||
)
|
||||
with patch.object(collection, "_get_collection", new=mock_get_collection) as mock_get:
|
||||
result_mock = AsyncMock(spec=UpdateResult)
|
||||
result_mock.upserted_ids = {0: "test_id"}
|
||||
mock_get.return_value.bulk_write.return_value = result_mock
|
||||
result = await collection._inner_upsert([{"_id": "test_id", "data": "test_data"}])
|
||||
assert result == ["test_id"]
|
||||
|
||||
|
||||
async def test_mongodb_atlas_collection_get(mongodb_atlas_unit_test_env, definition, mock_get_collection):
|
||||
collection = MongoDBAtlasCollection(
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
collection_name="test_collection",
|
||||
)
|
||||
with patch.object(collection, "_get_collection", new=mock_get_collection) as mock_get:
|
||||
result_mock = AsyncMock(spec=AsyncCursor)
|
||||
result_mock.to_list.return_value = [{"_id": "test_id", "data": "test_data"}]
|
||||
mock_get.return_value.find.return_value = result_mock
|
||||
result = await collection._inner_get(["test_id"])
|
||||
assert result == [{"_id": "test_id", "data": "test_data"}]
|
||||
|
||||
|
||||
async def test_mongodb_atlas_collection_delete(mongodb_atlas_unit_test_env, definition, mock_get_collection):
|
||||
collection = MongoDBAtlasCollection(
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
collection_name="test_collection",
|
||||
)
|
||||
with patch.object(collection, "_get_collection", new=mock_get_collection) as mock_get:
|
||||
await collection._inner_delete(["test_id"])
|
||||
mock_get.return_value.delete_many.assert_called_with({"_id": {"$in": ["test_id"]}})
|
||||
|
||||
|
||||
async def test_mongodb_atlas_collection_collection_exists(mongodb_atlas_unit_test_env, definition, mock_get_database):
|
||||
collection = MongoDBAtlasCollection(
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
collection_name="test_collection",
|
||||
)
|
||||
with patch.object(collection, "_get_database", new=mock_get_database) as mock_get:
|
||||
mock_get.return_value.list_collection_names.return_value = ["test_collection"]
|
||||
assert await collection.collection_exists()
|
||||
@@ -0,0 +1,30 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
|
||||
from pymongo import AsyncMongoClient
|
||||
|
||||
from semantic_kernel.connectors.mongodb import MongoDBAtlasCollection, MongoDBAtlasStore
|
||||
|
||||
|
||||
def test_mongodb_atlas_store_initialization(mongodb_atlas_unit_test_env):
|
||||
store = MongoDBAtlasStore()
|
||||
assert store.mongo_client is not None
|
||||
assert isinstance(store.mongo_client, AsyncMongoClient)
|
||||
|
||||
|
||||
def test_mongodb_atlas_store_get_collection(mongodb_atlas_unit_test_env, definition):
|
||||
store = MongoDBAtlasStore()
|
||||
collection = store.get_collection(
|
||||
collection_name="test_collection",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
)
|
||||
assert collection is not None
|
||||
assert isinstance(collection, MongoDBAtlasCollection)
|
||||
|
||||
|
||||
async def test_mongodb_atlas_store_list_collection_names(mongodb_atlas_unit_test_env, mock_mongo_client):
|
||||
store = MongoDBAtlasStore(mongo_client=mock_mongo_client, database_name="test_db")
|
||||
store.mongo_client.get_database().list_collection_names.return_value = ["test_collection"]
|
||||
result = await store.list_collection_names()
|
||||
assert result == ["test_collection"]
|
||||
@@ -0,0 +1,450 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import numpy as np
|
||||
from azure.search.documents.aio import SearchClient
|
||||
from azure.search.documents.indexes.aio import SearchIndexClient
|
||||
from pytest import fixture, mark, param, raises
|
||||
|
||||
from semantic_kernel.connectors.ai.embedding_generator_base import EmbeddingGeneratorBase
|
||||
from semantic_kernel.connectors.azure_ai_search import (
|
||||
AzureAISearchCollection,
|
||||
AzureAISearchSettings,
|
||||
AzureAISearchStore,
|
||||
_definition_to_azure_ai_search_index,
|
||||
_get_search_index_client,
|
||||
_resolve_credential,
|
||||
)
|
||||
from semantic_kernel.exceptions import (
|
||||
ServiceInitializationError,
|
||||
VectorStoreInitializationException,
|
||||
VectorStoreOperationException,
|
||||
)
|
||||
from semantic_kernel.utils.list_handler import desync_list
|
||||
from tests.unit.connectors.memory.conftest import filter_lambda_list
|
||||
|
||||
BASE_PATH_SEARCH_CLIENT = "azure.search.documents.aio.SearchClient"
|
||||
BASE_PATH_INDEX_CLIENT = "azure.search.documents.indexes.aio.SearchIndexClient"
|
||||
|
||||
|
||||
@fixture
|
||||
def vector_store(azure_ai_search_unit_test_env):
|
||||
"""Fixture to instantiate AzureCognitiveSearchMemoryStore with basic configuration."""
|
||||
return AzureAISearchStore()
|
||||
|
||||
|
||||
@fixture
|
||||
def mock_ensure_collection_exists():
|
||||
"""Fixture to patch 'SearchIndexClient' and its 'create_index' method."""
|
||||
with patch(f"{BASE_PATH_INDEX_CLIENT}.create_index") as mock_create_index:
|
||||
yield mock_create_index
|
||||
|
||||
|
||||
@fixture
|
||||
def mock_ensure_collection_deleted():
|
||||
"""Fixture to patch 'SearchIndexClient' and its 'create_index' method."""
|
||||
with patch(f"{BASE_PATH_INDEX_CLIENT}.delete_index") as mock_delete_index:
|
||||
yield mock_delete_index
|
||||
|
||||
|
||||
@fixture
|
||||
def mock_list_collection_names():
|
||||
"""Fixture to patch 'SearchIndexClient' and its 'create_index' method."""
|
||||
with patch(f"{BASE_PATH_INDEX_CLIENT}.list_index_names") as mock_list_index_names:
|
||||
# Setup the mock to return a specific SearchIndex instance when called
|
||||
mock_list_index_names.return_value = desync_list(["test"])
|
||||
yield mock_list_index_names
|
||||
|
||||
|
||||
@fixture
|
||||
def mock_upsert():
|
||||
with patch(f"{BASE_PATH_SEARCH_CLIENT}.merge_or_upload_documents") as mock_merge_or_upload_documents:
|
||||
from azure.search.documents.models import IndexingResult
|
||||
|
||||
result = MagicMock(spec=IndexingResult)
|
||||
result.key = "id1"
|
||||
mock_merge_or_upload_documents.return_value = [result]
|
||||
yield mock_merge_or_upload_documents
|
||||
|
||||
|
||||
@fixture
|
||||
def mock_get():
|
||||
with patch(f"{BASE_PATH_SEARCH_CLIENT}.get_document") as mock_get_document:
|
||||
mock_get_document.return_value = {"id": "id1", "content": "content", "vector": [1.0, 2.0, 3.0]}
|
||||
yield mock_get_document
|
||||
|
||||
|
||||
@fixture
|
||||
def mock_search():
|
||||
async def iter_search_results(*args, **kwargs):
|
||||
yield {"id": "id1", "content": "content", "vector": [1.0, 2.0, 3.0]}
|
||||
await asyncio.sleep(0.0)
|
||||
|
||||
with patch(f"{BASE_PATH_SEARCH_CLIENT}.search") as mock_search:
|
||||
mock_search.side_effect = iter_search_results
|
||||
yield mock_search
|
||||
|
||||
|
||||
@fixture
|
||||
def mock_delete():
|
||||
with patch(f"{BASE_PATH_SEARCH_CLIENT}.delete_documents") as mock_delete_documents:
|
||||
yield mock_delete_documents
|
||||
|
||||
|
||||
@fixture
|
||||
def collection(azure_ai_search_unit_test_env, definition):
|
||||
return AzureAISearchCollection(record_type=dict, definition=definition)
|
||||
|
||||
|
||||
async def test_init(azure_ai_search_unit_test_env, definition):
|
||||
async with AzureAISearchCollection(record_type=dict, definition=definition) as collection:
|
||||
assert collection is not None
|
||||
assert collection.record_type is dict
|
||||
assert collection.definition == definition
|
||||
assert collection.collection_name == "test-index-name"
|
||||
assert collection.search_index_client is not None
|
||||
assert collection.search_client is not None
|
||||
|
||||
|
||||
def test_init_with_type(azure_ai_search_unit_test_env, record_type):
|
||||
collection = AzureAISearchCollection(record_type=record_type)
|
||||
assert collection is not None
|
||||
assert collection.record_type is record_type
|
||||
assert collection.collection_name == "test-index-name"
|
||||
assert collection.search_index_client is not None
|
||||
assert collection.search_client is not None
|
||||
|
||||
|
||||
@mark.parametrize("exclude_list", [["AZURE_AI_SEARCH_ENDPOINT"]], indirect=True)
|
||||
def test_init_endpoint_fail(azure_ai_search_unit_test_env, definition):
|
||||
with raises(VectorStoreInitializationException):
|
||||
AzureAISearchCollection(record_type=dict, definition=definition, env_file_path="test.env")
|
||||
|
||||
|
||||
@mark.parametrize("exclude_list", [["AZURE_AI_SEARCH_INDEX_NAME"]], indirect=True)
|
||||
def test_init_index_fail(azure_ai_search_unit_test_env, definition):
|
||||
with raises(VectorStoreInitializationException):
|
||||
AzureAISearchCollection(record_type=dict, definition=definition, env_file_path="test.env")
|
||||
|
||||
|
||||
def test_init_with_clients(azure_ai_search_unit_test_env, definition):
|
||||
search_index_client = MagicMock(spec=SearchIndexClient)
|
||||
search_client = MagicMock(spec=SearchClient)
|
||||
search_client._index_name = "test-index-name"
|
||||
|
||||
collection = AzureAISearchCollection(
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
search_index_client=search_index_client,
|
||||
search_client=search_client,
|
||||
)
|
||||
assert collection is not None
|
||||
assert collection.record_type is dict
|
||||
assert collection.definition == definition
|
||||
assert collection.collection_name == "test-index-name"
|
||||
assert collection.search_index_client == search_index_client
|
||||
assert collection.search_client == search_client
|
||||
|
||||
|
||||
def test_init_with_search_index_client(azure_ai_search_unit_test_env, definition):
|
||||
search_index_client = MagicMock(spec=SearchIndexClient)
|
||||
with patch("semantic_kernel.connectors.azure_ai_search._get_search_client") as get_search_client:
|
||||
search_client = MagicMock(spec=SearchClient)
|
||||
get_search_client.return_value = search_client
|
||||
|
||||
collection = AzureAISearchCollection(
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
collection_name="test",
|
||||
search_index_client=search_index_client,
|
||||
)
|
||||
assert collection is not None
|
||||
assert collection.record_type is dict
|
||||
assert collection.definition == definition
|
||||
assert collection.collection_name == "test"
|
||||
assert collection.search_index_client == search_index_client
|
||||
assert collection.search_client == search_client
|
||||
|
||||
|
||||
@mark.parametrize("exclude_list", [["AZURE_AI_SEARCH_INDEX_NAME"]], indirect=True)
|
||||
def test_init_with_search_index_client_fail(azure_ai_search_unit_test_env, definition):
|
||||
search_index_client = MagicMock(spec=SearchIndexClient)
|
||||
with raises(VectorStoreInitializationException):
|
||||
AzureAISearchCollection(
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
search_index_client=search_index_client,
|
||||
env_file_path="test.env",
|
||||
)
|
||||
|
||||
|
||||
async def test_upsert(collection, mock_upsert):
|
||||
ids = await collection._inner_upsert({"id": "id1", "name": "test"})
|
||||
assert ids[0] == "id1"
|
||||
|
||||
ids = await collection.upsert(records={"id": "id1", "content": "content", "vector": [1.0, 2.0, 3.0]})
|
||||
assert ids == "id1"
|
||||
|
||||
|
||||
async def test_get(collection, mock_get):
|
||||
records = await collection._inner_get(["id1"])
|
||||
assert records is not None
|
||||
|
||||
records = await collection.get("id1")
|
||||
assert records is not None
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"order_by, ordering",
|
||||
[
|
||||
param("id", ["id"], id="single id"),
|
||||
param({"id": True}, ["id"], id="ascending id"),
|
||||
param({"id": False}, ["id desc"], id="descending id"),
|
||||
param(["id"], ["id"], id="ascending id list"),
|
||||
param(["id", "content"], ["id", "content"], id="multiple"),
|
||||
param([{"id": True}, {"content": False}], ["id", "content desc"], id="multiple desc"),
|
||||
param(["id", {"content": False}], ["id", "content desc"], id="multiple mix"),
|
||||
],
|
||||
)
|
||||
async def test_get_without_key(collection, mock_get, mock_search, order_by, ordering):
|
||||
records = await collection.get(top=10, order_by=order_by)
|
||||
assert records is not None
|
||||
mock_search.assert_called_once_with(
|
||||
search_text="*",
|
||||
top=10,
|
||||
skip=0,
|
||||
select=["id", "content"],
|
||||
order_by=ordering,
|
||||
)
|
||||
|
||||
|
||||
async def test_delete(collection, mock_delete):
|
||||
await collection._inner_delete(["id1"])
|
||||
|
||||
|
||||
async def test_collection_exists(collection, mock_list_collection_names):
|
||||
await collection.collection_exists()
|
||||
|
||||
|
||||
async def test_ensure_collection_deleted(collection, mock_ensure_collection_deleted):
|
||||
await collection.ensure_collection_deleted()
|
||||
|
||||
|
||||
@mark.parametrize("distance_function", [("cosine_distance")])
|
||||
async def test_create_index_from_index(collection, mock_ensure_collection_exists):
|
||||
from azure.search.documents.indexes.models import SearchIndex
|
||||
|
||||
index = MagicMock(spec=SearchIndex)
|
||||
await collection.ensure_collection_exists(index=index)
|
||||
|
||||
|
||||
@mark.parametrize("distance_function", [("cosine_distance")])
|
||||
async def test_create_index_from_definition(collection, mock_ensure_collection_exists):
|
||||
from azure.search.documents.indexes.models import SearchIndex
|
||||
|
||||
with patch(
|
||||
"semantic_kernel.connectors.azure_ai_search._definition_to_azure_ai_search_index",
|
||||
return_value=MagicMock(spec=SearchIndex),
|
||||
):
|
||||
await collection.ensure_collection_exists()
|
||||
|
||||
|
||||
async def test_create_index_from_index_fail(collection, mock_ensure_collection_exists):
|
||||
index = Mock()
|
||||
with raises(VectorStoreOperationException):
|
||||
await collection.ensure_collection_exists(index=index)
|
||||
|
||||
|
||||
@mark.parametrize("distance_function", [("cosine_distance")])
|
||||
def test_definition_to_azure_ai_search_index(definition):
|
||||
index = _definition_to_azure_ai_search_index("test", definition)
|
||||
assert index is not None
|
||||
assert index.name == "test"
|
||||
assert len(index.fields) == 3
|
||||
|
||||
|
||||
@mark.parametrize("exclude_list", [["AZURE_AI_SEARCH_ENDPOINT"]], indirect=True)
|
||||
async def test_vector_store_fail(azure_ai_search_unit_test_env):
|
||||
with raises(VectorStoreInitializationException):
|
||||
AzureAISearchStore(env_file_path="test.env")
|
||||
|
||||
|
||||
async def test_vector_store_list_collection_names(vector_store, mock_list_collection_names):
|
||||
assert vector_store.search_index_client is not None
|
||||
collection_names = await vector_store.list_collection_names()
|
||||
assert collection_names == ["test"]
|
||||
mock_list_collection_names.assert_called_once()
|
||||
|
||||
|
||||
async def test_vector_store_collection_existss(vector_store, mock_list_collection_names):
|
||||
assert vector_store.search_index_client is not None
|
||||
exists = await vector_store.collection_exists("test")
|
||||
assert exists
|
||||
mock_list_collection_names.assert_called_once()
|
||||
|
||||
|
||||
async def test_vector_store_ensure_collection_deleted(vector_store, mock_ensure_collection_deleted):
|
||||
assert vector_store.search_index_client is not None
|
||||
await vector_store.ensure_collection_deleted("test")
|
||||
mock_ensure_collection_deleted.assert_called_once()
|
||||
|
||||
|
||||
def test_get_collection(vector_store, definition):
|
||||
collection = vector_store.get_collection(
|
||||
collection_name="test",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
)
|
||||
assert collection is not None
|
||||
assert collection.collection_name == "test"
|
||||
assert collection.search_index_client == vector_store.search_index_client
|
||||
assert collection.search_client is not None
|
||||
assert collection.search_endpoint == vector_store.search_endpoint
|
||||
assert collection.search_credential == vector_store.search_credential
|
||||
|
||||
|
||||
def test_get_collection_with_provided_search_index_client(azure_ai_search_unit_test_env, definition):
|
||||
"""Test that get_collection works when AzureAISearchStore is created with a pre-built search_index_client.
|
||||
|
||||
When search_index_client is provided directly, search_endpoint and search_credential
|
||||
are not resolved at store creation time. get_collection() should still succeed
|
||||
by falling back to environment variables for endpoint/credential resolution.
|
||||
"""
|
||||
search_index_client = MagicMock(spec=SearchIndexClient)
|
||||
store = AzureAISearchStore(search_index_client=search_index_client)
|
||||
assert store.search_endpoint is None
|
||||
assert store.search_credential is None
|
||||
|
||||
collection = store.get_collection(
|
||||
collection_name="test",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
)
|
||||
assert collection is not None
|
||||
assert collection.collection_name == "test"
|
||||
assert collection.search_index_client == search_index_client
|
||||
assert collection.search_client is not None
|
||||
|
||||
|
||||
@mark.parametrize("exclude_list", [["AZURE_AI_SEARCH_API_KEY"]], indirect=True)
|
||||
def test_get_search_index_client(azure_ai_search_unit_test_env):
|
||||
from azure.core.credentials import AzureKeyCredential
|
||||
from azure.core.credentials_async import AsyncTokenCredential
|
||||
|
||||
settings = AzureAISearchSettings(**azure_ai_search_unit_test_env, env_file_path="test.env")
|
||||
|
||||
azure_credential = MagicMock(spec=AzureKeyCredential)
|
||||
client = _get_search_index_client(settings, azure_credential=azure_credential)
|
||||
assert client is not None
|
||||
|
||||
token_credential = MagicMock(spec=AsyncTokenCredential)
|
||||
client2 = _get_search_index_client(
|
||||
settings,
|
||||
token_credential=token_credential,
|
||||
)
|
||||
assert client2 is not None
|
||||
|
||||
with raises(ServiceInitializationError):
|
||||
_get_search_index_client(settings)
|
||||
|
||||
|
||||
@mark.parametrize("exclude_list", [["AZURE_AI_SEARCH_API_KEY"]], indirect=True)
|
||||
def test_resolve_credential(azure_ai_search_unit_test_env):
|
||||
from azure.core.credentials import AzureKeyCredential
|
||||
from azure.core.credentials_async import AsyncTokenCredential
|
||||
|
||||
settings = AzureAISearchSettings(**azure_ai_search_unit_test_env, env_file_path="test.env")
|
||||
|
||||
azure_credential = MagicMock(spec=AzureKeyCredential)
|
||||
resolved = _resolve_credential(settings, azure_credential=azure_credential)
|
||||
assert resolved == azure_credential
|
||||
|
||||
token_credential = MagicMock(spec=AsyncTokenCredential)
|
||||
resolved = _resolve_credential(settings, token_credential=token_credential)
|
||||
assert resolved == token_credential
|
||||
|
||||
with raises(ServiceInitializationError):
|
||||
_resolve_credential(settings)
|
||||
|
||||
|
||||
@mark.parametrize("include_vectors", [True, False])
|
||||
async def test_search_vectorized_search(collection, mock_search, include_vectors):
|
||||
results = await collection.search(vector=[0.1, 0.2, 0.3], include_vectors=include_vectors)
|
||||
assert results is not None
|
||||
async for result in results.results:
|
||||
assert result is not None
|
||||
assert result.record is not None
|
||||
assert result.record["id"] == "id1"
|
||||
assert result.record["content"] == "content"
|
||||
if include_vectors:
|
||||
assert result.record["vector"] == [1.0, 2.0, 3.0]
|
||||
for call in mock_search.call_args_list:
|
||||
assert call[1]["top"] == 3
|
||||
assert call[1]["skip"] == 0
|
||||
assert call[1]["include_total_count"] is False
|
||||
assert call[1]["select"] == ["*"] if include_vectors else ["id", "content"]
|
||||
assert call[1]["vector_queries"][0].vector == [0.1, 0.2, 0.3]
|
||||
assert call[1]["vector_queries"][0].fields == "vector"
|
||||
|
||||
|
||||
@mark.parametrize("include_vectors", [True, False])
|
||||
async def test_search_vectorizable_search(collection, mock_search, include_vectors):
|
||||
collection.embedding_generator = AsyncMock(spec=EmbeddingGeneratorBase)
|
||||
collection.embedding_generator.generate_embeddings.return_value = np.array([[0.1, 0.2, 0.3]])
|
||||
results = await collection.search("test", include_vectors=include_vectors)
|
||||
assert results is not None
|
||||
async for result in results.results:
|
||||
assert result is not None
|
||||
assert result.record is not None
|
||||
assert result.record["id"] == "id1"
|
||||
assert result.record["content"] == "content"
|
||||
if include_vectors:
|
||||
assert result.record["vector"] == [1.0, 2.0, 3.0]
|
||||
for call in mock_search.call_args_list:
|
||||
assert call[1]["top"] == 3
|
||||
assert call[1]["skip"] == 0
|
||||
assert call[1]["include_total_count"] is False
|
||||
assert call[1]["select"] == ["*"] if include_vectors else ["id", "content"]
|
||||
assert call[1]["vector_queries"][0].vector == [0.1, 0.2, 0.3]
|
||||
assert call[1]["vector_queries"][0].fields == "vector"
|
||||
|
||||
|
||||
@mark.parametrize("include_vectors", [True, False])
|
||||
@mark.parametrize("keywords", ["test", ["test1", "test2"]], ids=["single", "multiple"])
|
||||
async def test_search_keyword_hybrid_search(collection, mock_search, include_vectors, keywords):
|
||||
results = await collection.hybrid_search(
|
||||
values=keywords,
|
||||
vector=[0.1, 0.2, 0.3],
|
||||
include_vectors=include_vectors,
|
||||
additional_property_name="content",
|
||||
)
|
||||
assert results is not None
|
||||
async for result in results.results:
|
||||
assert result is not None
|
||||
assert result.record is not None
|
||||
assert result.record["id"] == "id1"
|
||||
assert result.record["content"] == "content"
|
||||
if include_vectors:
|
||||
assert result.record["vector"] == [1.0, 2.0, 3.0]
|
||||
for call in mock_search.call_args_list:
|
||||
assert call[1]["top"] == 3
|
||||
assert call[1]["skip"] == 0
|
||||
assert call[1]["include_total_count"] is False
|
||||
assert call[1]["select"] == ["*"] if include_vectors else ["id", "content"]
|
||||
assert call[1]["search_fields"] == ["content"]
|
||||
assert call[1]["search_text"] == "test" if keywords == "test" else "test1, test2"
|
||||
assert call[1]["vector_queries"][0].vector == [0.1, 0.2, 0.3]
|
||||
assert call[1]["vector_queries"][0].fields == "vector"
|
||||
|
||||
|
||||
@mark.parametrize("filter, result", filter_lambda_list("ai_search"))
|
||||
def test_lambda_filter(collection, filter, result):
|
||||
if isinstance(result, type) and issubclass(result, Exception):
|
||||
with raises(result):
|
||||
collection._build_filter(filter)
|
||||
else:
|
||||
filter_string = collection._build_filter(filter)
|
||||
assert filter_string == result
|
||||
@@ -0,0 +1,115 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from chromadb.api import ClientAPI
|
||||
from chromadb.api.models.Collection import Collection
|
||||
|
||||
from semantic_kernel.connectors.chroma import ChromaCollection, ChromaStore
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_client():
|
||||
return MagicMock(spec=ClientAPI)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def chroma_collection(mock_client, definition):
|
||||
return ChromaCollection(
|
||||
collection_name="test_collection",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
client=mock_client,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def chroma_store(mock_client):
|
||||
return ChromaStore(client=mock_client)
|
||||
|
||||
|
||||
def test_chroma_collection_initialization(chroma_collection):
|
||||
assert chroma_collection.collection_name == "test_collection"
|
||||
assert chroma_collection.record_type is dict
|
||||
|
||||
|
||||
def test_chroma_store_initialization(chroma_store):
|
||||
assert chroma_store.client is not None
|
||||
|
||||
|
||||
def test_chroma_collection_get_collection(chroma_collection, mock_client):
|
||||
mock_client.get_collection.return_value = "mock_collection"
|
||||
collection = chroma_collection._get_collection()
|
||||
assert collection == "mock_collection"
|
||||
|
||||
|
||||
def test_chroma_store_get_collection(chroma_store, mock_client, definition):
|
||||
collection = chroma_store.get_collection(collection_name="test_collection", record_type=dict, definition=definition)
|
||||
assert collection is not None
|
||||
assert isinstance(collection, ChromaCollection)
|
||||
|
||||
|
||||
async def test_chroma_collection_collection_exists(chroma_collection, mock_client):
|
||||
mock_client.get_collection.return_value = "mock_collection"
|
||||
exists = await chroma_collection.collection_exists()
|
||||
assert exists
|
||||
|
||||
|
||||
async def test_chroma_store_list_collection_names(chroma_store, mock_client):
|
||||
mock_collection = MagicMock(spec=Collection)
|
||||
mock_collection.name = "test_collection"
|
||||
mock_client.list_collections.return_value = [mock_collection]
|
||||
collections = await chroma_store.list_collection_names()
|
||||
assert collections == ["test_collection"]
|
||||
|
||||
|
||||
async def test_chroma_collection_ensure_collection_exists(chroma_collection, mock_client):
|
||||
await chroma_collection.ensure_collection_exists()
|
||||
mock_client.create_collection.assert_called_once_with(
|
||||
name="test_collection", embedding_function=None, configuration={"hnsw": {"space": "cosine"}}, get_or_create=True
|
||||
)
|
||||
|
||||
|
||||
async def test_chroma_collection_ensure_collection_deleted(chroma_collection, mock_client):
|
||||
await chroma_collection.ensure_collection_deleted()
|
||||
mock_client.delete_collection.assert_called_once_with(name="test_collection")
|
||||
|
||||
|
||||
async def test_chroma_collection_upsert(chroma_collection, mock_client):
|
||||
records = [{"id": "1", "vector": [0.1, 0.2, 0.3, 0.4, 0.5], "content": "test document"}]
|
||||
ids = await chroma_collection.upsert(records)
|
||||
assert ids == ["1"]
|
||||
mock_client.get_collection().add.assert_called_once()
|
||||
|
||||
|
||||
async def test_chroma_collection_get(chroma_collection, mock_client):
|
||||
mock_client.get_collection().get.return_value = {
|
||||
"ids": [["1"]],
|
||||
"documents": [["test document"]],
|
||||
"embeddings": [[[0.1, 0.2, 0.3, 0.4, 0.5]]],
|
||||
"metadatas": [[{}]],
|
||||
}
|
||||
records = await chroma_collection._inner_get(["1"])
|
||||
assert len(records) == 1
|
||||
assert records[0]["id"] == "1"
|
||||
|
||||
|
||||
async def test_chroma_collection_delete(chroma_collection, mock_client):
|
||||
await chroma_collection._inner_delete(["1"])
|
||||
mock_client.get_collection().delete.assert_called_once_with(ids=["1"])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("include_vectors", [True, False])
|
||||
async def test_chroma_collection_search(chroma_collection, mock_client, include_vectors):
|
||||
mock_client.get_collection().query.return_value = {
|
||||
"ids": [["1"]],
|
||||
"documents": [["test document"]],
|
||||
"embeddings": [[[0.1, 0.2, 0.3, 0.4, 0.5]]],
|
||||
"metadatas": [[{}]],
|
||||
"distances": [[0.1]],
|
||||
}
|
||||
results = await chroma_collection.search(vector=[0.1, 0.2, 0.3, 0.4, 0.5], top=1, include_vectors=include_vectors)
|
||||
async for res in results.results:
|
||||
assert res.record["id"] == "1"
|
||||
assert res.score == 0.1
|
||||
@@ -0,0 +1,176 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import faiss
|
||||
from pytest import fixture, mark, raises
|
||||
|
||||
from semantic_kernel.connectors.faiss import FaissCollection, FaissStore
|
||||
from semantic_kernel.data.vector import DistanceFunction, VectorStoreCollectionDefinition, VectorStoreField
|
||||
from semantic_kernel.exceptions import VectorStoreInitializationException
|
||||
|
||||
|
||||
@fixture(scope="function")
|
||||
def data_model_def() -> VectorStoreCollectionDefinition:
|
||||
return VectorStoreCollectionDefinition(
|
||||
fields=[
|
||||
VectorStoreField("key", name="id"),
|
||||
VectorStoreField("data", name="content"),
|
||||
VectorStoreField(
|
||||
"vector",
|
||||
name="vector",
|
||||
dimensions=5,
|
||||
index_kind="flat",
|
||||
distance_function="dot_prod",
|
||||
type="float",
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@fixture(scope="function")
|
||||
def store() -> FaissStore:
|
||||
return FaissStore()
|
||||
|
||||
|
||||
@fixture(scope="function")
|
||||
def faiss_collection(data_model_def):
|
||||
return FaissCollection(record_type=dict, definition=data_model_def, collection_name="test")
|
||||
|
||||
|
||||
async def test_store_get_collection(store, data_model_def):
|
||||
collection = store.get_collection(dict, definition=data_model_def, collection_name="test")
|
||||
assert collection.collection_name == "test"
|
||||
assert collection.record_type is dict
|
||||
assert collection.definition == data_model_def
|
||||
assert collection.inner_storage == {}
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"dist",
|
||||
[
|
||||
DistanceFunction.EUCLIDEAN_SQUARED_DISTANCE,
|
||||
DistanceFunction.DOT_PROD,
|
||||
],
|
||||
)
|
||||
async def test_ensure_collection_exists(store, data_model_def, dist):
|
||||
for field in data_model_def.fields:
|
||||
if field.name == "vector":
|
||||
field.distance_function = dist
|
||||
collection = store.get_collection(collection_name="test", record_type=dict, definition=data_model_def)
|
||||
await collection.ensure_collection_exists()
|
||||
assert collection.inner_storage == {}
|
||||
assert collection.indexes
|
||||
assert collection.indexes["vector"] is not None
|
||||
|
||||
|
||||
async def test_ensure_collection_exists_incompatible_dist(store, data_model_def):
|
||||
for field in data_model_def.fields:
|
||||
if field.name == "vector":
|
||||
field.distance_function = "cosine_distance"
|
||||
collection = store.get_collection(collection_name="test", record_type=dict, definition=data_model_def)
|
||||
with raises(VectorStoreInitializationException):
|
||||
await collection.ensure_collection_exists()
|
||||
|
||||
|
||||
async def test_ensure_collection_exists_custom(store, data_model_def):
|
||||
index = faiss.IndexFlat(5)
|
||||
collection = store.get_collection(collection_name="test", record_type=dict, definition=data_model_def)
|
||||
await collection.ensure_collection_exists(index=index)
|
||||
assert collection.inner_storage == {}
|
||||
assert collection.indexes
|
||||
assert collection.indexes["vector"] is not None
|
||||
assert collection.indexes["vector"] == index
|
||||
assert collection.indexes["vector"].is_trained is True
|
||||
await collection.ensure_collection_deleted()
|
||||
|
||||
|
||||
async def test_ensure_collection_exists_custom_untrained(store, data_model_def):
|
||||
index = faiss.IndexIVFFlat(faiss.IndexFlat(5), 5, 10)
|
||||
collection = store.get_collection(collection_name="test", record_type=dict, definition=data_model_def)
|
||||
with raises(VectorStoreInitializationException):
|
||||
await collection.ensure_collection_exists(index=index)
|
||||
del index
|
||||
|
||||
|
||||
async def test_ensure_collection_exists_custom_dict(store, data_model_def):
|
||||
index = faiss.IndexFlat(5)
|
||||
collection = store.get_collection(collection_name="test", record_type=dict, definition=data_model_def)
|
||||
await collection.ensure_collection_exists(indexes={"vector": index})
|
||||
assert collection.inner_storage == {}
|
||||
assert collection.indexes
|
||||
assert collection.indexes["vector"] is not None
|
||||
assert collection.indexes["vector"] == index
|
||||
await collection.ensure_collection_deleted()
|
||||
|
||||
|
||||
async def test_upsert(faiss_collection):
|
||||
await faiss_collection.ensure_collection_exists()
|
||||
record = {"id": "testid", "content": "test content", "vector": [0.1, 0.2, 0.3, 0.4, 0.5]}
|
||||
key = await faiss_collection.upsert(record)
|
||||
assert key == "testid"
|
||||
assert faiss_collection.inner_storage == {"testid": record}
|
||||
await faiss_collection.ensure_collection_deleted()
|
||||
|
||||
|
||||
async def test_get(faiss_collection):
|
||||
await faiss_collection.ensure_collection_exists()
|
||||
record = {"id": "testid", "content": "test content", "vector": [0.1, 0.2, 0.3, 0.4, 0.5]}
|
||||
await faiss_collection.upsert(record)
|
||||
result = await faiss_collection.get("testid")
|
||||
assert result["id"] == record["id"]
|
||||
assert result["content"] == record["content"]
|
||||
await faiss_collection.ensure_collection_deleted()
|
||||
|
||||
|
||||
async def test_get_missing(faiss_collection):
|
||||
await faiss_collection.ensure_collection_exists()
|
||||
result = await faiss_collection.get("testid")
|
||||
assert result is None
|
||||
await faiss_collection.ensure_collection_deleted()
|
||||
|
||||
|
||||
async def test_delete(faiss_collection):
|
||||
await faiss_collection.ensure_collection_exists()
|
||||
record = {"id": "testid", "content": "test content", "vector": [0.1, 0.2, 0.3, 0.4, 0.5]}
|
||||
await faiss_collection.upsert(record)
|
||||
await faiss_collection.delete("testid")
|
||||
assert faiss_collection.inner_storage == {}
|
||||
await faiss_collection.ensure_collection_deleted()
|
||||
|
||||
|
||||
async def test_collection_exists(faiss_collection):
|
||||
assert await faiss_collection.collection_exists() is False
|
||||
await faiss_collection.ensure_collection_exists()
|
||||
assert await faiss_collection.collection_exists() is True
|
||||
await faiss_collection.ensure_collection_deleted()
|
||||
|
||||
|
||||
async def test_ensure_collection_deleted(faiss_collection):
|
||||
await faiss_collection.ensure_collection_exists()
|
||||
record = {"id": "testid", "content": "test content", "vector": [0.1, 0.2, 0.3, 0.4, 0.5]}
|
||||
await faiss_collection.upsert(record)
|
||||
assert faiss_collection.inner_storage == {"testid": record}
|
||||
await faiss_collection.ensure_collection_deleted()
|
||||
assert faiss_collection.inner_storage == {}
|
||||
|
||||
|
||||
@mark.parametrize("dist", [DistanceFunction.EUCLIDEAN_SQUARED_DISTANCE, DistanceFunction.DOT_PROD])
|
||||
async def test_ensure_collection_exists_and_search(faiss_collection, dist):
|
||||
for field in faiss_collection.definition.fields:
|
||||
if field.name == "vector":
|
||||
field.distance_function = dist
|
||||
await faiss_collection.ensure_collection_exists()
|
||||
record1 = {"id": "testid1", "content": "test content", "vector": [1.0, 1.0, 1.0, 1.0, 1.0]}
|
||||
record2 = {"id": "testid2", "content": "test content", "vector": [-1.0, -1.0, -1.0, -1.0, -1.0]}
|
||||
await faiss_collection.upsert([record1, record2])
|
||||
results = await faiss_collection.search(
|
||||
vector=[0.9, 0.9, 0.9, 0.9, 0.9],
|
||||
vector_property_name="vector",
|
||||
include_total_count=True,
|
||||
include_vectors=True,
|
||||
)
|
||||
assert results.total_count == 2
|
||||
idx = 0
|
||||
async for res in results.results:
|
||||
assert res.record == record1 if idx == 0 else record2
|
||||
idx += 1
|
||||
await faiss_collection.ensure_collection_deleted()
|
||||
@@ -0,0 +1,294 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import ast
|
||||
|
||||
from pytest import fixture, mark, raises
|
||||
|
||||
from semantic_kernel.connectors.in_memory import InMemoryCollection, InMemoryStore
|
||||
from semantic_kernel.data._shared import default_dynamic_filter_function
|
||||
from semantic_kernel.data.vector import DistanceFunction
|
||||
from semantic_kernel.exceptions.vector_store_exceptions import VectorStoreOperationException
|
||||
|
||||
|
||||
@fixture
|
||||
def collection(definition):
|
||||
return InMemoryCollection(collection_name="test", record_type=dict, definition=definition)
|
||||
|
||||
|
||||
def test_store_init():
|
||||
store = InMemoryStore()
|
||||
assert store is not None
|
||||
|
||||
|
||||
def test_store_get_collection(definition):
|
||||
store = InMemoryStore()
|
||||
collection = store.get_collection(collection_name="test", record_type=dict, definition=definition)
|
||||
assert collection.collection_name == "test"
|
||||
assert collection.record_type is dict
|
||||
assert collection.definition == definition
|
||||
|
||||
|
||||
async def test_upsert(collection):
|
||||
record = {"id": "testid", "content": "test content", "vector": [0.1, 0.2, 0.3, 0.4, 0.5]}
|
||||
key = await collection.upsert(record)
|
||||
assert key == "testid"
|
||||
assert collection.inner_storage == {"testid": record}
|
||||
|
||||
|
||||
async def test_get(collection):
|
||||
record = {"id": "testid", "content": "test content", "vector": [0.1, 0.2, 0.3, 0.4, 0.5]}
|
||||
await collection.upsert(record)
|
||||
result = await collection.get("testid")
|
||||
assert result["id"] == record["id"]
|
||||
assert result["content"] == record["content"]
|
||||
|
||||
|
||||
async def test_get_missing(collection):
|
||||
result = await collection.get("testid")
|
||||
assert result is None
|
||||
|
||||
|
||||
async def test_delete(collection):
|
||||
record = {"id": "testid", "content": "test content", "vector": [0.1, 0.2, 0.3, 0.4, 0.5]}
|
||||
await collection.upsert(record)
|
||||
await collection.delete("testid")
|
||||
assert collection.inner_storage == {}
|
||||
|
||||
|
||||
async def test_collection_exists(collection):
|
||||
assert await collection.collection_exists() is True
|
||||
|
||||
|
||||
async def test_ensure_collection_deleted(collection):
|
||||
record = {"id": "testid", "content": "test content", "vector": [0.1, 0.2, 0.3, 0.4, 0.5]}
|
||||
await collection.upsert(record)
|
||||
assert collection.inner_storage == {"testid": record}
|
||||
await collection.ensure_collection_deleted()
|
||||
assert collection.inner_storage == {}
|
||||
|
||||
|
||||
async def test_ensure_collection_exists(collection):
|
||||
await collection.ensure_collection_exists()
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"distance_function",
|
||||
[
|
||||
DistanceFunction.COSINE_DISTANCE,
|
||||
DistanceFunction.COSINE_SIMILARITY,
|
||||
DistanceFunction.EUCLIDEAN_DISTANCE,
|
||||
DistanceFunction.MANHATTAN,
|
||||
DistanceFunction.EUCLIDEAN_SQUARED_DISTANCE,
|
||||
DistanceFunction.DOT_PROD,
|
||||
DistanceFunction.HAMMING,
|
||||
],
|
||||
)
|
||||
async def test_vectorized_search_similar(collection, distance_function):
|
||||
for field in collection.definition.fields:
|
||||
if field.name == "vector":
|
||||
field.distance_function = distance_function
|
||||
record1 = {"id": "testid1", "content": "test content", "vector": [1.0, 1.0, 1.0, 1.0, 1.0]}
|
||||
record2 = {"id": "testid2", "content": "test content", "vector": [-1.0, -1.0, -1.0, -1.0, -1.0]}
|
||||
await collection.upsert([record1, record2])
|
||||
results = await collection.search(
|
||||
vector=[0.9, 0.9, 0.9, 0.9, 0.9],
|
||||
vector_property_name="vector",
|
||||
include_total_count=True,
|
||||
include_vectors=True,
|
||||
)
|
||||
assert results.total_count == 2
|
||||
idx = 0
|
||||
async for res in results.results:
|
||||
assert res.record == record1 if idx == 0 else record2
|
||||
idx += 1
|
||||
|
||||
|
||||
async def test_valid_lambda_filter(collection):
|
||||
record1 = {"id": "1", "vector": [1, 2, 3, 4, 5]}
|
||||
record2 = {"id": "2", "vector": [5, 4, 3, 2, 1]}
|
||||
await collection.upsert([record1, record2])
|
||||
# Filter to select only record with id == '1'
|
||||
results = collection._get_filtered_records(type("opt", (), {"filter": "lambda x: x.id == '1'"})())
|
||||
assert len(results) == 1
|
||||
assert "1" in results
|
||||
|
||||
|
||||
async def test_valid_lambda_filter_attribute_access(collection):
|
||||
record1 = {"id": "1", "vector": [1, 2, 3, 4, 5]}
|
||||
record2 = {"id": "2", "vector": [5, 4, 3, 2, 1]}
|
||||
await collection.upsert([record1, record2])
|
||||
# Filter to select only record with id == '2' using attribute access
|
||||
results = collection._get_filtered_records(type("opt", (), {"filter": "lambda x: x['id'] == '2'"})())
|
||||
assert len(results) == 1
|
||||
assert "2" in results
|
||||
|
||||
|
||||
async def test_invalid_filter_not_lambda(collection):
|
||||
with raises(VectorStoreOperationException, match="must be a lambda expression"):
|
||||
collection._get_filtered_records(type("opt", (), {"filter": "x.id == '1'"})())
|
||||
|
||||
|
||||
async def test_invalid_filter_syntax(collection):
|
||||
with raises(VectorStoreOperationException, match="not valid Python"):
|
||||
collection._get_filtered_records(type("opt", (), {"filter": "lambda x: x.id == '1' and"})())
|
||||
|
||||
|
||||
async def test_malicious_filter_import(collection):
|
||||
# Should not allow import statement
|
||||
with raises(VectorStoreOperationException):
|
||||
collection._get_filtered_records(
|
||||
type("opt", (), {"filter": "lambda x: __import__('os').system('echo malicious')"})()
|
||||
)
|
||||
|
||||
|
||||
async def test_malicious_filter_exec(collection):
|
||||
# Should not allow exec or similar
|
||||
with raises(VectorStoreOperationException):
|
||||
collection._get_filtered_records(type("opt", (), {"filter": "lambda x: exec('print(1)')"})())
|
||||
|
||||
|
||||
async def test_malicious_filter_builtins(collection):
|
||||
# Should not allow access to builtins
|
||||
with raises(VectorStoreOperationException):
|
||||
collection._get_filtered_records(
|
||||
type("opt", (), {"filter": "lambda x: __builtins__.__import__('os').system('echo malicious')"})()
|
||||
)
|
||||
|
||||
|
||||
async def test_malicious_filter_open(collection):
|
||||
# Should not allow open()
|
||||
with raises(VectorStoreOperationException):
|
||||
collection._get_filtered_records(type("opt", (), {"filter": "lambda x: open('somefile.txt', 'w')"})())
|
||||
|
||||
|
||||
async def test_malicious_filter_eval(collection):
|
||||
# Should not allow eval()
|
||||
with raises(VectorStoreOperationException):
|
||||
collection._get_filtered_records(type("opt", (), {"filter": "lambda x: eval('2+2')"})())
|
||||
|
||||
|
||||
async def test_multiple_filters(collection):
|
||||
record1 = {"id": "1", "vector": [1, 2, 3, 4, 5]}
|
||||
record2 = {"id": "2", "vector": [5, 4, 3, 2, 1]}
|
||||
await collection.upsert([record1, record2])
|
||||
filters = ["lambda x: x.id == '1'", "lambda x: x.vector[0] == 1"]
|
||||
results = collection._get_filtered_records(type("opt", (), {"filter": filters})())
|
||||
assert len(results) == 1
|
||||
assert "1" in results
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"filter_str",
|
||||
[
|
||||
"lambda x: [x.clear][0]() or True",
|
||||
"lambda x: [x.update][0]({'role': 'admin'}) or True",
|
||||
"lambda x: [x.pop][0]('secret', '') or True",
|
||||
"lambda x: [x.__setitem__][0]('leaked', ['{0.__class__.__mro__}'.format][0](x)) or True",
|
||||
],
|
||||
)
|
||||
def test_malicious_subscript_call_patterns_blocked(collection, filter_str):
|
||||
with raises(VectorStoreOperationException, match="Call target node type 'Subscript' is not allowed"):
|
||||
collection._parse_and_validate_filter(filter_str)
|
||||
|
||||
|
||||
def test_direct_mutating_method_call_remains_blocked(collection):
|
||||
with raises(VectorStoreOperationException, match="Function 'clear' is not allowed"):
|
||||
collection._parse_and_validate_filter("lambda x: x.clear() or True")
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"attr",
|
||||
[
|
||||
"__base__",
|
||||
"__bases__",
|
||||
"__class__",
|
||||
"__mro__",
|
||||
"__subclasses__",
|
||||
"__globals__",
|
||||
],
|
||||
)
|
||||
def test_blocked_dunder_attributes_rejected(collection, attr):
|
||||
with raises(VectorStoreOperationException, match=f"Access to attribute '{attr}' is not allowed"):
|
||||
collection._parse_and_validate_filter(f"lambda x: x.{attr}")
|
||||
|
||||
|
||||
async def test_valid_lambda_filter_with_get_method(collection):
|
||||
record1 = {"id": "1", "vector": [1, 2, 3, 4, 5]}
|
||||
record2 = {"id": "2", "vector": [5, 4, 3, 2, 1]}
|
||||
await collection.upsert([record1, record2])
|
||||
results = collection._get_filtered_records(type("opt", (), {"filter": "lambda x: x.get('id') == '1'"})())
|
||||
assert len(results) == 1
|
||||
assert "1" in results
|
||||
|
||||
|
||||
async def test_valid_lambda_filter_with_bounded_sequence_repeat(collection):
|
||||
record = {"id": "1", "vector": [1, 2, 3, 4, 5]}
|
||||
await collection.upsert(record)
|
||||
|
||||
results = collection._get_filtered_records(type("opt", (), {"filter": "lambda x: ([0] * 2)[1] == 0"})())
|
||||
|
||||
assert len(results) == 1
|
||||
assert "1" in results
|
||||
|
||||
|
||||
async def test_sequence_repeat_limit_can_be_overridden(collection):
|
||||
record = {"id": "1", "vector": [1, 2, 3, 4, 5]}
|
||||
await collection.upsert(record)
|
||||
filter_options = type("opt", (), {"filter": "lambda x: ([0] * 2)[1] == 0"})()
|
||||
|
||||
collection.max_filter_sequence_repeat_size = 1
|
||||
with raises(VectorStoreOperationException, match="Sequence repetition in filter expressions exceeds the maximum"):
|
||||
collection._get_filtered_records(filter_options)
|
||||
|
||||
collection.max_filter_sequence_repeat_size = 2
|
||||
results = collection._get_filtered_records(filter_options)
|
||||
|
||||
assert len(results) == 1
|
||||
assert "1" in results
|
||||
|
||||
|
||||
async def test_callable_filter_cannot_mutate_stored_record(collection):
|
||||
record = {"id": "1", "content": "value", "vector": [1, 2, 3, 4, 5]}
|
||||
await collection.upsert(record)
|
||||
|
||||
def mutating_filter(x):
|
||||
x["role"] = "admin"
|
||||
return True
|
||||
|
||||
with raises(VectorStoreOperationException, match="Error running filter"):
|
||||
collection._get_filtered_records(type("opt", (), {"filter": mutating_filter})())
|
||||
|
||||
assert "role" not in collection.inner_storage["1"]
|
||||
assert collection.inner_storage["1"]["content"] == "value"
|
||||
|
||||
|
||||
def test_default_dynamic_filter_injection_payload_remains_string_literal(collection):
|
||||
class Param:
|
||||
def __init__(self, name, default_value=None):
|
||||
self.name = name
|
||||
self.default_value = default_value
|
||||
|
||||
injected_value = "' or [x.update][0]({'role':'admin'}) or x.name=='"
|
||||
generated_filter = default_dynamic_filter_function(
|
||||
filter=None,
|
||||
parameters=[Param("category")],
|
||||
category=injected_value,
|
||||
)
|
||||
|
||||
assert isinstance(generated_filter, str)
|
||||
tree = ast.parse(generated_filter, mode="eval")
|
||||
assert isinstance(tree.body, ast.Lambda)
|
||||
assert isinstance(tree.body.body, ast.Compare)
|
||||
assert isinstance(tree.body.body.comparators[0], ast.Constant)
|
||||
assert tree.body.body.comparators[0].value == injected_value
|
||||
|
||||
filter_func = collection._parse_and_validate_filter(generated_filter)
|
||||
assert filter_func({"category": "finance", "name": "alice", "vector": [0.1] * 5}) is False
|
||||
|
||||
|
||||
async def test_large_sequence_repeat_filter_is_blocked(collection):
|
||||
record = {"id": "1", "content": "value", "vector": [0.1, 0.2, 0.3, 0.4, 0.5]}
|
||||
await collection.upsert(record)
|
||||
|
||||
with raises(VectorStoreOperationException, match="Sequence repetition in filter expressions exceeds the maximum"):
|
||||
collection._get_filtered_records(type("opt", (), {"filter": "lambda x: [0] * 2000000000"})())
|
||||
@@ -0,0 +1,421 @@
|
||||
# Copyright (c) 2025, Oracle Corporation. All rights reserved. # noqa: CPY001
|
||||
|
||||
from array import array
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
from typing import Annotated
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import oracledb
|
||||
import pandas as pd
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from semantic_kernel.connectors.oracle import OracleCollection, OracleStore
|
||||
from semantic_kernel.data.vector import (
|
||||
DistanceFunction,
|
||||
IndexKind,
|
||||
VectorStoreCollectionDefinition,
|
||||
VectorStoreField,
|
||||
vectorstoremodel,
|
||||
)
|
||||
|
||||
|
||||
@vectorstoremodel
|
||||
@dataclass
|
||||
class SimpleModel:
|
||||
id: Annotated[int, VectorStoreField("key")]
|
||||
vector: Annotated[
|
||||
list[float] | None,
|
||||
VectorStoreField(
|
||||
"vector",
|
||||
type="float",
|
||||
dimensions=3,
|
||||
index_kind=IndexKind.HNSW,
|
||||
distance_function=DistanceFunction.EUCLIDEAN_SQUARED_DISTANCE,
|
||||
),
|
||||
] = None
|
||||
|
||||
|
||||
def PandasDataframeModel(record) -> tuple:
|
||||
definition = VectorStoreCollectionDefinition(
|
||||
fields=[
|
||||
VectorStoreField("key", name="id", type="int"),
|
||||
VectorStoreField(
|
||||
"vector",
|
||||
name="embedding",
|
||||
type="float32",
|
||||
dimensions=5,
|
||||
distance_function=DistanceFunction.COSINE_DISTANCE,
|
||||
index_kind=IndexKind.IVF_FLAT,
|
||||
),
|
||||
],
|
||||
to_dict=lambda record, **_: record.to_dict(orient="records"),
|
||||
from_dict=lambda records, **_: pd.DataFrame(records),
|
||||
container_mode=True,
|
||||
)
|
||||
df = pd.DataFrame([record]) if isinstance(record, dict) else pd.DataFrame(record)
|
||||
return definition, df
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def mock_connection_pool():
|
||||
mock_conn = AsyncMock()
|
||||
mock_conn.fetchall = AsyncMock(return_value=[("COLL1",), ("COLL2",)])
|
||||
|
||||
mock_context_manager = AsyncMock()
|
||||
mock_context_manager.__aenter__.return_value = mock_conn
|
||||
mock_context_manager.__aexit__.return_value = None
|
||||
|
||||
pool = MagicMock(spec=oracledb.AsyncConnectionPool)
|
||||
pool.acquire.return_value = mock_context_manager
|
||||
|
||||
return pool
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def oracle_store(mock_connection_pool):
|
||||
return OracleStore(
|
||||
connection_pool=mock_connection_pool,
|
||||
db_schema="MY_SCHEMA",
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_list_collection_names_with_schema(oracle_store):
|
||||
names = await oracle_store.list_collection_names()
|
||||
assert names == ["COLL1", "COLL2"]
|
||||
|
||||
|
||||
def test_get_collection_returns_oracle_collection(oracle_store):
|
||||
collection = oracle_store.get_collection(
|
||||
SimpleModel,
|
||||
collection_name="TEST",
|
||||
)
|
||||
assert isinstance(collection, OracleCollection)
|
||||
assert collection.collection_name == "TEST"
|
||||
assert collection.db_schema == "MY_SCHEMA"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collection_exists_true(oracle_store, mock_connection_pool):
|
||||
conn = AsyncMock()
|
||||
conn.fetchone = AsyncMock(return_value=(1,))
|
||||
mock_connection_pool.acquire.return_value.__aenter__.return_value = conn
|
||||
|
||||
collection = oracle_store.get_collection(
|
||||
SimpleModel,
|
||||
collection_name="EXISTING",
|
||||
)
|
||||
result = await collection.collection_exists()
|
||||
assert result is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_collection_exists_false(oracle_store, mock_connection_pool):
|
||||
conn = AsyncMock()
|
||||
conn.fetchone = AsyncMock(return_value=None)
|
||||
mock_connection_pool.acquire.return_value.__aenter__.return_value = conn
|
||||
|
||||
collection = oracle_store.get_collection(
|
||||
SimpleModel,
|
||||
collection_name="MISSING",
|
||||
)
|
||||
result = await collection.collection_exists()
|
||||
assert result is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_collection_exists_creates_when_missing(oracle_store, mock_connection_pool):
|
||||
conn = AsyncMock()
|
||||
conn.fetchone = AsyncMock(return_value=False)
|
||||
mock_connection_pool.acquire.return_value.__aenter__.return_value = conn
|
||||
collection = oracle_store.get_collection(SimpleModel, collection_name="NEW")
|
||||
await collection.ensure_collection_exists()
|
||||
conn.execute.assert_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_create_table_with_get_collection(oracle_store, mock_connection_pool):
|
||||
mock_conn = AsyncMock()
|
||||
mock_connection_pool.acquire.return_value.__aenter__.return_value = mock_conn
|
||||
|
||||
collection = oracle_store.get_collection(SimpleModel, collection_name="MY_COLLECTION")
|
||||
|
||||
await collection.ensure_collection_exists()
|
||||
|
||||
mock_conn.execute.assert_awaited()
|
||||
sql_statements = [args[0].lower() for name, args, _ in mock_conn.mock_calls if name == "execute"]
|
||||
|
||||
assert any("create table" in sql for sql in sql_statements)
|
||||
assert any("my_collection" in sql for sql in sql_statements)
|
||||
assert any("vector(3 , float64)" in sql for sql in sql_statements)
|
||||
|
||||
mock_conn.commit.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pandasDataframe_with_get_collection(oracle_store, mock_connection_pool):
|
||||
mock_conn = AsyncMock()
|
||||
mock_connection_pool.acquire.return_value.__aenter__.return_value = mock_conn
|
||||
|
||||
records = {
|
||||
"id": 1,
|
||||
"embedding": [1.1, 2.2, 3.3],
|
||||
}
|
||||
|
||||
definition, _ = PandasDataframeModel(records)
|
||||
|
||||
collection = oracle_store.get_collection(
|
||||
collection_name="MY_COLLECTION",
|
||||
record_type=pd.DataFrame,
|
||||
definition=definition,
|
||||
)
|
||||
await collection.ensure_collection_exists()
|
||||
|
||||
mock_conn.execute.assert_awaited()
|
||||
sql_statements = [args[0].lower() for name, args, _ in mock_conn.mock_calls if name == "execute"]
|
||||
|
||||
assert any("create table" in sql for sql in sql_statements)
|
||||
assert any("my_collection" in sql for sql in sql_statements)
|
||||
assert any("vector(5 , float32)" in sql for sql in sql_statements)
|
||||
|
||||
mock_conn.commit.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_index_creation_distance_with_get_collection(oracle_store, mock_connection_pool):
|
||||
mock_conn = AsyncMock()
|
||||
mock_connection_pool.acquire.return_value.__aenter__.return_value = mock_conn
|
||||
|
||||
collection = oracle_store.get_collection(SimpleModel, collection_name="COLLECTION_WITH_INDEX")
|
||||
|
||||
await collection.ensure_collection_exists()
|
||||
|
||||
mock_conn.execute.assert_awaited()
|
||||
called_sql = mock_conn.execute.call_args[0][0].lower()
|
||||
|
||||
assert "create vector index" in called_sql
|
||||
assert "collection_with_index_vector_idx" in called_sql
|
||||
assert "inmemory neighbor graph" in called_sql
|
||||
assert "distance euclidean_squared" in called_sql
|
||||
|
||||
mock_conn.commit.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ensure_collection_deleted(oracle_store, mock_connection_pool):
|
||||
conn = AsyncMock()
|
||||
conn.fetchone = AsyncMock(return_value=(1,))
|
||||
mock_connection_pool.acquire.return_value.__aenter__.return_value = conn
|
||||
|
||||
collection = oracle_store.get_collection(SimpleModel, collection_name="TO_DELETE")
|
||||
|
||||
await collection.ensure_collection_exists()
|
||||
await collection.ensure_collection_deleted()
|
||||
|
||||
assert any("DROP TABLE" in str(call.args[0]) for call in conn.execute.call_args_list)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upsert(oracle_store, mock_connection_pool):
|
||||
mock_conn = AsyncMock()
|
||||
mock_connection_pool.acquire.return_value.__aenter__.return_value = mock_conn
|
||||
|
||||
collection = oracle_store.get_collection(SimpleModel, collection_name="MY_COLLECTION")
|
||||
|
||||
await collection.ensure_collection_exists()
|
||||
await collection.upsert(SimpleModel(id=1, vector=[0.1, 0.2, 0.3]))
|
||||
|
||||
mock_conn.executemany.assert_called_once()
|
||||
|
||||
merge_sql, params = mock_conn.executemany.call_args[0]
|
||||
|
||||
assert merge_sql.startswith('MERGE INTO "MY_SCHEMA"."MY_COLLECTION"')
|
||||
assert 'UPDATE SET t."vector"' in merge_sql
|
||||
assert "WHEN NOT MATCHED THEN" in merge_sql
|
||||
assert 'INSERT ("id", "vector")' in merge_sql
|
||||
|
||||
expected_param = (1, array("d", [0.1, 0.2, 0.3]))
|
||||
assert params[0] == expected_param
|
||||
assert mock_conn.commit.call_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_with_include_vectors(oracle_store, mock_connection_pool):
|
||||
mock_conn = AsyncMock()
|
||||
mock_conn.description = [("id",), ("vector",)]
|
||||
mock_connection_pool.acquire.return_value.__aenter__.return_value = mock_conn
|
||||
|
||||
collection = oracle_store.get_collection(SimpleModel, collection_name="MY_COLLECTION")
|
||||
|
||||
await collection.ensure_collection_exists()
|
||||
await collection.upsert(SimpleModel(id=1, vector=[0.1, 0.2, 0.3]))
|
||||
|
||||
mock_conn.fetchall.return_value = [(1, [0.1, 0.2, 0.3])]
|
||||
results = await collection.get([1], include_vectors=True)
|
||||
assert results.id == 1
|
||||
assert results.vector == [0.1, 0.2, 0.3]
|
||||
|
||||
executed_sql = [args[0] for args, _ in mock_conn.fetchall.call_args_list]
|
||||
assert any('SELECT "id" AS "id", "vector" AS "vector"' in sql for sql in executed_sql), (
|
||||
"Expected vector column to be selected when include_vectors=True"
|
||||
)
|
||||
|
||||
mock_conn.fetchall.reset_mock()
|
||||
mock_conn.fetchall.return_value = [(1, None)]
|
||||
results = await collection.get([1], include_vectors=False)
|
||||
assert results.id == 1
|
||||
assert results.vector is None
|
||||
|
||||
executed_sql = [args[0] for args, _ in mock_conn.fetchall.call_args_list]
|
||||
assert any('SELECT "id" AS "id", "vector" AS "vector"' not in sql for sql in executed_sql), (
|
||||
"Vector column should not be selected when include_vectors=False"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upsert_and_get(oracle_store, mock_connection_pool):
|
||||
conn = AsyncMock()
|
||||
conn.description = [("id",), ("vector",)]
|
||||
conn.fetchall = AsyncMock(return_value=[(1, [0.1, 0.2, 0.3])])
|
||||
|
||||
mock_connection_pool.acquire.return_value.__aenter__.return_value = conn
|
||||
|
||||
collection = oracle_store.get_collection(SimpleModel, collection_name="TEST")
|
||||
|
||||
await collection.ensure_collection_exists()
|
||||
await collection.upsert(SimpleModel(id=1, vector=[0.1, 0.2, 0.3]))
|
||||
assert conn.executemany.await_count >= 1 or conn.execute.await_count >= 1, (
|
||||
"Expected upsert to call executemany() or execute()"
|
||||
)
|
||||
|
||||
results = await collection.get([1], include_vectors=True)
|
||||
|
||||
assert results.id == 1
|
||||
assert results.vector == [0.1, 0.2, 0.3]
|
||||
conn.fetchall.assert_awaited_once()
|
||||
conn.fetchall.reset_mock()
|
||||
conn.fetchall.return_value = [(1, None)]
|
||||
conn.description = [("id",)]
|
||||
|
||||
results = await collection.get([1], include_vectors=False)
|
||||
assert results.id == 1
|
||||
assert results.vector is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_record(oracle_store, mock_connection_pool):
|
||||
conn = AsyncMock()
|
||||
mock_connection_pool.acquire.return_value.__aenter__.return_value = conn
|
||||
|
||||
collection = oracle_store.get_collection(
|
||||
SimpleModel,
|
||||
collection_name="MY_COLLECTION",
|
||||
)
|
||||
|
||||
await collection.ensure_collection_exists()
|
||||
await collection.delete([1])
|
||||
conn.executemany.assert_awaited()
|
||||
called_sql = conn.executemany.call_args[0][0].lower()
|
||||
assert "delete" in called_sql
|
||||
assert "my_collection" in called_sql
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search(oracle_store, mock_connection_pool):
|
||||
class MockCursor:
|
||||
def __init__(self):
|
||||
self.execute_called_with = []
|
||||
self._rows = [(1, [0.1, 0.2, 0.3])]
|
||||
self.description = [SimpleNamespace(name="id"), SimpleNamespace(name="vector")]
|
||||
self._i = 0
|
||||
|
||||
async def execute(self, sql, binds=None):
|
||||
self.execute_called_with.append((sql, binds))
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
return None
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
return None
|
||||
|
||||
def __aiter__(self):
|
||||
self._i = 0
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
if self._i >= len(self._rows):
|
||||
raise StopAsyncIteration
|
||||
|
||||
r = self._rows[self._i]
|
||||
self._i += 1
|
||||
return r
|
||||
|
||||
mock_cursor = MockCursor()
|
||||
|
||||
class MockConnection:
|
||||
def __init__(self, cur):
|
||||
self._cur = cur
|
||||
self.inputtypehandler = None
|
||||
self.outputtypehandler = None
|
||||
self.execute = AsyncMock()
|
||||
self.commit = AsyncMock()
|
||||
|
||||
def cursor(self):
|
||||
return self._cur
|
||||
|
||||
mock_conn = MockConnection(mock_cursor)
|
||||
|
||||
class MockAcquire:
|
||||
def __init__(self, conn):
|
||||
self._conn = conn
|
||||
|
||||
def __await__(self):
|
||||
async def _():
|
||||
return self
|
||||
|
||||
return _().__await__()
|
||||
|
||||
async def __aenter__(self):
|
||||
return self._conn
|
||||
|
||||
async def __aexit__(self, exc_type, exc_val, exc_tb):
|
||||
return None
|
||||
|
||||
mock_connection_pool.acquire = lambda **kwargs: MockAcquire(mock_conn)
|
||||
|
||||
collection = oracle_store.get_collection(
|
||||
model=SimpleModel, record_type=SimpleModel, collection_name="MY_COLLECTION"
|
||||
)
|
||||
|
||||
await collection.ensure_collection_exists()
|
||||
assert mock_conn.execute.await_count >= 1
|
||||
|
||||
ks_results = await collection.search(
|
||||
vector_property_name="vector",
|
||||
vector=[0.1, 0.2, 0.3],
|
||||
top=1,
|
||||
filter=lambda x: x.id in [1, 7, 9],
|
||||
include_vectors=True,
|
||||
)
|
||||
results = [r async for r in ks_results.results]
|
||||
|
||||
assert mock_cursor.execute_called_with, "cursor.execute was not called"
|
||||
sql, binds = mock_cursor.execute_called_with[0]
|
||||
assert "SELECT" in sql.upper()
|
||||
assert '"id", "vector", VECTOR_DISTANCE' in sql
|
||||
expected_where = 'WHERE "id" IN (:bind_val1, :bind_val2, :bind_val3)'
|
||||
assert expected_where in sql
|
||||
|
||||
assert binds is None or isinstance(binds[0], array)
|
||||
|
||||
assert len(results) == 1
|
||||
assert results[0].record.id == 1
|
||||
assert results[0].record.vector == [0.1, 0.2, 0.3]
|
||||
@@ -0,0 +1,337 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
from pinecone import FetchResponse, IndexModel, Metric, QueryResponse, ServerlessSpec, Vector
|
||||
from pinecone.core.openapi.db_data.models import (
|
||||
Hit,
|
||||
ScoredVector,
|
||||
SearchRecordsResponse,
|
||||
SearchRecordsResponseResult,
|
||||
SearchUsage,
|
||||
)
|
||||
from pinecone.db_data.index_asyncio import _IndexAsyncio
|
||||
from pytest import fixture, mark, raises
|
||||
|
||||
from semantic_kernel.connectors.pinecone import PineconeCollection, PineconeStore
|
||||
from semantic_kernel.exceptions.vector_store_exceptions import VectorStoreInitializationException
|
||||
|
||||
BASE_PATH_ASYNCIO = "pinecone.PineconeAsyncio"
|
||||
BASE_PATH_INDEX_CLIENT_ASYNCIO = "pinecone.db_data.index_asyncio._IndexAsyncio"
|
||||
|
||||
|
||||
@fixture
|
||||
def embed(request) -> dict[str, Any] | None:
|
||||
if hasattr(request, "param"):
|
||||
return request.param
|
||||
return None
|
||||
|
||||
|
||||
@fixture
|
||||
def mock_index_model(embed: dict[str, Any] | None):
|
||||
"""Mock IndexModel for testing."""
|
||||
mock_index_model = Mock(spec=IndexModel)
|
||||
mock_index_model.name = "test"
|
||||
mock_index_model.embed = embed
|
||||
mock_index_model.host = "test_host"
|
||||
return mock_index_model
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_list_collection_names(mock_index_model):
|
||||
with patch(f"{BASE_PATH_ASYNCIO}.list_indexes") as mock_list_indexes:
|
||||
mock_list_indexes.return_value = [mock_index_model]
|
||||
yield mock_list_indexes
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_create_index(mock_index_model):
|
||||
with patch(f"{BASE_PATH_ASYNCIO}.create_index") as mock_create_index:
|
||||
mock_create_index.return_value = mock_index_model
|
||||
yield mock_create_index
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_create_index_for_model(mock_index_model):
|
||||
with patch(f"{BASE_PATH_ASYNCIO}.create_index_for_model") as mock_create_index_for_model:
|
||||
mock_create_index_for_model.return_value = mock_index_model
|
||||
yield mock_create_index_for_model
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_describe_index(mock_index_model):
|
||||
with patch(f"{BASE_PATH_ASYNCIO}.describe_index") as mock_describe_index:
|
||||
mock_describe_index.return_value = mock_index_model
|
||||
yield mock_describe_index
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_has_index():
|
||||
with patch(f"{BASE_PATH_ASYNCIO}.has_index") as mock_has_index:
|
||||
mock_create_index.return_value = True
|
||||
yield mock_has_index
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_index_asyncio():
|
||||
mock_index_asyncio = AsyncMock(spec=_IndexAsyncio)
|
||||
mock_index_asyncio.close.return_value = None
|
||||
with patch(f"{BASE_PATH_ASYNCIO}.IndexAsyncio") as mock_index:
|
||||
mock_index.return_value = mock_index_asyncio
|
||||
yield mock_index
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_delete_index():
|
||||
with patch(f"{BASE_PATH_ASYNCIO}.delete_index") as mock_delete:
|
||||
yield mock_delete
|
||||
|
||||
|
||||
@fixture
|
||||
async def store(pinecone_unit_test_env) -> PineconeStore:
|
||||
"""Fixture to create a Pinecone store."""
|
||||
async with PineconeStore() as store:
|
||||
yield store
|
||||
|
||||
|
||||
@fixture
|
||||
async def collection(pinecone_unit_test_env, definition) -> PineconeCollection:
|
||||
"""Fixture to create a Pinecone store."""
|
||||
async with PineconeCollection(
|
||||
collection_name="test_collection",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
) as collection:
|
||||
yield collection
|
||||
|
||||
|
||||
async def test_create_store(pinecone_unit_test_env):
|
||||
"""Test the creation of a Pinecone store."""
|
||||
# Create a Pinecone store
|
||||
store = PineconeStore()
|
||||
assert store is not None
|
||||
assert store.client is not None
|
||||
|
||||
|
||||
@mark.parametrize("exclude_list", [["PINECONE_API_KEY"]], indirect=True)
|
||||
async def test_create_store_fail(pinecone_unit_test_env):
|
||||
"""Test the creation of a Pinecone store."""
|
||||
with raises(VectorStoreInitializationException):
|
||||
PineconeStore(env_file_path="test.env")
|
||||
|
||||
|
||||
def test_create_store_grpc(pinecone_unit_test_env):
|
||||
"""Test the creation of a Pinecone store."""
|
||||
|
||||
# Create a Pinecone store
|
||||
store = PineconeStore(use_grpc=True)
|
||||
assert store is not None
|
||||
assert store.client is not None
|
||||
|
||||
|
||||
@mark.parametrize("exclude_list", [["PINECONE_API_KEY"]], indirect=True)
|
||||
async def test_ensure_collection_exists_fail(pinecone_unit_test_env, definition):
|
||||
with raises(VectorStoreInitializationException):
|
||||
PineconeCollection(
|
||||
collection_name="test_collection",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
env_file_path="test.env",
|
||||
)
|
||||
|
||||
|
||||
async def test_get_collection(store: PineconeStore, definition):
|
||||
"""Test the creation of a Pinecone collection."""
|
||||
# Create a collection
|
||||
collection = store.get_collection(collection_name="test_collection", record_type=dict, definition=definition)
|
||||
assert collection is not None
|
||||
assert collection.collection_name == "test_collection"
|
||||
|
||||
|
||||
async def test_list_collection_names(store: PineconeStore):
|
||||
"""Test the listing of Pinecone collections."""
|
||||
# List collections
|
||||
collections = await store.list_collection_names()
|
||||
assert collections is not None
|
||||
assert len(collections) == 1
|
||||
assert collections[0] == "test"
|
||||
|
||||
|
||||
@mark.parametrize("embed", [None, {"model": "test-model"}])
|
||||
async def test_load_index_client(collection, mock_index_asyncio):
|
||||
# Test loading the index client
|
||||
await collection._load_index_client()
|
||||
assert collection.index is not None
|
||||
assert collection.index_client is not None
|
||||
assert isinstance(collection.index_client, _IndexAsyncio)
|
||||
assert collection.embed_settings == collection.index.embed
|
||||
|
||||
|
||||
async def test_ensure_collection_exists(collection, mock_create_index):
|
||||
await collection.ensure_collection_exists()
|
||||
assert collection.index is not None
|
||||
assert collection.index_client is not None
|
||||
mock_create_index.assert_awaited_once_with(
|
||||
name=collection.collection_name,
|
||||
spec=ServerlessSpec(cloud="aws", region="us-east-1"),
|
||||
dimension=5,
|
||||
metric=Metric.COSINE,
|
||||
vector_type="dense",
|
||||
)
|
||||
|
||||
|
||||
@mark.parametrize("embed", [{"model": "test-model"}])
|
||||
async def test_ensure_collection_exists_integrated(collection, mock_create_index_for_model):
|
||||
await collection.ensure_collection_exists(embed={"model": "test-model"})
|
||||
assert collection.index is not None
|
||||
assert collection.index_client is not None
|
||||
mock_create_index_for_model.assert_awaited_once_with(
|
||||
name=collection.collection_name,
|
||||
cloud="aws",
|
||||
region="us-east-1",
|
||||
embed={"model": "test-model", "metric": Metric.COSINE, "field_map": {"text": "vector"}},
|
||||
)
|
||||
|
||||
|
||||
async def test_ensure_collection_deleted(collection):
|
||||
# Test deleting the collection
|
||||
await collection.ensure_collection_deleted()
|
||||
assert collection.index is None
|
||||
assert collection.index_client is None
|
||||
|
||||
|
||||
async def test_upsert(collection):
|
||||
record = {
|
||||
"id": "test_id",
|
||||
"vector": [0.1, 0.2, 0.3, 0.4, 0.5],
|
||||
"content": "test_content",
|
||||
}
|
||||
pinecone_vector = Vector(values=record["vector"], id=record["id"], metadata={"content": record["content"]})
|
||||
await collection._load_index_client()
|
||||
with patch.object(collection.index_client, "upsert", new_callable=AsyncMock) as mock_upsert:
|
||||
await collection.upsert(record)
|
||||
mock_upsert.assert_awaited_once_with(
|
||||
[pinecone_vector],
|
||||
namespace=collection.namespace,
|
||||
)
|
||||
|
||||
|
||||
@mark.parametrize("embed", [{"model": "test-model"}])
|
||||
async def test_upsert_embed(collection):
|
||||
record = {
|
||||
"id": "test_id",
|
||||
"content": "test_content",
|
||||
"vector": [0.1, 0.2, 0.3, 0.4, 0.5],
|
||||
}
|
||||
await collection._load_index_client()
|
||||
with patch.object(collection.index_client, "upsert_records", new_callable=AsyncMock) as mock_upsert:
|
||||
await collection.upsert(record)
|
||||
mock_upsert.assert_awaited_once_with(
|
||||
records=[{"_id": record["id"], "content": record["content"]}],
|
||||
namespace=collection.namespace,
|
||||
)
|
||||
|
||||
|
||||
async def test_get(collection):
|
||||
record = {
|
||||
"id": "test_id",
|
||||
"vector": [0.1, 0.2, 0.3, 0.4, 0.5],
|
||||
"content": "test_content",
|
||||
}
|
||||
fetch_response = FetchResponse(
|
||||
namespace="",
|
||||
vectors={
|
||||
record["id"]: Vector(values=record["vector"], id=record["id"], metadata={"content": record["content"]})
|
||||
},
|
||||
usage={},
|
||||
)
|
||||
await collection._load_index_client()
|
||||
with patch.object(collection.index_client, "fetch", new_callable=AsyncMock) as mock_fetch:
|
||||
mock_fetch.return_value = fetch_response
|
||||
get_record = await collection.get(record["id"])
|
||||
mock_fetch.assert_awaited_once_with(
|
||||
ids=[record["id"]],
|
||||
namespace=collection.namespace,
|
||||
)
|
||||
assert record["id"] == get_record["id"]
|
||||
assert record["content"] == get_record["content"]
|
||||
|
||||
|
||||
async def test_delete(collection):
|
||||
await collection._load_index_client()
|
||||
with patch.object(collection.index_client, "delete", new_callable=AsyncMock) as mock_delete:
|
||||
await collection.delete("test_id")
|
||||
mock_delete.assert_awaited_once_with(
|
||||
ids=["test_id"],
|
||||
namespace=collection.namespace,
|
||||
)
|
||||
|
||||
|
||||
async def test_search(collection):
|
||||
record = {
|
||||
"id": "test_id",
|
||||
"vector": [0.1, 0.2, 0.3, 0.4, 0.5],
|
||||
"content": "test_content",
|
||||
}
|
||||
query_response = QueryResponse._from_openapi_data(
|
||||
namespace="",
|
||||
matches=[
|
||||
ScoredVector(**{
|
||||
"values": record["vector"],
|
||||
"id": record["id"],
|
||||
"metadata": {"content": record["content"]},
|
||||
"score": 0.1,
|
||||
})
|
||||
],
|
||||
)
|
||||
await collection._load_index_client()
|
||||
with patch.object(collection.index_client, "query", new_callable=AsyncMock) as mock_query:
|
||||
mock_query.return_value = query_response
|
||||
query_response = await collection.search(
|
||||
vector=[0.1, 0.2, 0.3, 0.4, 0.5],
|
||||
top=1,
|
||||
include_vectors=True,
|
||||
filter=lambda x: x.content == "test_content",
|
||||
)
|
||||
mock_query.assert_awaited_once_with(
|
||||
vector=[0.1, 0.2, 0.3, 0.4, 0.5],
|
||||
top_k=1,
|
||||
include_metadata=True,
|
||||
include_values=True,
|
||||
namespace=collection.namespace,
|
||||
filter={"content": "test_content"},
|
||||
)
|
||||
assert query_response.total_count == 1
|
||||
async for result in query_response.results:
|
||||
assert result.record == record
|
||||
assert result.score == 0.1
|
||||
|
||||
|
||||
@mark.parametrize("embed", [{"model": "test-model"}])
|
||||
async def test_search_embed(collection):
|
||||
record = {"id": "test_id", "content": "test_content", "vector": None}
|
||||
query_response = SearchRecordsResponse._from_openapi_data(
|
||||
result=SearchRecordsResponseResult._from_openapi_data(**{
|
||||
"hits": [
|
||||
Hit(**{
|
||||
"_id": record["id"],
|
||||
"fields": {"id": record["id"], "content": record["content"]},
|
||||
"_score": 0.1,
|
||||
})
|
||||
]
|
||||
}),
|
||||
usage=SearchUsage(read_units=0),
|
||||
)
|
||||
await collection._load_index_client()
|
||||
with patch.object(collection.index_client, "search_records", new_callable=AsyncMock) as mock_query:
|
||||
mock_query.return_value = query_response
|
||||
query_response = await collection.search(values="test", top=1, include_vectors=True)
|
||||
mock_query.assert_awaited_once_with(
|
||||
query={"inputs": {"text": "test"}, "top_k": 1},
|
||||
namespace=collection.namespace,
|
||||
)
|
||||
assert query_response.total_count == 1
|
||||
async for result in query_response.results:
|
||||
assert result.record == record
|
||||
assert result.score == 0.1
|
||||
@@ -0,0 +1,415 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from collections.abc import AsyncGenerator
|
||||
from dataclasses import dataclass
|
||||
from typing import Annotated, Any
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
from psycopg import AsyncConnection, AsyncCursor
|
||||
from psycopg_pool import AsyncConnectionPool
|
||||
from pytest import fixture
|
||||
|
||||
from semantic_kernel.connectors.postgres import (
|
||||
DISTANCE_COLUMN_NAME,
|
||||
PostgresCollection,
|
||||
PostgresSettings,
|
||||
PostgresStore,
|
||||
)
|
||||
from semantic_kernel.data.vector import DistanceFunction, IndexKind, VectorStoreField, vectorstoremodel
|
||||
|
||||
|
||||
@fixture(scope="function")
|
||||
def mock_cursor():
|
||||
return AsyncMock(spec=AsyncCursor)
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_connection_pool(mock_cursor: Mock):
|
||||
with (
|
||||
patch(
|
||||
f"{AsyncConnectionPool.__module__}.{AsyncConnectionPool.__qualname__}.connection",
|
||||
) as mock_pool_connection,
|
||||
patch(
|
||||
f"{AsyncConnectionPool.__module__}.{AsyncConnectionPool.__qualname__}.open",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_pool_open,
|
||||
):
|
||||
mock_conn = AsyncMock(spec=AsyncConnection)
|
||||
|
||||
mock_pool_connection.return_value.__aenter__.return_value = mock_conn
|
||||
mock_conn.cursor.return_value.__aenter__.return_value = mock_cursor
|
||||
|
||||
mock_pool_open.return_value = None
|
||||
|
||||
yield mock_pool_connection, mock_pool_open
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def vector_store(postgres_unit_test_env) -> AsyncGenerator[PostgresStore, None]:
|
||||
async with await PostgresSettings(env_file_path="test.env").create_connection_pool() as pool:
|
||||
yield PostgresStore(connection_pool=pool)
|
||||
|
||||
|
||||
@vectorstoremodel
|
||||
@dataclass
|
||||
class SimpleDataModel:
|
||||
id: Annotated[int, VectorStoreField("key")]
|
||||
data: Annotated[
|
||||
list[float] | str | None,
|
||||
VectorStoreField(
|
||||
"vector",
|
||||
type="float",
|
||||
dimensions=1536,
|
||||
index_kind=IndexKind.HNSW,
|
||||
distance_function=DistanceFunction.COSINE_SIMILARITY,
|
||||
),
|
||||
] = None
|
||||
|
||||
|
||||
# region VectorStore Tests
|
||||
|
||||
|
||||
async def test_vector_store_defaults(vector_store: PostgresStore) -> None:
|
||||
assert vector_store.connection_pool is not None
|
||||
async with vector_store.connection_pool.connection() as conn:
|
||||
assert isinstance(conn, Mock)
|
||||
|
||||
|
||||
def test_vector_store_with_connection_pool(vector_store: PostgresStore) -> None:
|
||||
connection_pool = MagicMock(spec=AsyncConnectionPool)
|
||||
vector_store = PostgresStore(connection_pool=connection_pool)
|
||||
assert vector_store.connection_pool == connection_pool
|
||||
|
||||
|
||||
async def test_list_collection_names(vector_store: PostgresStore, mock_cursor: Mock) -> None:
|
||||
mock_cursor.fetchall.return_value = [
|
||||
("test_collection",),
|
||||
("test_collection_2",),
|
||||
]
|
||||
names = await vector_store.list_collection_names()
|
||||
assert names == ["test_collection", "test_collection_2"]
|
||||
|
||||
|
||||
def test_get_collection(vector_store: PostgresStore) -> None:
|
||||
collection = vector_store.get_collection(collection_name="test_collection", record_type=SimpleDataModel)
|
||||
assert collection.collection_name == "test_collection"
|
||||
|
||||
|
||||
async def test_collection_exists(vector_store: PostgresStore, mock_cursor: Mock) -> None:
|
||||
mock_cursor.fetchall.return_value = [("test_collection",)]
|
||||
collection = vector_store.get_collection(collection_name="test_collection", record_type=SimpleDataModel)
|
||||
result = await collection.collection_exists()
|
||||
assert result is True
|
||||
|
||||
|
||||
async def test_ensure_collection_deleted(vector_store: PostgresStore, mock_cursor: Mock) -> None:
|
||||
collection = vector_store.get_collection(collection_name="test_collection", record_type=SimpleDataModel)
|
||||
await collection.ensure_collection_deleted()
|
||||
|
||||
assert mock_cursor.execute.call_count == 1
|
||||
execute_args, _ = mock_cursor.execute.call_args
|
||||
statement = execute_args[0]
|
||||
statement_str = statement.as_string()
|
||||
|
||||
assert statement_str == 'DROP TABLE "public"."test_collection" CASCADE'
|
||||
|
||||
|
||||
async def test_delete_records(vector_store: PostgresStore, mock_cursor: Mock) -> None:
|
||||
collection = vector_store.get_collection(collection_name="test_collection", record_type=SimpleDataModel)
|
||||
await collection.delete([1, 2])
|
||||
|
||||
assert mock_cursor.execute.call_count == 1
|
||||
execute_args, _ = mock_cursor.execute.call_args
|
||||
statement = execute_args[0]
|
||||
statement_str = statement.as_string()
|
||||
|
||||
assert statement_str == """DELETE FROM "public"."test_collection" WHERE "id" IN (1, 2)"""
|
||||
|
||||
|
||||
async def test_ensure_collection_exists_simple_model(vector_store: PostgresStore, mock_cursor: Mock) -> None:
|
||||
collection = vector_store.get_collection(collection_name="test_collection", record_type=SimpleDataModel)
|
||||
await collection.ensure_collection_exists()
|
||||
|
||||
# 2 calls, once for the table creation and once for the index creation
|
||||
assert mock_cursor.execute.call_count == 2
|
||||
|
||||
# Check the table creation statement
|
||||
execute_args, _ = mock_cursor.execute.call_args_list[0]
|
||||
statement = execute_args[0]
|
||||
statement_str = statement.as_string()
|
||||
assert statement_str == ('CREATE TABLE "public"."test_collection" ("id" INTEGER PRIMARY KEY, "data" VECTOR(1536))')
|
||||
|
||||
# Check the index creation statement
|
||||
execute_args, _ = mock_cursor.execute.call_args_list[1]
|
||||
statement = execute_args[0]
|
||||
statement_str = statement.as_string()
|
||||
assert statement_str == (
|
||||
'CREATE INDEX "test_collection_data_idx" ON "public"."test_collection" USING hnsw ("data" vector_cosine_ops)'
|
||||
)
|
||||
|
||||
|
||||
async def test_ensure_collection_exists_model_with_python_types(vector_store: PostgresStore, mock_cursor: Mock) -> None:
|
||||
@vectorstoremodel
|
||||
@dataclass
|
||||
class ModelWithImplicitTypes:
|
||||
name: Annotated[str, VectorStoreField("key")]
|
||||
age: Annotated[int, VectorStoreField("data")]
|
||||
data: Annotated[dict[str, Any], VectorStoreField("data")]
|
||||
embedding: Annotated[list[float], VectorStoreField("vector", dimensions=20)]
|
||||
scores: Annotated[list[float], VectorStoreField("data")]
|
||||
tags: Annotated[list[str], VectorStoreField("data")]
|
||||
|
||||
collection = vector_store.get_collection(collection_name="test_collection", record_type=ModelWithImplicitTypes)
|
||||
|
||||
await collection.ensure_collection_exists()
|
||||
|
||||
assert mock_cursor.execute.call_count == 2
|
||||
|
||||
# Check the table creation statement
|
||||
execute_args, _ = mock_cursor.execute.call_args_list[0]
|
||||
statement = execute_args[0]
|
||||
statement_str = statement.as_string()
|
||||
assert statement_str == (
|
||||
'CREATE TABLE "public"."test_collection" '
|
||||
'("name" TEXT PRIMARY KEY, "age" INTEGER, "data" JSONB, '
|
||||
'"embedding" VECTOR(20), "scores" DOUBLE PRECISION[], "tags" TEXT[])'
|
||||
)
|
||||
|
||||
# Check the index creation statement
|
||||
execute_args, _ = mock_cursor.execute.call_args_list[1]
|
||||
statement = execute_args[0]
|
||||
statement_str = statement.as_string()
|
||||
assert statement_str == (
|
||||
'CREATE INDEX "test_collection_embedding_idx" ON "public"."test_collection" '
|
||||
'USING hnsw ("embedding" vector_cosine_ops)'
|
||||
)
|
||||
|
||||
|
||||
async def test_upsert_records(vector_store: PostgresStore, mock_cursor: Mock) -> None:
|
||||
collection = vector_store.get_collection(collection_name="test_collection", record_type=SimpleDataModel)
|
||||
await collection.upsert([
|
||||
SimpleDataModel(id=1, data=[1.0, 2.0, 3.0]),
|
||||
SimpleDataModel(id=2, data=[4.0, 5.0, 6.0]),
|
||||
SimpleDataModel(id=3, data=[5.0, 6.0, 1.0]),
|
||||
])
|
||||
|
||||
assert mock_cursor.executemany.call_count == 1
|
||||
execute_args, _ = mock_cursor.executemany.call_args
|
||||
statement_str = execute_args[0].as_string()
|
||||
values = execute_args[1]
|
||||
assert len(values) == 3
|
||||
|
||||
assert statement_str == (
|
||||
'INSERT INTO "public"."test_collection" ("id", "data") '
|
||||
"VALUES (%s, %s) "
|
||||
'ON CONFLICT ("id") DO UPDATE SET "data" = EXCLUDED."data"'
|
||||
)
|
||||
|
||||
assert values[0] == (1, [1.0, 2.0, 3.0])
|
||||
assert values[1] == (2, [4.0, 5.0, 6.0])
|
||||
assert values[2] == (3, [5.0, 6.0, 1.0])
|
||||
|
||||
|
||||
async def test_get_records(vector_store: PostgresStore, mock_cursor: Mock) -> None:
|
||||
mock_cursor.fetchall.return_value = [
|
||||
(1, "[1.0, 2.0, 3.0]", {"key": "value1"}),
|
||||
(2, "[4.0, 5.0, 6.0]", {"key": "value2"}),
|
||||
(3, "[5.0, 6.0, 1.0]", {"key": "value3"}),
|
||||
]
|
||||
|
||||
collection = vector_store.get_collection(collection_name="test_collection", record_type=SimpleDataModel)
|
||||
records = await collection.get([1, 2, 3])
|
||||
|
||||
assert len(records) == 3
|
||||
assert records[0].id == 1
|
||||
assert records[1].id == 2
|
||||
assert records[2].id == 3
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
# region Vector Search tests
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"distance_function, operator, subquery_distance, include_vectors, include_total_count",
|
||||
[
|
||||
(DistanceFunction.COSINE_SIMILARITY, "<=>", f'1 - subquery."{DISTANCE_COLUMN_NAME}"', False, False),
|
||||
(DistanceFunction.COSINE_DISTANCE, "<=>", None, False, False),
|
||||
(DistanceFunction.DOT_PROD, "<#>", f'-1 * subquery."{DISTANCE_COLUMN_NAME}"', True, False),
|
||||
(DistanceFunction.EUCLIDEAN_DISTANCE, "<->", None, False, True),
|
||||
(DistanceFunction.MANHATTAN, "<+>", None, True, True),
|
||||
],
|
||||
)
|
||||
async def test_vector_search(
|
||||
vector_store: PostgresStore,
|
||||
mock_cursor: Mock,
|
||||
distance_function: DistanceFunction,
|
||||
operator: str,
|
||||
subquery_distance: str | None,
|
||||
include_vectors: bool,
|
||||
include_total_count: bool,
|
||||
) -> None:
|
||||
@vectorstoremodel
|
||||
@dataclass
|
||||
class SimpleDataModel:
|
||||
id: Annotated[int, VectorStoreField("key")]
|
||||
embedding: Annotated[
|
||||
list[float] | str | None,
|
||||
VectorStoreField(
|
||||
"vector",
|
||||
index_kind=IndexKind.HNSW,
|
||||
dimensions=1536,
|
||||
distance_function=distance_function,
|
||||
type="float",
|
||||
),
|
||||
]
|
||||
data: Annotated[
|
||||
dict[str, Any],
|
||||
VectorStoreField("data", type="JSONB"),
|
||||
]
|
||||
|
||||
def model_post_init(self, context: Any) -> None:
|
||||
if self.embedding is None:
|
||||
self.embedding = self.data
|
||||
|
||||
collection = vector_store.get_collection(collection_name="test_collection", record_type=SimpleDataModel)
|
||||
assert isinstance(collection, PostgresCollection)
|
||||
|
||||
search_results = await collection.search(
|
||||
vector=[1.0, 2.0, 3.0],
|
||||
top=10,
|
||||
skip=5,
|
||||
include_vectors=include_vectors,
|
||||
include_total_count=include_total_count,
|
||||
)
|
||||
if include_total_count:
|
||||
# Including total count issues query directly
|
||||
assert mock_cursor.execute.call_count == 1
|
||||
else:
|
||||
# Total count is not included, query is issued when iterating over results
|
||||
assert mock_cursor.execute.call_count == 0
|
||||
async for _ in search_results.results:
|
||||
pass
|
||||
assert mock_cursor.execute.call_count == 1
|
||||
|
||||
execute_args, _ = mock_cursor.execute.call_args
|
||||
|
||||
assert (search_results.total_count is not None) == include_total_count
|
||||
|
||||
statement = execute_args[0]
|
||||
statement_str = statement.as_string()
|
||||
|
||||
expected_columns = '"id", "data"'
|
||||
if include_vectors:
|
||||
expected_columns = '"id", "embedding", "data"'
|
||||
|
||||
expected_statement = (
|
||||
f'SELECT {expected_columns}, "embedding" {operator} %s as "{DISTANCE_COLUMN_NAME}" '
|
||||
'FROM "public"."test_collection" '
|
||||
f'ORDER BY "{DISTANCE_COLUMN_NAME}" LIMIT 10 OFFSET 5'
|
||||
)
|
||||
|
||||
if subquery_distance:
|
||||
expected_statement = (
|
||||
f'SELECT subquery.*, {subquery_distance} AS "{DISTANCE_COLUMN_NAME}" FROM ('
|
||||
+ expected_statement
|
||||
+ ") AS subquery"
|
||||
)
|
||||
|
||||
assert statement_str == expected_statement
|
||||
|
||||
|
||||
async def test_model_post_init_conflicting_distance_column_name(vector_store: PostgresStore) -> None:
|
||||
@vectorstoremodel
|
||||
@dataclass
|
||||
class ConflictingDataModel:
|
||||
id: Annotated[int, VectorStoreField("key")]
|
||||
sk_pg_distance: Annotated[
|
||||
float, VectorStoreField("data")
|
||||
] # Note: test depends on value of DISTANCE_COLUMN_NAME constant
|
||||
|
||||
embedding: Annotated[
|
||||
list[float],
|
||||
VectorStoreField(
|
||||
"vector",
|
||||
index_kind=IndexKind.HNSW,
|
||||
dimensions=1536,
|
||||
distance_function=DistanceFunction.COSINE_SIMILARITY,
|
||||
type="float",
|
||||
),
|
||||
]
|
||||
data: Annotated[
|
||||
dict[str, Any],
|
||||
VectorStoreField("data", type="JSONB"),
|
||||
]
|
||||
|
||||
collection = vector_store.get_collection(collection_name="test_collection", record_type=ConflictingDataModel)
|
||||
assert isinstance(collection, PostgresCollection)
|
||||
|
||||
# Ensure that the distance column name has been changed to avoid conflict
|
||||
assert collection._distance_column_name != DISTANCE_COLUMN_NAME
|
||||
assert collection._distance_column_name.startswith(f"{DISTANCE_COLUMN_NAME}_")
|
||||
|
||||
|
||||
# endregion
|
||||
|
||||
# region Settings tests
|
||||
|
||||
|
||||
def test_settings_connection_string(monkeypatch) -> None:
|
||||
monkeypatch.delenv("PGHOST", raising=False)
|
||||
monkeypatch.delenv("PGPORT", raising=False)
|
||||
monkeypatch.delenv("PGDATABASE", raising=False)
|
||||
monkeypatch.delenv("PGUSER", raising=False)
|
||||
monkeypatch.delenv("PGPASSWORD", raising=False)
|
||||
|
||||
settings = PostgresSettings(connection_string="host=localhost port=5432 dbname=dbname user=user password=password")
|
||||
conn_info = settings.get_connection_args()
|
||||
|
||||
assert conn_info["host"] == "localhost"
|
||||
assert conn_info["port"] == 5432
|
||||
assert conn_info["dbname"] == "dbname"
|
||||
assert conn_info["user"] == "user"
|
||||
assert conn_info["password"] == "password"
|
||||
|
||||
|
||||
def test_settings_env_connection_string(monkeypatch) -> None:
|
||||
monkeypatch.delenv("PGHOST", raising=False)
|
||||
monkeypatch.delenv("PGPORT", raising=False)
|
||||
monkeypatch.delenv("PGDATABASE", raising=False)
|
||||
monkeypatch.delenv("PGUSER", raising=False)
|
||||
monkeypatch.delenv("PGPASSWORD", raising=False)
|
||||
|
||||
monkeypatch.setenv(
|
||||
"POSTGRES_CONNECTION_STRING", "host=localhost port=5432 dbname=dbname user=user password=password"
|
||||
)
|
||||
|
||||
settings = PostgresSettings()
|
||||
conn_info = settings.get_connection_args()
|
||||
assert conn_info["host"] == "localhost"
|
||||
assert conn_info["port"] == 5432
|
||||
assert conn_info["dbname"] == "dbname"
|
||||
assert conn_info["user"] == "user"
|
||||
assert conn_info["password"] == "password"
|
||||
|
||||
|
||||
def test_settings_env_vars(monkeypatch) -> None:
|
||||
monkeypatch.setenv("PGHOST", "localhost")
|
||||
monkeypatch.setenv("PGPORT", "5432")
|
||||
monkeypatch.setenv("PGDATABASE", "dbname")
|
||||
monkeypatch.setenv("PGUSER", "user")
|
||||
monkeypatch.setenv("PGPASSWORD", "password")
|
||||
|
||||
settings = PostgresSettings()
|
||||
conn_info = settings.get_connection_args()
|
||||
assert conn_info["host"] == "localhost"
|
||||
assert conn_info["port"] == 5432
|
||||
assert conn_info["dbname"] == "dbname"
|
||||
assert conn_info["user"] == "user"
|
||||
assert conn_info["password"] == "password"
|
||||
|
||||
|
||||
# endregion
|
||||
@@ -0,0 +1,341 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from pytest import fixture, mark, raises
|
||||
from qdrant_client.async_qdrant_client import AsyncQdrantClient
|
||||
from qdrant_client.models import Datatype, Distance, FieldCondition, MatchValue, VectorParams
|
||||
|
||||
from semantic_kernel.connectors.qdrant import QdrantCollection, QdrantStore
|
||||
from semantic_kernel.data.vector import DistanceFunction, VectorStoreField
|
||||
from semantic_kernel.exceptions import (
|
||||
VectorSearchExecutionException,
|
||||
VectorStoreInitializationException,
|
||||
VectorStoreModelValidationError,
|
||||
VectorStoreOperationException,
|
||||
)
|
||||
|
||||
BASE_PATH = "qdrant_client.async_qdrant_client.AsyncQdrantClient"
|
||||
|
||||
|
||||
@fixture
|
||||
def vector_store(qdrant_unit_test_env):
|
||||
return QdrantStore(env_file_path="test.env")
|
||||
|
||||
|
||||
@fixture
|
||||
def collection(qdrant_unit_test_env, definition):
|
||||
return QdrantCollection(
|
||||
record_type=dict,
|
||||
collection_name="test",
|
||||
definition=definition,
|
||||
env_file_path="test.env",
|
||||
)
|
||||
|
||||
|
||||
@fixture
|
||||
def collection_without_named_vectors(qdrant_unit_test_env, definition):
|
||||
return QdrantCollection(
|
||||
record_type=dict,
|
||||
collection_name="test",
|
||||
definition=definition,
|
||||
named_vectors=False,
|
||||
env_file_path="test.env",
|
||||
)
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_list_collection_names():
|
||||
with patch(f"{BASE_PATH}.get_collections") as mock_get_collections:
|
||||
from qdrant_client.conversions.common_types import CollectionsResponse
|
||||
from qdrant_client.http.models import CollectionDescription
|
||||
|
||||
response = MagicMock(spec=CollectionsResponse)
|
||||
response.collections = [CollectionDescription(name="test")]
|
||||
mock_get_collections.return_value = response
|
||||
yield mock_get_collections
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_collection_exists():
|
||||
with patch(f"{BASE_PATH}.collection_exists") as mock_collection_exists:
|
||||
mock_collection_exists.return_value = True
|
||||
yield mock_collection_exists
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_ensure_collection_exists():
|
||||
with patch(f"{BASE_PATH}.create_collection") as mock_ensure_collection_exists:
|
||||
yield mock_ensure_collection_exists
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_ensure_collection_deleted():
|
||||
with patch(f"{BASE_PATH}.delete_collection") as mock_ensure_collection_deleted:
|
||||
mock_ensure_collection_deleted.return_value = True
|
||||
yield mock_ensure_collection_deleted
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_upsert():
|
||||
with patch(f"{BASE_PATH}.upsert") as mock_upsert:
|
||||
from qdrant_client.conversions.common_types import UpdateResult
|
||||
|
||||
result = MagicMock(spec=UpdateResult)
|
||||
result.status = "completed"
|
||||
mock_upsert.return_value = result
|
||||
yield mock_upsert
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_get(collection):
|
||||
with patch(f"{BASE_PATH}.retrieve") as mock_retrieve:
|
||||
from qdrant_client.http.models import Record
|
||||
|
||||
if collection.named_vectors:
|
||||
mock_retrieve.return_value = [
|
||||
Record(id="id1", payload={"content": "content"}, vector={"vector": [1.0, 2.0, 3.0]})
|
||||
]
|
||||
else:
|
||||
mock_retrieve.return_value = [Record(id="id1", payload={"content": "content"}, vector=[1.0, 2.0, 3.0])]
|
||||
yield mock_retrieve
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_delete():
|
||||
with patch(f"{BASE_PATH}.delete") as mock_delete:
|
||||
yield mock_delete
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_search():
|
||||
with patch(f"{BASE_PATH}.search") as mock_search:
|
||||
from qdrant_client.models import ScoredPoint
|
||||
|
||||
response1 = ScoredPoint(id="id1", version=1, score=0.0, payload={"content": "content"})
|
||||
response2 = ScoredPoint(id="id2", version=1, score=0.0, payload={"content": "content"})
|
||||
mock_search.return_value = [response1, response2]
|
||||
yield mock_search
|
||||
|
||||
|
||||
async def test_vector_store_defaults(vector_store):
|
||||
async with vector_store:
|
||||
assert vector_store.qdrant_client is not None
|
||||
assert vector_store.qdrant_client._client.rest_uri == "http://localhost:6333"
|
||||
|
||||
|
||||
def test_vector_store_with_client():
|
||||
qdrant_store = QdrantStore(client=AsyncQdrantClient())
|
||||
assert qdrant_store.qdrant_client is not None
|
||||
assert qdrant_store.qdrant_client._client.rest_uri == "http://localhost:6333"
|
||||
|
||||
|
||||
@mark.parametrize("exclude_list", [["QDRANT_LOCATION"]], indirect=True)
|
||||
def test_vector_store_in_memory(qdrant_unit_test_env):
|
||||
from qdrant_client.local.async_qdrant_local import AsyncQdrantLocal
|
||||
|
||||
qdrant_store = QdrantStore(api_key="supersecretkey", env_file_path="test.env")
|
||||
assert qdrant_store.qdrant_client is not None
|
||||
assert isinstance(qdrant_store.qdrant_client._client, AsyncQdrantLocal)
|
||||
assert qdrant_store.qdrant_client._client.location == ":memory:"
|
||||
|
||||
|
||||
def test_vector_store_fail():
|
||||
with raises(VectorStoreInitializationException, match="Failed to create Qdrant settings."):
|
||||
QdrantStore(location="localhost", url="localhost", env_file_path="test.env")
|
||||
|
||||
with raises(VectorStoreInitializationException, match="Failed to create Qdrant client."):
|
||||
QdrantStore(location="localhost", url="http://localhost", env_file_path="test.env")
|
||||
|
||||
|
||||
async def test_store_list_collection_names(vector_store):
|
||||
collections = await vector_store.list_collection_names()
|
||||
assert collections == ["test"]
|
||||
|
||||
|
||||
def test_get_collection(vector_store: QdrantStore, definition, qdrant_unit_test_env):
|
||||
collection = vector_store.get_collection(collection_name="test", record_type=dict, definition=definition)
|
||||
assert collection.collection_name == "test"
|
||||
assert collection.qdrant_client == vector_store.qdrant_client
|
||||
assert collection.record_type is dict
|
||||
assert collection.definition == definition
|
||||
|
||||
|
||||
async def test_collection_init(definition, qdrant_unit_test_env):
|
||||
async with QdrantCollection(
|
||||
record_type=dict,
|
||||
collection_name="test",
|
||||
definition=definition,
|
||||
env_file_path="test.env",
|
||||
) as collection:
|
||||
assert collection.collection_name == "test"
|
||||
assert collection.qdrant_client is not None
|
||||
assert collection.record_type is dict
|
||||
assert collection.definition == definition
|
||||
assert collection.named_vectors
|
||||
|
||||
|
||||
def test_collection_init_fail(definition):
|
||||
with raises(VectorStoreInitializationException, match="Failed to create Qdrant settings."):
|
||||
QdrantCollection(
|
||||
record_type=dict,
|
||||
collection_name="test",
|
||||
definition=definition,
|
||||
url="localhost",
|
||||
env_file_path="test.env",
|
||||
)
|
||||
with raises(VectorStoreInitializationException, match="Failed to create Qdrant client."):
|
||||
QdrantCollection(
|
||||
record_type=dict,
|
||||
collection_name="test",
|
||||
definition=definition,
|
||||
location="localhost",
|
||||
url="http://localhost",
|
||||
env_file_path="test.env",
|
||||
)
|
||||
with raises(
|
||||
VectorStoreModelValidationError, match="Only one vector field is allowed when not using named vectors."
|
||||
):
|
||||
definition.fields.append(VectorStoreField("vector", name="vector2", dimensions=3))
|
||||
QdrantCollection(
|
||||
record_type=dict,
|
||||
collection_name="test",
|
||||
definition=definition,
|
||||
named_vectors=False,
|
||||
env_file_path="test.env",
|
||||
)
|
||||
|
||||
|
||||
@mark.parametrize("collection_to_use", ["collection", "collection_without_named_vectors"])
|
||||
async def test_upsert(collection_to_use, request):
|
||||
from qdrant_client.models import PointStruct
|
||||
|
||||
collection = request.getfixturevalue(collection_to_use)
|
||||
if collection.named_vectors:
|
||||
record = PointStruct(id="id1", payload={"content": "content"}, vector={"vector": [1.0, 2.0, 3.0]})
|
||||
else:
|
||||
record = PointStruct(id="id1", payload={"content": "content"}, vector=[1.0, 2.0, 3.0])
|
||||
ids = await collection._inner_upsert([record])
|
||||
assert ids[0] == "id1"
|
||||
|
||||
ids = await collection.upsert(records={"id": "id1", "content": "content", "vector": [1.0, 2.0, 3.0]})
|
||||
assert ids == "id1"
|
||||
|
||||
|
||||
async def test_get(collection):
|
||||
records = await collection._inner_get(["id1"])
|
||||
assert records is not None
|
||||
|
||||
records = await collection.get("id1")
|
||||
assert records is not None
|
||||
|
||||
|
||||
async def test_delete(collection):
|
||||
await collection._inner_delete(["id1"])
|
||||
|
||||
|
||||
async def test_collection_exists(collection):
|
||||
await collection.collection_exists()
|
||||
|
||||
|
||||
async def test_ensure_collection_deleted(collection):
|
||||
await collection.ensure_collection_deleted()
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"collection_to_use, results",
|
||||
[
|
||||
(
|
||||
"collection",
|
||||
{
|
||||
"collection_name": "test",
|
||||
"vectors_config": {"vector": VectorParams(size=5, distance=Distance.COSINE, datatype=Datatype.FLOAT32)},
|
||||
},
|
||||
),
|
||||
(
|
||||
"collection_without_named_vectors",
|
||||
{
|
||||
"collection_name": "test",
|
||||
"vectors_config": VectorParams(size=5, distance=Distance.COSINE, datatype=Datatype.FLOAT32),
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_create_index_with_named_vectors(collection_to_use, results, mock_ensure_collection_exists, request):
|
||||
await request.getfixturevalue(collection_to_use).ensure_collection_exists()
|
||||
mock_ensure_collection_exists.assert_called_once_with(**results)
|
||||
|
||||
|
||||
@mark.parametrize("collection_to_use", ["collection", "collection_without_named_vectors"])
|
||||
async def test_create_index_fail(collection_to_use, request):
|
||||
collection = request.getfixturevalue(collection_to_use)
|
||||
for field in collection.definition.vector_fields:
|
||||
field.distance_function = DistanceFunction.HAMMING
|
||||
with raises(VectorStoreOperationException):
|
||||
await collection.ensure_collection_exists()
|
||||
|
||||
|
||||
async def test_search(collection, mock_search):
|
||||
collection.named_vectors = False
|
||||
results = await collection.search(vector=[1.0, 2.0, 3.0], include_vectors=False)
|
||||
async for result in results.results:
|
||||
assert result.record["id"] == "id1"
|
||||
break
|
||||
|
||||
assert mock_search.call_count == 1
|
||||
mock_search.assert_called_with(
|
||||
collection_name="test",
|
||||
query_vector=[1.0, 2.0, 3.0],
|
||||
query_filter=None,
|
||||
with_vectors=False,
|
||||
limit=3,
|
||||
offset=0,
|
||||
)
|
||||
|
||||
|
||||
async def test_search_named_vectors(collection, mock_search):
|
||||
collection.named_vectors = True
|
||||
results = await collection.search(
|
||||
vector=[1.0, 2.0, 3.0],
|
||||
vector_property_name="vector",
|
||||
include_vectors=False,
|
||||
)
|
||||
async for result in results.results:
|
||||
assert result.record["id"] == "id1"
|
||||
break
|
||||
|
||||
assert mock_search.call_count == 1
|
||||
mock_search.assert_called_with(
|
||||
collection_name="test",
|
||||
query_vector=("vector", [1.0, 2.0, 3.0]),
|
||||
query_filter=None,
|
||||
with_vectors=False,
|
||||
limit=3,
|
||||
offset=0,
|
||||
)
|
||||
|
||||
|
||||
async def test_search_filter(collection, mock_search):
|
||||
results = await collection.search(
|
||||
vector=[1.0, 2.0, 3.0],
|
||||
include_vectors=False,
|
||||
filter=lambda x: x.id == "id1",
|
||||
)
|
||||
async for result in results.results:
|
||||
assert result.record["id"] == "id1"
|
||||
break
|
||||
|
||||
assert mock_search.call_count == 1
|
||||
mock_search.assert_called_with(
|
||||
collection_name="test",
|
||||
query_vector=("vector", [1.0, 2.0, 3.0]),
|
||||
query_filter=FieldCondition(key="id", match=MatchValue(value="id1")),
|
||||
with_vectors=False,
|
||||
limit=3,
|
||||
offset=0,
|
||||
)
|
||||
|
||||
|
||||
async def test_search_fail(collection):
|
||||
with raises(VectorSearchExecutionException, match="Search requires a vector."):
|
||||
await collection.search(include_vectors=False)
|
||||
@@ -0,0 +1,308 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import numpy as np
|
||||
from pytest import fixture, mark, raises
|
||||
from redis.asyncio.client import Redis
|
||||
|
||||
from semantic_kernel.connectors.redis import (
|
||||
RedisCollectionTypes,
|
||||
RedisHashsetCollection,
|
||||
RedisJsonCollection,
|
||||
RedisStore,
|
||||
)
|
||||
from semantic_kernel.exceptions import VectorStoreInitializationException, VectorStoreOperationException
|
||||
|
||||
BASE_PATH = "redis.asyncio.client.Redis"
|
||||
BASE_PATH_FT = "redis.commands.search.AsyncSearch"
|
||||
BASE_PATH_JSON = "redis.commands.json.commands.JSONCommands"
|
||||
|
||||
|
||||
@fixture
|
||||
def vector_store(redis_unit_test_env):
|
||||
return RedisStore(env_file_path="test.env")
|
||||
|
||||
|
||||
@fixture
|
||||
def collection_hash(redis_unit_test_env, definition):
|
||||
return RedisHashsetCollection(
|
||||
record_type=dict,
|
||||
collection_name="test",
|
||||
definition=definition,
|
||||
env_file_path="test.env",
|
||||
)
|
||||
|
||||
|
||||
@fixture
|
||||
def collection_json(redis_unit_test_env, definition):
|
||||
return RedisJsonCollection(
|
||||
record_type=dict,
|
||||
collection_name="test",
|
||||
definition=definition,
|
||||
env_file_path="test.env",
|
||||
)
|
||||
|
||||
|
||||
@fixture
|
||||
def collection_with_prefix_hash(redis_unit_test_env, definition):
|
||||
return RedisHashsetCollection(
|
||||
record_type=dict,
|
||||
collection_name="test",
|
||||
definition=definition,
|
||||
prefix_collection_name_to_key_names=True,
|
||||
env_file_path="test.env",
|
||||
)
|
||||
|
||||
|
||||
@fixture
|
||||
def collection_with_prefix_json(redis_unit_test_env, definition):
|
||||
return RedisJsonCollection(
|
||||
record_type=dict,
|
||||
collection_name="test",
|
||||
definition=definition,
|
||||
prefix_collection_name_to_key_names=True,
|
||||
env_file_path="test.env",
|
||||
)
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def moc_list_collection_names():
|
||||
with patch(f"{BASE_PATH}.execute_command") as mock_get_collections:
|
||||
mock_get_collections.return_value = [b"test"]
|
||||
yield mock_get_collections
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_collection_exists():
|
||||
with patch(f"{BASE_PATH_FT}.info", new=AsyncMock()) as mock_collection_exists:
|
||||
mock_collection_exists.return_value = True
|
||||
yield mock_collection_exists
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_ensure_collection_exists():
|
||||
with patch(f"{BASE_PATH_FT}.create_index", new=AsyncMock()) as mock_reensure_collection_exists:
|
||||
yield mock_reensure_collection_exists
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_ensure_collection_deleted():
|
||||
with patch(f"{BASE_PATH_FT}.dropindex", new=AsyncMock()) as mock_ensure_collection_deleted:
|
||||
yield mock_ensure_collection_deleted
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_upsert_hash():
|
||||
with patch(f"{BASE_PATH}.hset", new=AsyncMock()) as mock_upsert:
|
||||
yield mock_upsert
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_upsert_json():
|
||||
with patch(f"{BASE_PATH_JSON}.set", new=AsyncMock()) as mock_upsert:
|
||||
yield mock_upsert
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_get_hash():
|
||||
with patch(f"{BASE_PATH}.hgetall", new=AsyncMock()) as mock_get:
|
||||
mock_get.return_value = {
|
||||
b"content": b"content",
|
||||
b"vector": np.array([1.0, 2.0, 3.0]).tobytes(),
|
||||
}
|
||||
yield mock_get
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_get_json():
|
||||
with patch(f"{BASE_PATH_JSON}.mget", new=AsyncMock()) as mock_get:
|
||||
mock_get.return_value = [
|
||||
[
|
||||
{
|
||||
"content": "content",
|
||||
"vector": [1.0, 2.0, 3.0],
|
||||
}
|
||||
]
|
||||
]
|
||||
yield mock_get
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_delete_hash():
|
||||
with patch(f"{BASE_PATH}.delete", new=AsyncMock()) as mock_delete:
|
||||
yield mock_delete
|
||||
|
||||
|
||||
@fixture(autouse=True)
|
||||
def mock_delete_json():
|
||||
with patch(f"{BASE_PATH_JSON}.delete", new=AsyncMock()) as mock_delete:
|
||||
yield mock_delete
|
||||
|
||||
|
||||
def test_vector_store_defaults(vector_store):
|
||||
assert vector_store.redis_database is not None
|
||||
assert vector_store.redis_database.connection_pool.connection_kwargs["host"] == "localhost"
|
||||
|
||||
|
||||
def test_vector_store_with_client(redis_unit_test_env):
|
||||
vector_store = RedisStore(redis_database=Redis.from_url(redis_unit_test_env["REDIS_CONNECTION_STRING"]))
|
||||
assert vector_store.redis_database is not None
|
||||
assert vector_store.redis_database.connection_pool.connection_kwargs["host"] == "localhost"
|
||||
|
||||
|
||||
@mark.parametrize("exclude_list", [["REDIS_CONNECTION_STRING"]], indirect=True)
|
||||
def test_vector_store_fail(redis_unit_test_env):
|
||||
with raises(VectorStoreInitializationException, match="Failed to create Redis settings."):
|
||||
RedisStore(env_file_path="test.env")
|
||||
|
||||
|
||||
async def test_store_list_collection_names(vector_store, moc_list_collection_names):
|
||||
collections = await vector_store.list_collection_names()
|
||||
assert collections == ["test"]
|
||||
|
||||
|
||||
@mark.parametrize("type_", ["hashset", "json"])
|
||||
def test_get_collection(vector_store, definition, type_):
|
||||
if type_ == "hashset":
|
||||
collection = vector_store.get_collection(
|
||||
collection_name="test",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
collection_type=RedisCollectionTypes.HASHSET,
|
||||
)
|
||||
assert isinstance(collection, RedisHashsetCollection)
|
||||
else:
|
||||
collection = vector_store.get_collection(
|
||||
collection_name="test",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
collection_type=RedisCollectionTypes.JSON,
|
||||
)
|
||||
assert isinstance(collection, RedisJsonCollection)
|
||||
assert collection.collection_name == "test"
|
||||
assert collection.redis_database == vector_store.redis_database
|
||||
assert collection.record_type is dict
|
||||
assert collection.definition == definition
|
||||
|
||||
|
||||
@mark.parametrize("type_", ["hashset", "json"])
|
||||
def test_collection_init(redis_unit_test_env, definition, type_):
|
||||
if type_ == "hashset":
|
||||
collection = RedisHashsetCollection(
|
||||
record_type=dict,
|
||||
collection_name="test",
|
||||
definition=definition,
|
||||
env_file_path="test.env",
|
||||
)
|
||||
else:
|
||||
collection = RedisJsonCollection(
|
||||
record_type=dict,
|
||||
collection_name="test",
|
||||
definition=definition,
|
||||
env_file_path="test.env",
|
||||
)
|
||||
assert collection.collection_name == "test"
|
||||
assert collection.redis_database is not None
|
||||
assert collection.record_type is dict
|
||||
assert collection.definition == definition
|
||||
assert collection.prefix_collection_name_to_key_names is False
|
||||
|
||||
|
||||
@mark.parametrize("type_", ["hashset", "json"])
|
||||
def test_init_with_type(redis_unit_test_env, record_type, type_):
|
||||
if type_ == "hashset":
|
||||
collection = RedisHashsetCollection(record_type=record_type, collection_name="test")
|
||||
else:
|
||||
collection = RedisJsonCollection(record_type=record_type, collection_name="test")
|
||||
assert collection is not None
|
||||
assert collection.record_type is record_type
|
||||
assert collection.collection_name == "test"
|
||||
|
||||
|
||||
@mark.parametrize("exclude_list", [["REDIS_CONNECTION_STRING"]], indirect=True)
|
||||
def test_collection_fail(redis_unit_test_env, definition):
|
||||
with raises(VectorStoreInitializationException, match="Failed to create Redis settings."):
|
||||
RedisHashsetCollection(
|
||||
record_type=dict,
|
||||
collection_name="test",
|
||||
definition=definition,
|
||||
env_file_path="test.env",
|
||||
)
|
||||
with raises(VectorStoreInitializationException, match="Failed to create Redis settings."):
|
||||
RedisJsonCollection(
|
||||
record_type=dict,
|
||||
collection_name="test",
|
||||
definition=definition,
|
||||
env_file_path="test.env",
|
||||
)
|
||||
|
||||
|
||||
@mark.parametrize("type_", ["hashset", "json"])
|
||||
async def test_upsert(collection_hash, collection_json, type_):
|
||||
collection = collection_hash if type_ == "hashset" else collection_json
|
||||
ids = await collection.upsert(records={"id": "id1", "content": "content", "vector": [1.0, 2.0, 3.0]})
|
||||
assert ids == "id1"
|
||||
|
||||
|
||||
async def test_upsert_with_prefix(collection_with_prefix_hash, collection_with_prefix_json):
|
||||
ids = await collection_with_prefix_hash.upsert(
|
||||
records={"id": "id1", "content": "content", "vector": [1.0, 2.0, 3.0]}
|
||||
)
|
||||
assert ids == "id1"
|
||||
ids = await collection_with_prefix_json.upsert(
|
||||
records={"id": "id1", "content": "content", "vector": [1.0, 2.0, 3.0]}
|
||||
)
|
||||
assert ids == "id1"
|
||||
|
||||
|
||||
@mark.parametrize("prefix", [True, False])
|
||||
@mark.parametrize("type_", ["hashset", "json"])
|
||||
async def test_get(
|
||||
collection_hash, collection_json, collection_with_prefix_hash, collection_with_prefix_json, type_, prefix
|
||||
):
|
||||
if prefix:
|
||||
collection = collection_with_prefix_hash if type_ == "hashset" else collection_with_prefix_json
|
||||
else:
|
||||
collection = collection_hash if type_ == "hashset" else collection_json
|
||||
|
||||
records = await collection.get("id1")
|
||||
assert records is not None
|
||||
|
||||
|
||||
@mark.parametrize("type_", ["hashset", "json"])
|
||||
async def test_delete(collection_hash, collection_json, type_):
|
||||
collection = collection_hash if type_ == "hashset" else collection_json
|
||||
await collection._inner_delete(["id1"])
|
||||
|
||||
|
||||
async def test_collection_exists(collection_hash, mock_collection_exists):
|
||||
await collection_hash.collection_exists()
|
||||
|
||||
|
||||
async def test_collection_exists_false(collection_hash, mock_collection_exists):
|
||||
mock_collection_exists.side_effect = Exception
|
||||
exists = await collection_hash.collection_exists()
|
||||
assert not exists
|
||||
|
||||
|
||||
async def test_ensure_collection_deleted(collection_hash, mock_ensure_collection_deleted):
|
||||
await collection_hash.ensure_collection_deleted()
|
||||
await collection_hash.ensure_collection_deleted()
|
||||
|
||||
|
||||
async def test_create_index(collection_hash, mock_ensure_collection_exists):
|
||||
await collection_hash.ensure_collection_exists()
|
||||
|
||||
|
||||
async def test_create_index_manual(collection_hash, mock_ensure_collection_exists):
|
||||
from redis.commands.search.index_definition import IndexDefinition, IndexType
|
||||
|
||||
fields = ["fields"]
|
||||
index_definition = IndexDefinition(prefix="test:", index_type=IndexType.HASH)
|
||||
await collection_hash.ensure_collection_exists(index_definition=index_definition, fields=fields)
|
||||
|
||||
|
||||
async def test_create_index_fail(collection_hash, mock_ensure_collection_exists):
|
||||
with raises(VectorStoreOperationException, match="Invalid index type supplied."):
|
||||
await collection_hash.ensure_collection_exists(index_definition="index_definition", fields="fields")
|
||||
@@ -0,0 +1,601 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from typing import NamedTuple
|
||||
from unittest.mock import AsyncMock, MagicMock, NonCallableMagicMock, patch
|
||||
|
||||
from pytest import fixture, mark, param, raises
|
||||
|
||||
from semantic_kernel.connectors.sql_server import (
|
||||
QueryBuilder,
|
||||
SqlCommand,
|
||||
SqlServerCollection,
|
||||
SqlServerStore,
|
||||
_build_create_table_query,
|
||||
_build_delete_query,
|
||||
_build_delete_table_query,
|
||||
_build_merge_query,
|
||||
_build_search_query,
|
||||
_build_select_query,
|
||||
_build_select_table_names_query,
|
||||
)
|
||||
from semantic_kernel.data.vector import DistanceFunction, IndexKind, VectorSearchOptions, VectorStoreField
|
||||
from semantic_kernel.exceptions.vector_store_exceptions import (
|
||||
VectorStoreInitializationException,
|
||||
VectorStoreOperationException,
|
||||
)
|
||||
|
||||
|
||||
class TestQueryBuilder:
|
||||
def test_query_builder_append(self):
|
||||
qb = QueryBuilder()
|
||||
qb.append("SELECT * FROM")
|
||||
qb.append(" table", suffix=";")
|
||||
result = str(qb).strip()
|
||||
assert result == "SELECT * FROM table;"
|
||||
|
||||
def test_query_builder_append_list(self):
|
||||
qb = QueryBuilder()
|
||||
qb.append_list(["id", "name", "age"], sep=", ", suffix=";")
|
||||
result = str(qb).strip()
|
||||
assert result == "id, name, age;"
|
||||
|
||||
def test_query_builder_escape_identifier(self):
|
||||
assert QueryBuilder.escape_identifier("simple") == "[simple]"
|
||||
assert QueryBuilder.escape_identifier("has]bracket") == "[has]]bracket]"
|
||||
assert QueryBuilder.escape_identifier("two]]brackets") == "[two]]]]brackets]"
|
||||
assert QueryBuilder.escape_identifier("") == "[]"
|
||||
|
||||
def test_query_builder_append_table_name(self):
|
||||
qb = QueryBuilder()
|
||||
qb.append_table_name("dbo", "Users", prefix="SELECT * FROM", suffix=";", newline=False)
|
||||
result = str(qb).strip()
|
||||
assert result == "SELECT * FROM [dbo].[Users] ;"
|
||||
|
||||
def test_query_builder_append_table_name_escapes_closing_bracket(self):
|
||||
qb = QueryBuilder()
|
||||
qb.append_table_name("my]schema", "my]table", prefix="SELECT * FROM", suffix=";")
|
||||
result = str(qb).strip()
|
||||
assert result == "SELECT * FROM [my]]schema].[my]]table] ;"
|
||||
|
||||
def test_query_builder_append_table_name_prevents_sql_injection(self):
|
||||
qb = QueryBuilder()
|
||||
qb.append("DROP TABLE IF EXISTS")
|
||||
qb.append_table_name("dbo", "]; EXEC xp_cmdshell('whoami'); --", suffix=";")
|
||||
result = str(qb)
|
||||
assert "EXEC xp_cmdshell" not in result.split("].[")[0], "SQL injection should not escape bracket quoting"
|
||||
assert "[dbo].[]]; EXEC xp_cmdshell('whoami'); --]" in result
|
||||
|
||||
def test_query_builder_remove_last(self):
|
||||
qb = QueryBuilder("SELECT * FROM table;")
|
||||
qb.remove_last(1) # remove trailing semicolon
|
||||
result = str(qb).strip()
|
||||
assert result == "SELECT * FROM table"
|
||||
|
||||
def test_query_builder_in_parenthesis(self):
|
||||
qb = QueryBuilder("INSERT INTO table")
|
||||
with qb.in_parenthesis():
|
||||
qb.append("id, name, age")
|
||||
result = str(qb).strip()
|
||||
assert result == "INSERT INTO table (id, name, age)"
|
||||
|
||||
def test_query_builder_in_parenthesis_with_prefix_suffix(self):
|
||||
qb = QueryBuilder()
|
||||
with qb.in_parenthesis(prefix="VALUES", suffix=";"):
|
||||
qb.append_list(["1", "'John'", "30"])
|
||||
result = str(qb).strip()
|
||||
assert result == "VALUES (1, 'John', 30) ;"
|
||||
|
||||
def test_query_builder_in_logical_group(self):
|
||||
qb = QueryBuilder()
|
||||
with qb.in_logical_group():
|
||||
qb.append("UPDATE Users SET name = 'John'")
|
||||
result = str(qb).strip()
|
||||
lines = result.splitlines()
|
||||
assert lines[0] == "BEGIN"
|
||||
assert lines[1] == "UPDATE Users SET name = 'John'"
|
||||
assert lines[2] == "END"
|
||||
|
||||
|
||||
class TestSqlCommand:
|
||||
def test_sql_command_initial_query(self):
|
||||
cmd = SqlCommand("SELECT 1")
|
||||
assert str(cmd.query) == "SELECT 1"
|
||||
|
||||
def test_sql_command_add_parameter(self):
|
||||
cmd = SqlCommand("SELECT * FROM Test WHERE id = ?")
|
||||
cmd.add_parameter("42")
|
||||
assert cmd.parameters[0] == "42"
|
||||
|
||||
def test_sql_command_add_parameters(self):
|
||||
cmd = SqlCommand("SELECT * FROM Test WHERE id = ?")
|
||||
cmd.add_parameters(["42", "43"])
|
||||
assert cmd.parameters[0] == "42"
|
||||
assert cmd.parameters[1] == "43"
|
||||
|
||||
def test_parameter_limit(self):
|
||||
cmd = SqlCommand()
|
||||
cmd.add_parameters(["42"] * 2100)
|
||||
with raises(VectorStoreOperationException):
|
||||
cmd.add_parameter("43")
|
||||
with raises(VectorStoreOperationException):
|
||||
cmd.add_parameters(["43", "44"])
|
||||
|
||||
|
||||
class TestQueryBuildFunctions:
|
||||
def test_build_create_table_query(self):
|
||||
schema = "dbo"
|
||||
table = "Test"
|
||||
key_field = VectorStoreField("key", name="id", type="str")
|
||||
data_fields = [
|
||||
VectorStoreField("data", name="name", type="str"),
|
||||
VectorStoreField("data", name="age", type="int"),
|
||||
]
|
||||
vector_fields = [
|
||||
VectorStoreField("vector", name="embedding", type="float", dimensions=1536),
|
||||
]
|
||||
cmd = _build_create_table_query(schema, table, key_field, data_fields, vector_fields)
|
||||
assert not cmd.parameters
|
||||
cmd_str = str(cmd.query)
|
||||
assert (
|
||||
cmd_str
|
||||
== "BEGIN\nCREATE TABLE [dbo].[Test] \n ([id] nvarchar(255) NOT NULL,\n[name] nvarchar(max) NULL,\n[age] "
|
||||
"int NULL,\n[embedding] VECTOR(1536) NULL,\nPRIMARY KEY ([id]) \n) ;\nEND\n"
|
||||
)
|
||||
|
||||
def test_build_create_table_query_escapes_single_quote_in_object_id(self):
|
||||
key_field = VectorStoreField("key", name="id", type="str")
|
||||
cmd = _build_create_table_query("dbo", "test'table", key_field, [], [], if_not_exists=True)
|
||||
cmd_str = str(cmd.query)
|
||||
# Single quote must be escaped inside the OBJECT_ID N'...' string literal
|
||||
assert "OBJECT_ID(N'[dbo].[test''table]'" in cmd_str
|
||||
|
||||
def test_build_create_table_query_uses_storage_name_for_primary_key(self):
|
||||
key_field = VectorStoreField("key", name="id", type="str", storage_name="pk_id")
|
||||
cmd = _build_create_table_query("dbo", "Test", key_field, [], [])
|
||||
cmd_str = str(cmd.query)
|
||||
assert "[pk_id] nvarchar" in cmd_str
|
||||
assert "PRIMARY KEY ([pk_id])" in cmd_str
|
||||
|
||||
def test_build_merge_query_output_uses_storage_name(self):
|
||||
key_field = VectorStoreField("key", name="id", type="str", storage_name="pk_id")
|
||||
records = [{"pk_id": "1"}]
|
||||
cmd = _build_merge_query("dbo", "Test", key_field, [], [], records)
|
||||
cmd_str = str(cmd.query)
|
||||
assert "OUTPUT inserted.[pk_id] INTO @UpsertedKeys" in cmd_str
|
||||
|
||||
def test_delete_table_query(self):
|
||||
schema = "dbo"
|
||||
table = "Test"
|
||||
cmd = _build_delete_table_query(schema, table)
|
||||
assert str(cmd.query) == f"DROP TABLE IF EXISTS [{schema}].[{table}] ;"
|
||||
|
||||
@mark.parametrize("schema", ["dbo", None])
|
||||
def test_build_select_table_names_query(self, schema):
|
||||
cmd = _build_select_table_names_query(schema)
|
||||
if schema:
|
||||
assert cmd.parameters == [schema]
|
||||
assert str(cmd) == (
|
||||
"SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES "
|
||||
"WHERE TABLE_TYPE = 'BASE TABLE' "
|
||||
"AND (@schema is NULL or TABLE_SCHEMA = ?);"
|
||||
)
|
||||
else:
|
||||
assert str(cmd) == "SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE = 'BASE TABLE';"
|
||||
|
||||
def test_build_merge_query(self):
|
||||
schema = "dbo"
|
||||
table = "Test"
|
||||
key_field = VectorStoreField("key", name="id", type="str")
|
||||
data_fields = [
|
||||
VectorStoreField("data", name="name", type="str"),
|
||||
VectorStoreField("data", name="age", type="int"),
|
||||
]
|
||||
vector_fields = [
|
||||
VectorStoreField("vector", name="embedding", type="float", dimensions=5),
|
||||
]
|
||||
records = [
|
||||
{
|
||||
"id": "test",
|
||||
"name": "name",
|
||||
"age": 50,
|
||||
"embedding": [0.1, 0.2, 0.3, 0.4, 0.5],
|
||||
}
|
||||
]
|
||||
cmd = _build_merge_query(schema, table, key_field, data_fields, vector_fields, records)
|
||||
assert cmd.parameters[0] == records[0]["id"]
|
||||
assert cmd.parameters[1] == records[0]["name"]
|
||||
assert cmd.parameters[2] == str(records[0]["age"])
|
||||
assert cmd.parameters[3] == json.dumps(records[0]["embedding"])
|
||||
str_cmd = str(cmd)
|
||||
assert str_cmd == (
|
||||
"DECLARE @UpsertedKeys TABLE (KeyColumn nvarchar(255));\n"
|
||||
"MERGE INTO [dbo].[Test] AS t\n"
|
||||
"USING ( VALUES (?, ?, ?, ?) ) AS s ([id], [name], [age], [embedding]) "
|
||||
"ON (t.[id] = s.[id]) \n"
|
||||
"WHEN MATCHED THEN\n"
|
||||
"UPDATE SET t.[name] = s.[name], t.[age] = s.[age], t.[embedding] = s.[embedding]\n"
|
||||
"WHEN NOT MATCHED THEN\n"
|
||||
"INSERT ([id], [name], [age], [embedding]) "
|
||||
"VALUES (s.[id], s.[name], s.[age], s.[embedding]) \n"
|
||||
"OUTPUT inserted.[id] INTO @UpsertedKeys (KeyColumn);\n"
|
||||
"SELECT KeyColumn FROM @UpsertedKeys;\n"
|
||||
)
|
||||
|
||||
def test_build_select_query(self):
|
||||
schema = "dbo"
|
||||
table = "Test"
|
||||
key_field = VectorStoreField("key", name="id", type="str")
|
||||
data_fields = [
|
||||
VectorStoreField("data", name="name", type="str"),
|
||||
VectorStoreField("data", name="age", type="int"),
|
||||
]
|
||||
vector_fields = [
|
||||
VectorStoreField("vector", name="embedding", type="float", dimensions=5),
|
||||
]
|
||||
keys = ["test"]
|
||||
cmd = _build_select_query(schema, table, key_field, data_fields, vector_fields, keys)
|
||||
assert cmd.parameters == ["test"]
|
||||
str_cmd = str(cmd)
|
||||
assert str_cmd == "SELECT\n[id], [name], [age], [embedding] FROM [dbo].[Test] \nWHERE [id] IN\n (?) ;"
|
||||
|
||||
def test_build_delete_query(self):
|
||||
schema = "dbo"
|
||||
table = "Test"
|
||||
key_field = VectorStoreField("key", name="id", type="str")
|
||||
keys = ["test"]
|
||||
cmd = _build_delete_query(schema, table, key_field, keys)
|
||||
str_cmd = str(cmd)
|
||||
assert cmd.parameters[0] == "test"
|
||||
assert str_cmd == "DELETE FROM [dbo].[Test] WHERE [id] IN (?) ;"
|
||||
|
||||
def test_build_search_query(self):
|
||||
schema = "dbo"
|
||||
table = "Test"
|
||||
key_field = VectorStoreField("key", name="id", type="str")
|
||||
data_fields = [
|
||||
VectorStoreField("data", name="name", type="str"),
|
||||
VectorStoreField("data", name="age", type="int"),
|
||||
]
|
||||
vector_fields = [
|
||||
VectorStoreField(
|
||||
"vector",
|
||||
name="embedding",
|
||||
type="float",
|
||||
dimensions=5,
|
||||
distance_function=DistanceFunction.COSINE_DISTANCE,
|
||||
),
|
||||
]
|
||||
vector = [0.1, 0.2, 0.3, 0.4, 0.5]
|
||||
options = VectorSearchOptions(
|
||||
vector_property_name="embedding",
|
||||
)
|
||||
|
||||
cmd = _build_search_query(schema, table, key_field, data_fields, vector_fields, vector, options)
|
||||
assert cmd.parameters[0] == json.dumps(vector)
|
||||
str_cmd = str(cmd)
|
||||
assert (
|
||||
str_cmd == "SELECT [id], [name], [age], VECTOR_DISTANCE('cosine', [embedding], CAST(? AS VECTOR(5))) as "
|
||||
"_vector_distance_value\n FROM [dbo].[Test] \nORDER BY "
|
||||
"_vector_distance_value ASC\nOFFSET 0 ROWS FETCH NEXT 3 ROWS ONLY;"
|
||||
)
|
||||
|
||||
|
||||
@fixture
|
||||
async def mock_connection(*args, **kwargs):
|
||||
return MagicMock()
|
||||
|
||||
|
||||
@mark.parametrize(
|
||||
"connection_string",
|
||||
[
|
||||
param(
|
||||
"Driver={ODBC Driver 18 for SQL Server};Server=localhost;Database=testdb;uid=testuserLongAsMax=yes;",
|
||||
id="with uid",
|
||||
),
|
||||
param(
|
||||
"Driver={ODBC Driver 18 for SQL Server};Server=localhost;Database=testdb;LongAsMax=yes;", id="credential"
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_get_mssql_connection(connection_string):
|
||||
mock_pyodbc = NonCallableMagicMock()
|
||||
sys.modules["pyodbc"] = mock_pyodbc
|
||||
|
||||
with patch("pyodbc.connect") as patched_connection:
|
||||
from azure.core.credentials_async import AsyncTokenCredential
|
||||
|
||||
from semantic_kernel.connectors.sql_server import SqlSettings, _get_mssql_connection
|
||||
|
||||
token = MagicMock()
|
||||
token.token.return_value = "test_token"
|
||||
token.token.encode.return_value = b"test_token"
|
||||
credential = AsyncMock(spec=AsyncTokenCredential)
|
||||
credential.__aenter__.return_value = credential
|
||||
credential.get_token.return_value = token
|
||||
|
||||
settings = SqlSettings(connection_string=connection_string)
|
||||
with patch("semantic_kernel.connectors.sql_server.AsyncTokenCredential", return_value=credential):
|
||||
connection = await _get_mssql_connection(settings, credential=credential)
|
||||
assert connection is not None
|
||||
assert isinstance(connection, MagicMock)
|
||||
if "uid" in connection_string:
|
||||
assert patched_connection.call_args.kwargs["attrs_before"] is None
|
||||
else:
|
||||
assert patched_connection.call_args.kwargs["attrs_before"] == {
|
||||
1256: b"\n\x00\x00\x00test_token",
|
||||
}
|
||||
|
||||
|
||||
class TestSqlServerStore:
|
||||
async def test_create_store(self, sql_server_unit_test_env):
|
||||
store = SqlServerStore()
|
||||
assert store is not None
|
||||
assert store.settings is not None
|
||||
assert store.settings.connection_string is not None
|
||||
assert "LongAsMax=yes;" in store.settings.connection_string.get_secret_value()
|
||||
|
||||
with patch("semantic_kernel.connectors.sql_server._get_mssql_connection") as mock_get_connection:
|
||||
mock_get_connection.return_value = AsyncMock()
|
||||
await store.__aenter__()
|
||||
assert store.connection is not None
|
||||
|
||||
@mark.parametrize(
|
||||
"override_env_param_dict",
|
||||
[
|
||||
{
|
||||
"SQL_SERVER_CONNECTION_STRING": "Driver={ODBC Driver 18 for SQL Server};Server=localhost;Database=testdb;User Id=testuser;Password=example;LongAsMax=yes;" # noqa: E501
|
||||
}
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
def test_create_store_with_long_as_max(self, sql_server_unit_test_env):
|
||||
store = SqlServerStore()
|
||||
assert store is not None
|
||||
assert store.settings is not None
|
||||
assert store.settings.connection_string is not None
|
||||
|
||||
@mark.parametrize("exclude_list", ["SQL_SERVER_CONNECTION_STRING"], indirect=True)
|
||||
def test_create_without_connection_string(self, sql_server_unit_test_env):
|
||||
with raises(VectorStoreInitializationException):
|
||||
SqlServerStore(env_file_path="test.env")
|
||||
|
||||
def test_get_collection(self, sql_server_unit_test_env, definition):
|
||||
store = SqlServerStore()
|
||||
collection = store.get_collection(collection_name="test", record_type=dict, definition=definition)
|
||||
assert collection is not None
|
||||
|
||||
async def test_list_collection_names(self, sql_server_unit_test_env, mock_connection):
|
||||
async with SqlServerStore(connection=mock_connection) as store:
|
||||
mock_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [
|
||||
["Test1"],
|
||||
["Test2"],
|
||||
]
|
||||
collection_names = await store.list_collection_names()
|
||||
assert collection_names == ["Test1", "Test2"]
|
||||
|
||||
async def test_no_connection(self, sql_server_unit_test_env):
|
||||
store = SqlServerStore()
|
||||
with raises(VectorStoreOperationException):
|
||||
await store.list_collection_names()
|
||||
|
||||
|
||||
class TestSqlServerCollection:
|
||||
@mark.parametrize("exclude_list", ["SQL_SERVER_CONNECTION_STRING"], indirect=True)
|
||||
def test_create_without_connection_string(self, sql_server_unit_test_env, definition):
|
||||
with raises(VectorStoreInitializationException):
|
||||
SqlServerCollection(
|
||||
collection_name="test",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
env_file_path="test.env",
|
||||
)
|
||||
|
||||
async def test_create(self, sql_server_unit_test_env, definition):
|
||||
collection = SqlServerCollection(collection_name="test", record_type=dict, definition=definition)
|
||||
assert collection is not None
|
||||
assert collection.collection_name == "test"
|
||||
assert collection.settings is not None
|
||||
assert collection.settings.connection_string is not None
|
||||
|
||||
with patch("semantic_kernel.connectors.sql_server._get_mssql_connection") as mock_get_connection:
|
||||
mock_get_connection.return_value = AsyncMock()
|
||||
await collection.__aenter__()
|
||||
assert collection.connection is not None
|
||||
|
||||
async def test_upsert(
|
||||
self,
|
||||
sql_server_unit_test_env,
|
||||
mock_connection,
|
||||
definition,
|
||||
):
|
||||
collection = SqlServerCollection(
|
||||
collection_name="test",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
connection=mock_connection,
|
||||
)
|
||||
record = {"id": "1", "content": "test", "vector": [0.1, 0.2, 0.3, 0.4, 0.5]}
|
||||
mock_connection.cursor.return_value.__enter__.return_value.nextset.side_effect = [True, False]
|
||||
mock_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [
|
||||
["1"],
|
||||
]
|
||||
await collection.upsert(record)
|
||||
mock_connection.cursor.return_value.__enter__.return_value.execute.assert_called_with(
|
||||
(
|
||||
"DECLARE @UpsertedKeys TABLE (KeyColumn nvarchar(255));\n"
|
||||
"MERGE INTO [dbo].[test] AS t\n"
|
||||
"USING ( VALUES (?, ?, ?) ) AS s ([id], [content], [vector]) "
|
||||
"ON (t.[id] = s.[id]) \n"
|
||||
"WHEN MATCHED THEN\n"
|
||||
"UPDATE SET t.[content] = s.[content], t.[vector] = s.[vector]\n"
|
||||
"WHEN NOT MATCHED THEN\n"
|
||||
"INSERT ([id], [content], [vector]) "
|
||||
"VALUES (s.[id], s.[content], s.[vector]) \n"
|
||||
"OUTPUT inserted.[id] INTO @UpsertedKeys (KeyColumn);\n"
|
||||
"SELECT KeyColumn FROM @UpsertedKeys;\n"
|
||||
),
|
||||
("1", "test", json.dumps([0.1, 0.2, 0.3, 0.4, 0.5])),
|
||||
)
|
||||
|
||||
async def test_get(
|
||||
self,
|
||||
sql_server_unit_test_env,
|
||||
mock_connection,
|
||||
definition,
|
||||
):
|
||||
class MockRow(NamedTuple):
|
||||
id: str
|
||||
content: str
|
||||
vector: str
|
||||
|
||||
mock_cursor = MagicMock()
|
||||
mock_connection.cursor.return_value.__enter__.return_value = mock_cursor
|
||||
|
||||
collection = SqlServerCollection(
|
||||
collection_name="test",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
connection=mock_connection,
|
||||
)
|
||||
key = "1"
|
||||
|
||||
row = MockRow("1", "test", "[0.1, 0.2, 0.3, 0.4, 0.5]")
|
||||
mock_cursor.description = [["id"], ["content"], ["vector"]]
|
||||
|
||||
mock_cursor.__iter__.return_value = [row]
|
||||
record = await collection.get(key, include_vectors=True)
|
||||
mock_cursor.execute.assert_called_with(
|
||||
"SELECT\n[id], [content], [vector] FROM [dbo].[test] \nWHERE [id] IN\n (?) ;", ("1",)
|
||||
)
|
||||
assert record["id"] == "1"
|
||||
assert record["content"] == "test"
|
||||
assert record["vector"] == [0.1, 0.2, 0.3, 0.4, 0.5]
|
||||
|
||||
async def test_delete(
|
||||
self,
|
||||
sql_server_unit_test_env,
|
||||
mock_connection,
|
||||
definition,
|
||||
):
|
||||
collection = SqlServerCollection(
|
||||
collection_name="test",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
connection=mock_connection,
|
||||
)
|
||||
key = "1"
|
||||
await collection.delete(key)
|
||||
mock_connection.cursor.return_value.__enter__.return_value.execute.assert_called_with(
|
||||
"DELETE FROM [dbo].[test] WHERE [id] IN (?) ;", ("1",)
|
||||
)
|
||||
|
||||
async def test_search(
|
||||
self,
|
||||
sql_server_unit_test_env,
|
||||
mock_connection,
|
||||
definition,
|
||||
):
|
||||
mock_cursor = MagicMock()
|
||||
mock_connection.cursor.return_value.__enter__.return_value = mock_cursor
|
||||
for field in definition.vector_fields:
|
||||
field.distance_function = DistanceFunction.COSINE_DISTANCE
|
||||
collection = SqlServerCollection(
|
||||
collection_name="test",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
connection=mock_connection,
|
||||
)
|
||||
vector = [0.1, 0.2, 0.3, 0.4, 0.5]
|
||||
|
||||
@dataclass
|
||||
class MockRow:
|
||||
id: str
|
||||
content: str
|
||||
_vector_distance_value: float
|
||||
|
||||
row = MockRow("1", "test", 0.1)
|
||||
mock_cursor.description = [["id"], ["content"], ["_vector_distance_value"]]
|
||||
|
||||
mock_cursor.__iter__.return_value = [row]
|
||||
search_result = await collection.search(
|
||||
vector=vector,
|
||||
vector_property_name="vector",
|
||||
filter=lambda x: x.content == "test",
|
||||
)
|
||||
async for record in search_result.results:
|
||||
assert record.record["id"] == "1"
|
||||
assert record.record["content"] == "test"
|
||||
assert record.score == 0.1
|
||||
mock_cursor.execute.assert_called_with(
|
||||
(
|
||||
"SELECT [id], [content], VECTOR_DISTANCE('cosine', [vector], CAST(? AS VECTOR(5))) as "
|
||||
"_vector_distance_value\n FROM [dbo].[test] \n WHERE [content] = ? \nORDER BY _vector_distance_value "
|
||||
"ASC\nOFFSET 0 ROWS FETCH NEXT 3 ROWS ONLY;"
|
||||
),
|
||||
(json.dumps(vector), "test"),
|
||||
)
|
||||
|
||||
async def test_ensure_collection_exists(
|
||||
self,
|
||||
sql_server_unit_test_env,
|
||||
mock_connection,
|
||||
definition,
|
||||
):
|
||||
for field in definition.vector_fields:
|
||||
field.index_kind = IndexKind.FLAT
|
||||
collection = SqlServerCollection(
|
||||
collection_name="test",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
connection=mock_connection,
|
||||
)
|
||||
await collection.ensure_collection_exists()
|
||||
mock_connection.cursor.return_value.__enter__.return_value.execute.assert_called_with(
|
||||
(
|
||||
"IF OBJECT_ID(N'[dbo].[test]', N'U') IS NULL\n"
|
||||
"BEGIN\nCREATE TABLE [dbo].[test] \n ([id] nvarchar"
|
||||
"(255) NOT NULL,\n[content] nvarchar(max) NULL,\n[vector] VECTOR(5) NULL,\n"
|
||||
"PRIMARY KEY ([id]) \n) ;"
|
||||
"\nEND\n"
|
||||
),
|
||||
(),
|
||||
)
|
||||
|
||||
async def test_ensure_collection_deleted(
|
||||
self,
|
||||
sql_server_unit_test_env,
|
||||
mock_connection,
|
||||
definition,
|
||||
):
|
||||
collection = SqlServerCollection(
|
||||
collection_name="test",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
connection=mock_connection,
|
||||
)
|
||||
await collection.ensure_collection_deleted()
|
||||
mock_connection.cursor.return_value.__enter__.return_value.execute.assert_called_with(
|
||||
"DROP TABLE IF EXISTS [dbo].[test] ;", ()
|
||||
)
|
||||
|
||||
async def test_no_connection(self, sql_server_unit_test_env, definition):
|
||||
collection = SqlServerCollection(
|
||||
collection_name="test",
|
||||
record_type=dict,
|
||||
definition=definition,
|
||||
)
|
||||
with raises(VectorStoreOperationException):
|
||||
await collection.ensure_collection_exists()
|
||||
with raises(VectorStoreOperationException):
|
||||
await collection.ensure_collection_deleted()
|
||||
with raises(VectorStoreOperationException):
|
||||
await collection.collection_exists()
|
||||
with raises(VectorStoreOperationException):
|
||||
await collection.upsert({"id": "1", "content": "test", "vector": [0.1, 0.2, 0.3, 0.4, 0.5]})
|
||||
with raises(VectorStoreOperationException):
|
||||
await collection.get("1")
|
||||
with raises(VectorStoreOperationException):
|
||||
await collection.delete("1")
|
||||
@@ -0,0 +1,70 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from weaviate import WeaviateAsyncClient
|
||||
from weaviate.collections.collections.async_ import _CollectionsAsync
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def collections_side_effects(request):
|
||||
"""Fixture that returns a dictionary of side effects for the mock async client methods."""
|
||||
return request.param if hasattr(request, "param") else {}
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def mock_async_client(collections_side_effects) -> AsyncMock:
|
||||
"""Fixture to create a mock async client."""
|
||||
async_mock = AsyncMock(spec=WeaviateAsyncClient)
|
||||
async_mock.collections = AsyncMock(spec=_CollectionsAsync)
|
||||
async_mock.collections.create = AsyncMock()
|
||||
async_mock.collections.delete = AsyncMock()
|
||||
async_mock.collections.exists = AsyncMock()
|
||||
|
||||
if collections_side_effects:
|
||||
for method_name, exception in collections_side_effects.items():
|
||||
getattr(async_mock.collections, method_name).side_effect = exception
|
||||
|
||||
return async_mock
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def weaviate_unit_test_env(monkeypatch, exclude_list, override_env_param_dict):
|
||||
"""Fixture to set environment variables for Weaviate unit tests."""
|
||||
if exclude_list is None:
|
||||
exclude_list = []
|
||||
|
||||
if override_env_param_dict is None:
|
||||
override_env_param_dict = {}
|
||||
|
||||
env_vars = {
|
||||
"WEAVIATE_URL": "test-api-key",
|
||||
"WEAVIATE_API_KEY": "https://test-endpoint.com",
|
||||
"WEAVIATE_LOCAL_HOST": "localhost",
|
||||
"WEAVIATE_LOCAL_PORT": 8080,
|
||||
"WEAVIATE_LOCAL_GRPC_PORT": 8081,
|
||||
"WEAVIATE_USE_EMBED": True,
|
||||
}
|
||||
|
||||
env_vars.update(override_env_param_dict)
|
||||
|
||||
for key, value in env_vars.items():
|
||||
if key not in exclude_list:
|
||||
monkeypatch.setenv(key, value)
|
||||
else:
|
||||
monkeypatch.delenv(key, raising=False)
|
||||
|
||||
return env_vars
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def clear_weaviate_env(monkeypatch):
|
||||
"""Fixture to clear the environment variables for Weaviate unit tests."""
|
||||
monkeypatch.delenv("WEAVIATE_URL", raising=False)
|
||||
monkeypatch.delenv("WEAVIATE_API_KEY", raising=False)
|
||||
monkeypatch.delenv("WEAVIATE_LOCAL_HOST", raising=False)
|
||||
monkeypatch.delenv("WEAVIATE_LOCAL_PORT", raising=False)
|
||||
monkeypatch.delenv("WEAVIATE_LOCAL_GRPC_PORT", raising=False)
|
||||
monkeypatch.delenv("WEAVIATE_USE_EMBED", raising=False)
|
||||
@@ -0,0 +1,429 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from unittest.mock import ANY, AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from weaviate import WeaviateAsyncClient
|
||||
from weaviate.classes.config import Configure, DataType, Property
|
||||
from weaviate.collections.classes.config_vectorizers import VectorDistances
|
||||
from weaviate.collections.classes.data import DataObject
|
||||
|
||||
from semantic_kernel.connectors.weaviate import WeaviateCollection
|
||||
from semantic_kernel.exceptions import (
|
||||
ServiceInvalidExecutionSettingsError,
|
||||
VectorStoreInitializationException,
|
||||
VectorStoreOperationException,
|
||||
)
|
||||
|
||||
|
||||
@patch(
|
||||
"semantic_kernel.connectors.weaviate.use_async_with_weaviate_cloud",
|
||||
return_value=AsyncMock(spec=WeaviateAsyncClient),
|
||||
)
|
||||
def test_weaviate_collection_init_with_weaviate_cloud(
|
||||
mock_use_weaviate_cloud,
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
) -> None:
|
||||
"""Test the initialization of a WeaviateCollection object with Weaviate Cloud."""
|
||||
collection_name = "TestCollection"
|
||||
|
||||
collection = WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
url="https://test.cloud.weaviate.com/",
|
||||
api_key="test_api_key",
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
assert collection.collection_name == collection_name
|
||||
assert collection.async_client is not None
|
||||
mock_use_weaviate_cloud.assert_called_once_with(
|
||||
cluster_url="https://test.cloud.weaviate.com/",
|
||||
auth_credentials=ANY,
|
||||
)
|
||||
|
||||
|
||||
@patch(
|
||||
"semantic_kernel.connectors.weaviate.use_async_with_local",
|
||||
return_value=AsyncMock(spec=WeaviateAsyncClient),
|
||||
)
|
||||
def test_weaviate_collection_init_with_local(
|
||||
mock_use_weaviate_local,
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
) -> None:
|
||||
"""Test the initialization of a WeaviateCollection object with Weaviate local deployment."""
|
||||
collection_name = "TestCollection"
|
||||
|
||||
collection = WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
local_host="localhost",
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
assert collection.collection_name == collection_name
|
||||
assert collection.async_client is not None
|
||||
mock_use_weaviate_local.assert_called_once_with(host="localhost")
|
||||
|
||||
|
||||
@patch(
|
||||
"semantic_kernel.connectors.weaviate.use_async_with_embedded",
|
||||
return_value=AsyncMock(spec=WeaviateAsyncClient),
|
||||
)
|
||||
def test_weaviate_collection_init_with_embedded(
|
||||
mock_use_weaviate_embedded,
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
) -> None:
|
||||
"""Test the initialization of a WeaviateCollection object with Weaviate embedded deployment."""
|
||||
collection_name = "TestCollection"
|
||||
|
||||
collection = WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
use_embed=True,
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
assert collection.collection_name == collection_name
|
||||
assert collection.async_client is not None
|
||||
mock_use_weaviate_embedded.assert_called_once()
|
||||
|
||||
|
||||
def test_weaviate_collection_init_with_invalid_settings_more_than_one_backends(
|
||||
weaviate_unit_test_env,
|
||||
record_type,
|
||||
definition,
|
||||
) -> None:
|
||||
"""Test the initialization of a WeaviateCollection object with multiple backend options enabled."""
|
||||
collection_name = "TestCollection"
|
||||
|
||||
with pytest.raises(ServiceInvalidExecutionSettingsError):
|
||||
WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
|
||||
def test_weaviate_collection_init_with_invalid_settings_no_backends(
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
) -> None:
|
||||
"""Test the initialization of a WeaviateCollection object with no backend options enabled."""
|
||||
collection_name = "TestCollection"
|
||||
|
||||
with pytest.raises(ServiceInvalidExecutionSettingsError):
|
||||
WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
|
||||
def test_weaviate_collection_init_with_custom_client(
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
) -> None:
|
||||
"""Test the initialization of a WeaviateCollection object with a custom client."""
|
||||
collection_name = "TestCollection"
|
||||
|
||||
collection = WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
async_client=AsyncMock(spec=WeaviateAsyncClient),
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
assert collection.collection_name == collection_name
|
||||
assert collection.async_client is not None
|
||||
|
||||
|
||||
@patch(
|
||||
"semantic_kernel.connectors.weaviate.use_async_with_local",
|
||||
side_effect=Exception,
|
||||
)
|
||||
def test_weaviate_collection_init_fail_to_create_client(
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
) -> None:
|
||||
"""Test the initialization of a WeaviateCollection object raises an error when failing to create a client."""
|
||||
collection_name = "TestCollection"
|
||||
|
||||
with pytest.raises(VectorStoreInitializationException):
|
||||
WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
local_host="localhost",
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
|
||||
@patch(
|
||||
"semantic_kernel.connectors.weaviate.use_async_with_weaviate_cloud",
|
||||
return_value=AsyncMock(spec=WeaviateAsyncClient),
|
||||
)
|
||||
def test_weaviate_collection_init_with_lower_case_collection_name(
|
||||
mock_use_weaviate_cloud,
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
) -> None:
|
||||
"""Test a collection name with lower case letters."""
|
||||
collection_name = "testCollection"
|
||||
|
||||
collection = WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
url="https://test.cloud.weaviate.com",
|
||||
api_key="test_api_key",
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
assert collection.collection_name[0].isupper()
|
||||
assert collection.async_client is not None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("index_kind, distance_function", [("hnsw", "cosine_distance")])
|
||||
async def test_weaviate_collection_ensure_collection_exists(
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
mock_async_client,
|
||||
) -> None:
|
||||
"""Test the creation of a collection in Weaviate."""
|
||||
collection_name = "TestCollection"
|
||||
|
||||
collection = WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
async_client=mock_async_client,
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
await collection.ensure_collection_exists()
|
||||
|
||||
mock_async_client.collections.create.assert_called_once_with(
|
||||
name=collection_name,
|
||||
properties=[
|
||||
Property(
|
||||
name="content",
|
||||
data_type=DataType.TEXT,
|
||||
)
|
||||
],
|
||||
vector_index_config=None,
|
||||
vectorizer_config=[
|
||||
Configure.NamedVectors.none(
|
||||
name="vector",
|
||||
vector_index_config=Configure.VectorIndex.hnsw(distance_metric=VectorDistances.COSINE),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"collections_side_effects",
|
||||
[
|
||||
{"create": Exception},
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_weaviate_collection_ensure_collection_exists_fail(
|
||||
mock_async_client,
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
) -> None:
|
||||
"""Test failing to create a collection in Weaviate."""
|
||||
collection_name = "TestCollection"
|
||||
|
||||
collection = WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
async_client=mock_async_client,
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
with pytest.raises(VectorStoreOperationException):
|
||||
await collection.ensure_collection_exists()
|
||||
|
||||
|
||||
async def test_weaviate_collection_ensure_collection_deleted(
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
mock_async_client,
|
||||
) -> None:
|
||||
"""Test the deletion of a collection in Weaviate."""
|
||||
collection_name = "TestCollection"
|
||||
|
||||
collection = WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
async_client=mock_async_client,
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
await collection.ensure_collection_deleted()
|
||||
|
||||
mock_async_client.collections.delete.assert_called_once_with(collection_name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"collections_side_effects",
|
||||
[
|
||||
{"delete": Exception},
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_weaviate_collection_ensure_collection_deleted_fail(
|
||||
mock_async_client,
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
) -> None:
|
||||
"""Test failing to delete a collection in Weaviate."""
|
||||
collection_name = "TestCollection"
|
||||
|
||||
collection = WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
async_client=mock_async_client,
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
with pytest.raises(VectorStoreOperationException):
|
||||
await collection.ensure_collection_deleted()
|
||||
|
||||
|
||||
async def test_weaviate_collection_collection_exist(
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
mock_async_client,
|
||||
) -> None:
|
||||
"""Test checking if a collection exists in Weaviate."""
|
||||
collection_name = "TestCollection"
|
||||
|
||||
collection = WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
async_client=mock_async_client,
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
await collection.collection_exists()
|
||||
|
||||
mock_async_client.collections.exists.assert_called_once_with(collection_name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"collections_side_effects",
|
||||
[
|
||||
{"exists": Exception},
|
||||
],
|
||||
indirect=True,
|
||||
)
|
||||
async def test_weaviate_collection_collection_exist_fail(
|
||||
mock_async_client,
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
) -> None:
|
||||
"""Test failing to check if a collection exists in Weaviate."""
|
||||
collection_name = "TestCollection"
|
||||
|
||||
collection = WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
async_client=mock_async_client,
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
with pytest.raises(VectorStoreOperationException):
|
||||
await collection.collection_exists()
|
||||
|
||||
|
||||
async def test_weaviate_collection_serialize_data(
|
||||
mock_async_client,
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
dataclass_vector_data_model,
|
||||
) -> None:
|
||||
"""Test upserting data into a collection in Weaviate."""
|
||||
collection_name = "TestCollection"
|
||||
|
||||
collection = WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
async_client=mock_async_client,
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
with patch.object(collection, "_inner_upsert") as mock_inner_upsert:
|
||||
data = dataclass_vector_data_model()
|
||||
await collection.upsert(data)
|
||||
|
||||
mock_inner_upsert.assert_called_once_with([
|
||||
DataObject(
|
||||
properties={"content": "content1"},
|
||||
uuid=data.id,
|
||||
vector={"vector": data.vector},
|
||||
references=None,
|
||||
)
|
||||
])
|
||||
|
||||
|
||||
async def test_weaviate_collection_deserialize_data(
|
||||
mock_async_client,
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
dataclass_vector_data_model,
|
||||
) -> None:
|
||||
"""Test getting data from a collection in Weaviate."""
|
||||
collection_name = "TestCollection"
|
||||
|
||||
collection = WeaviateCollection(
|
||||
record_type=record_type,
|
||||
definition=definition,
|
||||
collection_name=collection_name,
|
||||
async_client=mock_async_client,
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
data = dataclass_vector_data_model()
|
||||
weaviate_data_object = DataObject(
|
||||
properties={"content": "content1"},
|
||||
uuid=data.id,
|
||||
vector={"vector": data.vector or [1, 2, 3]},
|
||||
)
|
||||
|
||||
with patch.object(collection, "_inner_get", return_value=[weaviate_data_object]) as mock_inner_get:
|
||||
await collection.get(key=data.id)
|
||||
|
||||
mock_inner_get.assert_called_once_with([data.id], include_vectors=False, options=None)
|
||||
@@ -0,0 +1,120 @@
|
||||
# Copyright (c) Microsoft. All rights reserved.
|
||||
|
||||
from unittest.mock import ANY, AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
from weaviate import WeaviateAsyncClient
|
||||
|
||||
from semantic_kernel.connectors.weaviate import WeaviateStore
|
||||
from semantic_kernel.exceptions import ServiceInvalidExecutionSettingsError, VectorStoreInitializationException
|
||||
|
||||
|
||||
@patch(
|
||||
"semantic_kernel.connectors.weaviate.use_async_with_weaviate_cloud",
|
||||
return_value=AsyncMock(spec=WeaviateAsyncClient),
|
||||
)
|
||||
def test_weaviate_store_init_with_weaviate_cloud(
|
||||
mock_use_weaviate_cloud,
|
||||
clear_weaviate_env,
|
||||
) -> None:
|
||||
"""Test the initialization of a WeaviateStore object with Weaviate Cloud."""
|
||||
store = WeaviateStore(
|
||||
url="https://test.cloud.weaviate.com/",
|
||||
api_key="test_api_key",
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
assert store.async_client is not None
|
||||
mock_use_weaviate_cloud.assert_called_once_with(
|
||||
cluster_url="https://test.cloud.weaviate.com/",
|
||||
auth_credentials=ANY,
|
||||
)
|
||||
|
||||
|
||||
@patch(
|
||||
"semantic_kernel.connectors.weaviate.use_async_with_local",
|
||||
return_value=AsyncMock(spec=WeaviateAsyncClient),
|
||||
)
|
||||
def test_weaviate_store_init_with_local(
|
||||
mock_use_weaviate_local,
|
||||
clear_weaviate_env,
|
||||
) -> None:
|
||||
"""Test the initialization of a WeaviateStore object with Weaviate local deployment."""
|
||||
store = WeaviateStore(
|
||||
local_host="localhost",
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
assert store.async_client is not None
|
||||
mock_use_weaviate_local.assert_called_once_with(host="localhost")
|
||||
|
||||
|
||||
@patch(
|
||||
"semantic_kernel.connectors.weaviate.use_async_with_embedded",
|
||||
return_value=AsyncMock(spec=WeaviateAsyncClient),
|
||||
)
|
||||
def test_weaviate_store_init_with_embedded(
|
||||
mock_use_weaviate_embedded,
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
) -> None:
|
||||
"""Test the initialization of a WeaviateStore object with Weaviate embedded deployment."""
|
||||
store = WeaviateStore(
|
||||
use_embed=True,
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
assert store.async_client is not None
|
||||
mock_use_weaviate_embedded.assert_called_once()
|
||||
|
||||
|
||||
def test_weaviate_store_init_with_invalid_settings_more_than_one_backends(
|
||||
weaviate_unit_test_env,
|
||||
) -> None:
|
||||
"""Test the initialization of a WeaviateStore object with multiple backend options enabled."""
|
||||
with pytest.raises(ServiceInvalidExecutionSettingsError):
|
||||
WeaviateStore(
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
|
||||
def test_weaviate_store_init_with_invalid_settings_no_backends(
|
||||
clear_weaviate_env,
|
||||
) -> None:
|
||||
"""Test the initialization of a WeaviateStore object with no backend options enabled."""
|
||||
with pytest.raises(ServiceInvalidExecutionSettingsError):
|
||||
WeaviateStore(
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
|
||||
def test_weaviate_store_init_with_custom_client(
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
) -> None:
|
||||
"""Test the initialization of a WeaviateStore object with a custom client."""
|
||||
store = WeaviateStore(
|
||||
async_client=AsyncMock(spec=WeaviateAsyncClient),
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
|
||||
assert store.async_client is not None
|
||||
|
||||
|
||||
@patch(
|
||||
"semantic_kernel.connectors.weaviate.use_async_with_local",
|
||||
side_effect=Exception,
|
||||
)
|
||||
def test_weaviate_store_init_fail_to_create_client(
|
||||
clear_weaviate_env,
|
||||
record_type,
|
||||
definition,
|
||||
) -> None:
|
||||
"""Test the initialization of a WeaviateStore object raises an error when failing to create a client."""
|
||||
with pytest.raises(VectorStoreInitializationException):
|
||||
WeaviateStore(
|
||||
local_host="localhost",
|
||||
env_file_path="fake_env_file_path.env",
|
||||
)
|
||||
Reference in New Issue
Block a user