chore: import upstream snapshot with attribution

test/components/embedders/test_azure_document_embedder.py

@@ -0,0 +1,385 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from unittest.mock import AsyncMock, MagicMock, Mock, patch
+
+import pytest
+from openai import APIError, OpenAIError
+
+import haystack.components.embedders.azure_document_embedder as azure_document_embedder_module
+from haystack import Document
+from haystack.components.embedders import AzureOpenAIDocumentEmbedder
+from haystack.utils.auth import Secret
+from haystack.utils.azure import default_azure_ad_token_provider
+
+
+class TestAzureOpenAIDocumentEmbedder:
+    def test_init_default(self, monkeypatch):
+        monkeypatch.setenv("AZURE_OPENAI_API_KEY", "fake-api-key")
+        embedder = AzureOpenAIDocumentEmbedder(azure_endpoint="https://example-resource.azure.openai.com/")
+        assert embedder.azure_deployment == "text-embedding-ada-002"
+        assert embedder.model == "text-embedding-ada-002"
+        assert embedder.dimensions is None
+        assert embedder.organization is None
+        assert embedder.prefix == ""
+        assert embedder.suffix == ""
+        assert embedder.batch_size == 32
+        assert embedder.progress_bar is True
+        assert embedder.meta_fields_to_embed == []
+        assert embedder.embedding_separator == "\n"
+        assert embedder.timeout is None
+        assert embedder.max_retries is None
+        assert embedder.default_headers == {}
+        assert embedder.azure_ad_token_provider is None
+        assert embedder.http_client_kwargs is None
+        assert embedder.client is None
+        assert embedder.async_client is None
+
+    def test_init_with_0_max_retries(self, monkeypatch):
+        """Tests that the max_retries init param is set correctly if equal 0"""
+        monkeypatch.setenv("AZURE_OPENAI_API_KEY", "fake-api-key")
+        embedder = AzureOpenAIDocumentEmbedder(
+            azure_endpoint="https://example-resource.azure.openai.com/", max_retries=0
+        )
+        assert embedder.azure_deployment == "text-embedding-ada-002"
+        assert embedder.model == "text-embedding-ada-002"
+        assert embedder.dimensions is None
+        assert embedder.organization is None
+        assert embedder.prefix == ""
+        assert embedder.suffix == ""
+        assert embedder.batch_size == 32
+        assert embedder.progress_bar is True
+        assert embedder.meta_fields_to_embed == []
+        assert embedder.embedding_separator == "\n"
+        assert embedder.default_headers == {}
+        assert embedder.azure_ad_token_provider is None
+        assert embedder.max_retries == 0
+        assert embedder.client is None
+        assert embedder.async_client is None
+
+    def test_to_dict(self, monkeypatch):
+        monkeypatch.setenv("AZURE_OPENAI_API_KEY", "fake-api-key")
+        component = AzureOpenAIDocumentEmbedder(azure_endpoint="https://example-resource.azure.openai.com/")
+        data = component.to_dict()
+        assert data == {
+            "type": "haystack.components.embedders.azure_document_embedder.AzureOpenAIDocumentEmbedder",
+            "init_parameters": {
+                "api_key": {"env_vars": ["AZURE_OPENAI_API_KEY"], "strict": False, "type": "env_var"},
+                "azure_ad_token": {"env_vars": ["AZURE_OPENAI_AD_TOKEN"], "strict": False, "type": "env_var"},
+                "api_version": "2023-05-15",
+                "azure_deployment": "text-embedding-ada-002",
+                "dimensions": None,
+                "azure_endpoint": "https://example-resource.azure.openai.com/",
+                "organization": None,
+                "prefix": "",
+                "suffix": "",
+                "batch_size": 32,
+                "progress_bar": True,
+                "meta_fields_to_embed": [],
+                "embedding_separator": "\n",
+                "max_retries": None,
+                "timeout": None,
+                "default_headers": {},
+                "azure_ad_token_provider": None,
+                "http_client_kwargs": None,
+                "raise_on_failure": False,
+            },
+        }
+
+    def test_to_dict_with_parameters(self, monkeypatch):
+        monkeypatch.setenv("AZURE_OPENAI_API_KEY", "fake-api-key")
+        component = AzureOpenAIDocumentEmbedder(
+            azure_endpoint="https://example-resource.azure.openai.com/",
+            azure_deployment="text-embedding-ada-002",
+            dimensions=768,
+            organization="HaystackCI",
+            timeout=60.0,
+            max_retries=10,
+            prefix="prefix ",
+            suffix=" suffix",
+            default_headers={"x-custom-header": "custom-value"},
+            azure_ad_token_provider=default_azure_ad_token_provider,
+            http_client_kwargs={"proxy": "http://example.com:3128", "verify": False},
+            raise_on_failure=True,
+        )
+        data = component.to_dict()
+        assert data == {
+            "type": "haystack.components.embedders.azure_document_embedder.AzureOpenAIDocumentEmbedder",
+            "init_parameters": {
+                "api_key": {"env_vars": ["AZURE_OPENAI_API_KEY"], "strict": False, "type": "env_var"},
+                "azure_ad_token": {"env_vars": ["AZURE_OPENAI_AD_TOKEN"], "strict": False, "type": "env_var"},
+                "api_version": "2023-05-15",
+                "azure_deployment": "text-embedding-ada-002",
+                "dimensions": 768,
+                "azure_endpoint": "https://example-resource.azure.openai.com/",
+                "organization": "HaystackCI",
+                "prefix": "prefix ",
+                "suffix": " suffix",
+                "batch_size": 32,
+                "progress_bar": True,
+                "meta_fields_to_embed": [],
+                "embedding_separator": "\n",
+                "max_retries": 10,
+                "timeout": 60.0,
+                "default_headers": {"x-custom-header": "custom-value"},
+                "azure_ad_token_provider": "haystack.utils.azure.default_azure_ad_token_provider",
+                "http_client_kwargs": {"proxy": "http://example.com:3128", "verify": False},
+                "raise_on_failure": True,
+            },
+        }
+
+    def test_from_dict(self, monkeypatch):
+        monkeypatch.setenv("AZURE_OPENAI_API_KEY", "fake-api-key")
+        data = {
+            "type": "haystack.components.embedders.azure_document_embedder.AzureOpenAIDocumentEmbedder",
+            "init_parameters": {
+                "api_key": {"env_vars": ["AZURE_OPENAI_API_KEY"], "strict": False, "type": "env_var"},
+                "azure_ad_token": {"env_vars": ["AZURE_OPENAI_AD_TOKEN"], "strict": False, "type": "env_var"},
+                "api_version": "2023-05-15",
+                "azure_deployment": "text-embedding-ada-002",
+                "dimensions": None,
+                "azure_endpoint": "https://example-resource.azure.openai.com/",
+                "organization": None,
+                "prefix": "",
+                "suffix": "",
+                "batch_size": 32,
+                "progress_bar": True,
+                "meta_fields_to_embed": [],
+                "embedding_separator": "\n",
+                "max_retries": 5,
+                "timeout": 30.0,
+                "default_headers": {},
+                "azure_ad_token_provider": None,
+                "http_client_kwargs": None,
+                "raise_on_failure": False,
+            },
+        }
+        component = AzureOpenAIDocumentEmbedder.from_dict(data)
+        assert component.azure_deployment == "text-embedding-ada-002"
+        assert component.azure_endpoint == "https://example-resource.azure.openai.com/"
+        assert component.api_version == "2023-05-15"
+        assert component.max_retries == 5
+        assert component.timeout == 30.0
+        assert component.prefix == ""
+        assert component.suffix == ""
+        assert component.default_headers == {}
+        assert component.azure_ad_token_provider is None
+        assert component.http_client_kwargs is None
+        assert component.raise_on_failure is False
+
+    def test_from_dict_with_parameters(self, monkeypatch):
+        monkeypatch.setenv("AZURE_OPENAI_API_KEY", "fake-api-key")
+        data = {
+            "type": "haystack.components.embedders.azure_document_embedder.AzureOpenAIDocumentEmbedder",
+            "init_parameters": {
+                "api_key": {"env_vars": ["AZURE_OPENAI_API_KEY"], "strict": False, "type": "env_var"},
+                "azure_ad_token": {"env_vars": ["AZURE_OPENAI_AD_TOKEN"], "strict": False, "type": "env_var"},
+                "api_version": "2023-05-15",
+                "azure_deployment": "text-embedding-ada-002",
+                "dimensions": 768,
+                "azure_endpoint": "https://example-resource.azure.openai.com/",
+                "organization": "HaystackCI",
+                "prefix": "prefix ",
+                "suffix": " suffix",
+                "batch_size": 32,
+                "progress_bar": True,
+                "meta_fields_to_embed": [],
+                "embedding_separator": "\n",
+                "max_retries": 10,
+                "timeout": 60.0,
+                "default_headers": {"x-custom-header": "custom-value"},
+                "azure_ad_token_provider": "haystack.utils.azure.default_azure_ad_token_provider",
+                "http_client_kwargs": {"proxy": "http://example.com:3128", "verify": False},
+                "raise_on_failure": True,
+            },
+        }
+        component = AzureOpenAIDocumentEmbedder.from_dict(data)
+        assert component.azure_deployment == "text-embedding-ada-002"
+        assert component.azure_endpoint == "https://example-resource.azure.openai.com/"
+        assert component.api_version == "2023-05-15"
+        assert component.max_retries == 10
+        assert component.timeout == 60.0
+        assert component.prefix == "prefix "
+        assert component.suffix == " suffix"
+        assert component.default_headers == {"x-custom-header": "custom-value"}
+        assert component.azure_ad_token_provider is not None
+        assert component.http_client_kwargs == {"proxy": "http://example.com:3128", "verify": False}
+        assert component.raise_on_failure is True
+
+    def test_embed_batch_handles_exceptions_gracefully(self, caplog):
+        embedder = AzureOpenAIDocumentEmbedder(
+            azure_endpoint="https://test.openai.azure.com",
+            api_key=Secret.from_token("fake-api-key"),
+            azure_deployment="text-embedding-ada-002",
+            embedding_separator=" | ",
+        )
+        embedder.warm_up()
+        assert embedder.client is not None
+
+        fake_texts_to_embed = {"1": "text1", "2": "text2"}
+
+        with patch.object(
+            embedder.client.embeddings,
+            "create",
+            side_effect=APIError(message="Mocked error", request=Mock(), body=None),
+        ):
+            embedder._embed_batch(texts_to_embed=fake_texts_to_embed, batch_size=32)
+
+        assert len(caplog.records) == 1
+        assert "Failed embedding of documents 1, 2 caused by Mocked error" in caplog.text
+
+    def test_embed_batch_raises_exception_on_failure(self):
+        embedder = AzureOpenAIDocumentEmbedder(
+            azure_endpoint="https://test.openai.azure.com",
+            api_key=Secret.from_token("fake-api-key"),
+            azure_deployment="text-embedding-ada-002",
+            raise_on_failure=True,
+        )
+        embedder.warm_up()
+        assert embedder.client is not None
+        fake_texts_to_embed = {"1": "text1", "2": "text2"}
+        with patch.object(
+            embedder.client.embeddings,
+            "create",
+            side_effect=APIError(message="Mocked error", request=Mock(), body=None),
+        ):
+            with pytest.raises(APIError, match="Mocked error"):
+                embedder._embed_batch(texts_to_embed=fake_texts_to_embed, batch_size=2)
+
+    @pytest.mark.integration
+    @pytest.mark.skipif(
+        not os.environ.get("AZURE_OPENAI_API_KEY", None) and not os.environ.get("AZURE_OPENAI_ENDPOINT", None),
+        reason=(
+            "Please export env variables called AZURE_OPENAI_API_KEY containing "
+            "the Azure OpenAI key, AZURE_OPENAI_ENDPOINT containing "
+            "the Azure OpenAI endpoint URL to run this test."
+        ),
+    )
+    def test_run(self):
+        docs = [
+            Document(content="I love cheese", meta={"topic": "Cuisine"}),
+            Document(content="A transformer is a deep learning architecture", meta={"topic": "ML"}),
+        ]
+        # the default model is text-embedding-ada-002 even if we don't specify it, but let's be explicit
+        embedder = AzureOpenAIDocumentEmbedder(
+            azure_deployment="text-embedding-ada-002",
+            meta_fields_to_embed=["topic"],
+            embedding_separator=" | ",
+            organization="HaystackCI",
+        )
+
+        result = embedder.run(documents=docs)
+        documents_with_embeddings = result["documents"]
+        metadata = result["meta"]
+
+        assert isinstance(documents_with_embeddings, list)
+        assert len(documents_with_embeddings) == len(docs)
+        for doc, new_doc in zip(docs, documents_with_embeddings, strict=True):
+            assert doc.embedding is None
+            assert new_doc is not doc
+            assert isinstance(new_doc, Document)
+            assert isinstance(new_doc.embedding, list)
+            assert len(new_doc.embedding) == 1536
+            assert all(isinstance(x, float) for x in new_doc.embedding)
+
+        assert metadata["usage"]["prompt_tokens"] == 15
+        assert metadata["usage"]["total_tokens"] == 15
+        assert "ada" in metadata["model"]
+
+
+@pytest.fixture
+def mock_azure_clients(monkeypatch):
+    monkeypatch.setenv("AZURE_OPENAI_API_KEY", "fake")
+    sync_cls = MagicMock(name="AzureOpenAI")
+    async_cls = MagicMock(name="AsyncAzureOpenAI")
+    async_cls.return_value.close = AsyncMock()
+    monkeypatch.setattr(azure_document_embedder_module, "AzureOpenAI", sync_cls)
+    monkeypatch.setattr(azure_document_embedder_module, "AsyncAzureOpenAI", async_cls)
+    return sync_cls, async_cls
+
+
+class TestComponentLifecycle:
+    def test_warm_up_uses_default_timeout_and_max_retries(self, monkeypatch):
+        monkeypatch.setenv("AZURE_OPENAI_API_KEY", "fake-api-key")
+        embedder = AzureOpenAIDocumentEmbedder(azure_endpoint="https://example-resource.azure.openai.com/")
+        embedder.warm_up()
+        assert embedder.client is not None
+        assert embedder.client.max_retries == 5
+        assert embedder.client.timeout == 30.0
+
+    def test_warm_up_uses_timeout_and_max_retries_from_parameters(self, monkeypatch):
+        monkeypatch.setenv("AZURE_OPENAI_API_KEY", "fake-api-key")
+        embedder = AzureOpenAIDocumentEmbedder(
+            azure_endpoint="https://example-resource.azure.openai.com/", timeout=40.0, max_retries=1
+        )
+        embedder.warm_up()
+        assert embedder.client is not None
+        assert embedder.client.max_retries == 1
+        assert embedder.client.timeout == 40.0
+
+    def test_warm_up_uses_timeout_and_max_retries_from_env_vars(self, monkeypatch):
+        monkeypatch.setenv("AZURE_OPENAI_API_KEY", "fake-api-key")
+        monkeypatch.setenv("OPENAI_TIMEOUT", "100")
+        monkeypatch.setenv("OPENAI_MAX_RETRIES", "10")
+        embedder = AzureOpenAIDocumentEmbedder(azure_endpoint="https://example-resource.azure.openai.com/")
+        embedder.warm_up()
+        assert embedder.client is not None
+        assert embedder.client.max_retries == 10
+        assert embedder.client.timeout == 100.0
+
+    def test_key_resolved_at_warm_up_not_init(self, monkeypatch):
+        monkeypatch.delenv("AZURE_OPENAI_API_KEY", raising=False)
+        monkeypatch.delenv("AZURE_OPENAI_AD_TOKEN", raising=False)
+        embedder = AzureOpenAIDocumentEmbedder(azure_endpoint="https://example-resource.azure.openai.com/")
+        assert embedder.client is None
+        with pytest.raises(OpenAIError):
+            embedder.warm_up()
+
+    def test_sync_lifecycle(self, mock_azure_clients):
+        sync_cls, _ = mock_azure_clients
+        sync_client = sync_cls.return_value
+        embedder = AzureOpenAIDocumentEmbedder(azure_endpoint="https://example-resource.azure.openai.com/")
+        assert embedder.client is None
+        assert embedder.async_client is None
+
+        embedder.warm_up()
+        assert embedder.client is sync_cls.return_value
+        assert embedder.async_client is None
+
+        embedder.close()
+        sync_client.close.assert_called_once()
+        assert embedder.client is None
+
+    async def test_async_lifecycle(self, mock_azure_clients):
+        _, async_cls = mock_azure_clients
+        async_client = async_cls.return_value
+        embedder = AzureOpenAIDocumentEmbedder(azure_endpoint="https://example-resource.azure.openai.com/")
+
+        await embedder.warm_up_async()
+        assert embedder.async_client is async_cls.return_value
+        assert embedder.client is None
+
+        await embedder.close_async()
+        async_client.close.assert_awaited_once()
+        assert embedder.async_client is None
+
+    async def test_close_is_safe_without_warm_up(self, mock_azure_clients):
+        embedder = AzureOpenAIDocumentEmbedder(azure_endpoint="https://example-resource.azure.openai.com/")
+        embedder.close()
+        await embedder.close_async()
+        assert embedder.client is None
+        assert embedder.async_client is None
+
+    async def test_close_and_close_async_are_independent(self, mock_azure_clients):
+        embedder = AzureOpenAIDocumentEmbedder(azure_endpoint="https://example-resource.azure.openai.com/")
+        embedder.warm_up()
+        await embedder.warm_up_async()
+
+        embedder.close()
+        assert embedder.client is None
+        assert embedder.async_client is not None
+
+        await embedder.close_async()
+        assert embedder.async_client is None