chore: import upstream snapshot with attribution

test/components/evaluators/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0

test/components/evaluators/test_answer_exact_match.py

@@ -0,0 +1,73 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+from haystack.components.evaluators import AnswerExactMatchEvaluator
+
+
+def test_run_with_all_matching():
+    evaluator = AnswerExactMatchEvaluator()
+    result = evaluator.run(ground_truth_answers=["Berlin", "Paris"], predicted_answers=["Berlin", "Paris"])
+
+    assert result == {"individual_scores": [1, 1], "score": 1.0}
+
+
+def test_run_with_no_matching():
+    evaluator = AnswerExactMatchEvaluator()
+    result = evaluator.run(ground_truth_answers=["Berlin", "Paris"], predicted_answers=["Paris", "London"])
+
+    assert result == {"individual_scores": [0, 0], "score": 0.0}
+
+
+def test_run_with_partial_matching():
+    evaluator = AnswerExactMatchEvaluator()
+    result = evaluator.run(ground_truth_answers=["Berlin", "Paris"], predicted_answers=["Berlin", "London"])
+
+    assert result == {"individual_scores": [1, 0], "score": 0.5}
+
+
+def test_run_with_complex_data():
+    evaluator = AnswerExactMatchEvaluator()
+    result = evaluator.run(
+        ground_truth_answers=[
+            "France",
+            "9th century",
+            "9th",
+            "classical music",
+            "classical",
+            "11th century",
+            "the 11th",
+            "Denmark",
+            "Iceland",
+            "Norway",
+            "10th century",
+            "10th",
+        ],
+        predicted_answers=[
+            "France",
+            "9th century",
+            "10th century",
+            "9th",
+            "classic music",
+            "rock music",
+            "dubstep",
+            "the 11th",
+            "11th century",
+            "Denmark, Iceland and Norway",
+            "10th century",
+            "10th",
+        ],
+    )
+    assert result == {"individual_scores": [1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1], "score": 0.3333333333333333}
+
+
+def test_run_with_different_lengths():
+    evaluator = AnswerExactMatchEvaluator()
+
+    with pytest.raises(ValueError):
+        evaluator.run(ground_truth_answers=["Berlin"], predicted_answers=["Berlin", "London"])
+
+    with pytest.raises(ValueError):
+        evaluator.run(ground_truth_answers=["Berlin", "Paris"], predicted_answers=["Berlin"])

test/components/evaluators/test_context_relevance_evaluator.py

@@ -0,0 +1,346 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import math
+import os
+
+import pytest
+
+from haystack import Pipeline
+from haystack.components.evaluators import ContextRelevanceEvaluator
+from haystack.components.generators.chat.openai import OpenAIChatGenerator
+from haystack.dataclasses.chat_message import ChatMessage
+from haystack.utils.auth import Secret
+
+
+class TestContextRelevanceEvaluator:
+    def test_init_default(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = ContextRelevanceEvaluator()
+
+        assert component.instructions == (
+            "Please extract only sentences from the provided context which are absolutely relevant and "
+            "required to answer the following question. If no relevant sentences are found, or if you "
+            "believe the question cannot be answered from the given context, return an empty list, example: []"
+        )
+        assert component.inputs == [("questions", list[str]), ("contexts", list[list[str]])]
+        assert component.outputs == ["relevant_statements"]
+        assert component.examples == [
+            {
+                "inputs": {
+                    "questions": "What is the capital of Germany?",
+                    "contexts": ["Berlin is the capital of Germany. Berlin and was founded in 1244."],
+                },
+                "outputs": {"relevant_statements": ["Berlin is the capital of Germany."]},
+            },
+            {
+                "inputs": {
+                    "questions": "What is the capital of France?",
+                    "contexts": [
+                        "Berlin is the capital of Germany and was founded in 1244.",
+                        "Europe is a continent with 44 countries.",
+                        "Madrid is the capital of Spain.",
+                    ],
+                },
+                "outputs": {"relevant_statements": []},
+            },
+            {
+                "inputs": {"questions": "What is the capital of Italy?", "contexts": ["Rome is the capital of Italy."]},
+                "outputs": {"relevant_statements": ["Rome is the capital of Italy."]},
+            },
+        ]
+
+        assert isinstance(component._chat_generator, OpenAIChatGenerator)
+        assert component._chat_generator.api_key.resolve_value() == "test-api-key"
+        assert component._chat_generator.generation_kwargs == {"response_format": {"type": "json_object"}, "seed": 42}
+
+    def test_key_resolved_at_warm_up_not_init(self, monkeypatch):
+        monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+        component = ContextRelevanceEvaluator()
+        with pytest.raises(ValueError, match="None of the .* environment variables are set"):
+            component.warm_up()
+
+    def test_init_with_parameters(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = ContextRelevanceEvaluator(
+            examples=[
+                {"inputs": {"questions": "Damn, this is straight outta hell!!!"}, "outputs": {"custom_score": 1}},
+                {"inputs": {"questions": "Football is the most popular sport."}, "outputs": {"custom_score": 0}},
+            ]
+        )
+
+        assert component.examples == [
+            {"inputs": {"questions": "Damn, this is straight outta hell!!!"}, "outputs": {"custom_score": 1}},
+            {"inputs": {"questions": "Football is the most popular sport."}, "outputs": {"custom_score": 0}},
+        ]
+
+        assert isinstance(component._chat_generator, OpenAIChatGenerator)
+        assert component._chat_generator.api_key.resolve_value() == "test-api-key"
+        assert component._chat_generator.generation_kwargs == {"response_format": {"type": "json_object"}, "seed": 42}
+
+    def test_init_with_chat_generator(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        chat_generator = OpenAIChatGenerator(generation_kwargs={"response_format": {"type": "json_object"}, "seed": 42})
+        component = ContextRelevanceEvaluator(chat_generator=chat_generator)
+
+        assert component._chat_generator is chat_generator
+
+    def test_to_dict_with_parameters(self, monkeypatch):
+        monkeypatch.setenv("ENV_VAR", "test-api-key")
+        chat_generator = OpenAIChatGenerator(
+            generation_kwargs={"response_format": {"type": "json_object"}, "seed": 42},
+            api_key=Secret.from_env_var("ENV_VAR"),
+        )
+
+        component = ContextRelevanceEvaluator(
+            chat_generator=chat_generator,
+            examples=[{"inputs": {"questions": "What is football?"}, "outputs": {"score": 0}}],
+            raise_on_failure=False,
+            progress_bar=False,
+        )
+        data = component.to_dict()
+        assert data == {
+            "type": "haystack.components.evaluators.context_relevance.ContextRelevanceEvaluator",
+            "init_parameters": {
+                "chat_generator": chat_generator.to_dict(),
+                "examples": [{"inputs": {"questions": "What is football?"}, "outputs": {"score": 0}}],
+                "progress_bar": False,
+                "raise_on_failure": False,
+            },
+        }
+
+    def test_from_dict(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        chat_generator = OpenAIChatGenerator(generation_kwargs={"response_format": {"type": "json_object"}, "seed": 42})
+
+        data = {
+            "type": "haystack.components.evaluators.context_relevance.ContextRelevanceEvaluator",
+            "init_parameters": {
+                "chat_generator": chat_generator.to_dict(),
+                "examples": [{"inputs": {"questions": "What is football?"}, "outputs": {"score": 0}}],
+            },
+        }
+
+        component = ContextRelevanceEvaluator.from_dict(data)
+        assert isinstance(component._chat_generator, OpenAIChatGenerator)
+        assert component._chat_generator.api_key.resolve_value() == "test-api-key"
+        assert component._chat_generator.generation_kwargs == {"response_format": {"type": "json_object"}, "seed": 42}
+        assert component.examples == [{"inputs": {"questions": "What is football?"}, "outputs": {"score": 0}}]
+
+    def test_pipeline_serde(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+
+        component = ContextRelevanceEvaluator()
+        pipeline = Pipeline()
+        pipeline.add_component("evaluator", component)
+
+        serialized_pipeline = pipeline.dumps()
+        deserialized_pipeline = Pipeline.loads(serialized_pipeline)
+        assert deserialized_pipeline == pipeline
+
+    def test_run_calculates_mean_score(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = ContextRelevanceEvaluator()
+
+        def chat_generator_run(self, *args, **kwargs):
+            if "Football" in kwargs["messages"][0].text:
+                return {"replies": [ChatMessage.from_assistant('{"relevant_statements": ["a", "b"], "score": 1}')]}
+            return {"replies": [ChatMessage.from_assistant('{"relevant_statements": [], "score": 0}')]}
+
+        monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
+
+        questions = ["Which is the most popular global sport?", "Who created the Python language?"]
+        contexts = [
+            [
+                "The popularity of sports can be measured in various ways, including TV viewership, social media "
+                "presence, number of participants, and economic impact. Football is undoubtedly the world's most "
+                "popular sport with major events like the FIFA World Cup and sports personalities like Ronaldo and "
+                "Messi, drawing a followership of more than 4 billion people."
+            ],
+            [
+                "Python is design philosophy emphasizes code readability, and its language constructs aim to help "
+                "programmers write clear, logical code for both small and large-scale software projects."
+            ],
+        ]
+        results = component.run(questions=questions, contexts=contexts)
+
+        assert results == {
+            "results": [{"score": 1, "relevant_statements": ["a", "b"]}, {"score": 0, "relevant_statements": []}],
+            "score": 0.5,
+            "meta": None,
+            "individual_scores": [1, 0],
+        }
+
+    def test_run_no_statements_extracted(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = ContextRelevanceEvaluator()
+
+        def chat_generator_run(self, *args, **kwargs):
+            if "Football" in kwargs["messages"][0].text:
+                return {"replies": [ChatMessage.from_assistant('{"relevant_statements": ["a", "b"], "score": 1}')]}
+            return {"replies": [ChatMessage.from_assistant('{"relevant_statements": [], "score": 0}')]}
+
+        monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
+
+        questions = ["Which is the most popular global sport?", "Who created the Python language?"]
+        contexts = [
+            [
+                "The popularity of sports can be measured in various ways, including TV viewership, social media "
+                "presence, number of participants, and economic impact. Football is undoubtedly the world's most "
+                "popular sport with major events like the FIFA World Cup and sports personalities like Ronaldo and "
+                "Messi, drawing a followership of more than 4 billion people."
+            ],
+            [],
+        ]
+        results = component.run(questions=questions, contexts=contexts)
+        assert results == {
+            "results": [{"score": 1, "relevant_statements": ["a", "b"]}, {"score": 0, "relevant_statements": []}],
+            "score": 0.5,
+            "meta": None,
+            "individual_scores": [1, 0],
+        }
+
+    def test_run_missing_parameters(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = ContextRelevanceEvaluator()
+        with pytest.raises(ValueError, match="LLM evaluator expected input parameter"):
+            component.run()
+
+    def test_run_returns_nan_raise_on_failure_false(self, monkeypatch, caplog):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = ContextRelevanceEvaluator(raise_on_failure=False)
+
+        def chat_generator_run(self, *args, **kwargs):
+            if "Python" in kwargs["messages"][0].text:
+                raise Exception("OpenAI API request failed.")
+            return {"replies": [ChatMessage.from_assistant('{"relevant_statements": ["c", "d"], "score": 1}')]}
+
+        monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
+
+        questions = ["Which is the most popular global sport?", "Who created the Python language?"]
+        contexts = [
+            [
+                "The popularity of sports can be measured in various ways, including TV viewership, social media "
+                "presence, number of participants, and economic impact. Football is undoubtedly the world's most "
+                "popular sport with major events like the FIFA World Cup and sports personalities like Ronaldo and "
+                "Messi, drawing a followership of more than 4 billion people."
+            ],
+            [
+                "Python, created by Guido van Rossum in the late 1980s, is a high-level general-purpose programming "
+                "language. Its design philosophy emphasizes code readability, and its language constructs aim to help "
+                "programmers write clear, logical code for both small and large-scale software projects."
+            ],
+        ]
+        with caplog.at_level("WARNING", logger="haystack.components.evaluators.context_relevance"):
+            results = component.run(questions=questions, contexts=contexts)
+
+        assert results["score"] == 1
+        assert results["results"][0] == {"relevant_statements": ["c", "d"], "score": 1}
+        assert results["results"][1]["relevant_statements"] == []
+        assert math.isnan(results["results"][1]["score"])
+
+        assert "1 query(s) failed and were excluded from the score." in caplog.text
+
+    @pytest.mark.skipif(
+        not os.environ.get("OPENAI_API_KEY", None),
+        reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.",
+    )
+    @pytest.mark.integration
+    def test_live_run(self):
+        questions = ["Who created the Python language?"]
+        contexts = [["Python, created by Guido van Rossum, is a high-level general-purpose programming language."]]
+
+        evaluator = ContextRelevanceEvaluator(chat_generator=OpenAIChatGenerator(model="gpt-4.1-nano"))
+        result = evaluator.run(questions=questions, contexts=contexts)
+
+        required_fields = {"results"}
+        assert all(field in result for field in required_fields)
+        nested_required_fields = {"score", "relevant_statements"}
+        assert all(field in result["results"][0] for field in nested_required_fields)
+
+        assert "meta" in result
+        assert "prompt_tokens" in result["meta"][0]["usage"]
+        assert "completion_tokens" in result["meta"][0]["usage"]
+        assert "total_tokens" in result["meta"][0]["usage"]
+
+
+class TestContextRelevanceEvaluatorAsync:
+    @pytest.mark.asyncio
+    async def test_run_async_calculates_mean_score(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = ContextRelevanceEvaluator()
+
+        async def chat_generator_run_async(self, *args, **kwargs):
+            if "Football" in kwargs["messages"][0].text:
+                return {"replies": [ChatMessage.from_assistant('{"relevant_statements": ["a", "b"], "score": 1}')]}
+            return {"replies": [ChatMessage.from_assistant('{"relevant_statements": [], "score": 0}')]}
+
+        monkeypatch.setattr(
+            "haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run_async", chat_generator_run_async
+        )
+
+        questions = ["Which is the most popular global sport?", "Who created the Python language?"]
+        contexts = [
+            ["Football is the world's most popular sport."],
+            ["Python is a cross-platform programming language."],
+        ]
+
+        results = await component.run_async(questions=questions, contexts=contexts)
+
+        assert results == {
+            "results": [{"score": 1, "relevant_statements": ["a", "b"]}, {"score": 0, "relevant_statements": []}],
+            "score": 0.5,
+            "meta": None,
+            "individual_scores": [1, 0],
+        }
+
+    @pytest.mark.asyncio
+    async def test_run_async_returns_nan_raise_on_failure_false(self, monkeypatch, caplog):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = ContextRelevanceEvaluator(raise_on_failure=False)
+
+        async def chat_generator_run_async(self, *args, **kwargs):
+            if "Python" in kwargs["messages"][0].text:
+                raise Exception("OpenAI API request failed.")
+            return {"replies": [ChatMessage.from_assistant('{"relevant_statements": ["c", "d"], "score": 1}')]}
+
+        monkeypatch.setattr(
+            "haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run_async", chat_generator_run_async
+        )
+
+        questions = ["Which is the most popular global sport?", "Who created the Python language?"]
+        contexts = [["Football is popular."], ["Python was created by Guido van Rossum."]]
+
+        with caplog.at_level("WARNING", logger="haystack.components.evaluators.context_relevance"):
+            results = await component.run_async(questions=questions, contexts=contexts)
+
+        assert results["score"] == 1
+        assert results["results"][0] == {"relevant_statements": ["c", "d"], "score": 1}
+        assert results["results"][1]["relevant_statements"] == []
+        assert math.isnan(results["results"][1]["score"])
+
+        assert "1 query(s) failed and were excluded from the score." in caplog.text
+
+    @pytest.mark.asyncio
+    @pytest.mark.skipif(
+        not os.environ.get("OPENAI_API_KEY", None),
+        reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.",
+    )
+    @pytest.mark.integration
+    async def test_live_run_async(self):
+        questions = ["Who created the Python language?"]
+        contexts = [["Python, created by Guido van Rossum, is a high-level general-purpose programming language."]]
+
+        evaluator = ContextRelevanceEvaluator(chat_generator=OpenAIChatGenerator(model="gpt-4.1-nano"))
+        result = await evaluator.run_async(questions=questions, contexts=contexts)
+
+        required_fields = {"results"}
+        assert all(field in result for field in required_fields)
+        nested_required_fields = {"score", "relevant_statements"}
+        assert all(field in result["results"][0] for field in nested_required_fields)
+
+        assert "meta" in result
+        assert "prompt_tokens" in result["meta"][0]["usage"]
+        assert "completion_tokens" in result["meta"][0]["usage"]
+        assert "total_tokens" in result["meta"][0]["usage"]

test/components/evaluators/test_document_map.py

@@ -0,0 +1,149 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+from haystack import Document, default_from_dict
+from haystack.components.evaluators.document_map import DocumentMAPEvaluator
+
+
+def test_to_dict():
+    evaluator = DocumentMAPEvaluator()
+    data = evaluator.to_dict()
+    assert data == {
+        "type": "haystack.components.evaluators.document_map.DocumentMAPEvaluator",
+        "init_parameters": {"document_comparison_field": "content"},
+    }
+
+
+def test_from_dict():
+    data = {
+        "type": "haystack.components.evaluators.document_map.DocumentMAPEvaluator",
+        "init_parameters": {"document_comparison_field": "id"},
+    }
+    evaluator = default_from_dict(DocumentMAPEvaluator, data)
+    assert evaluator.document_comparison_field == "id"
+
+
+def test_run_with_id_comparison():
+    evaluator = DocumentMAPEvaluator(document_comparison_field="id")
+    result = evaluator.run(
+        ground_truth_documents=[[Document(id="doc1", content="foo")], [Document(id="doc2", content="bar")]],
+        retrieved_documents=[[Document(id="doc1", content="different")], [Document(id="wrong", content="bar")]],
+    )
+    assert result == {"individual_scores": [1.0, 0.0], "score": 0.5}
+
+
+def test_run_with_meta_comparison():
+    evaluator = DocumentMAPEvaluator(document_comparison_field="meta.file_id")
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="x", meta={"file_id": "a"})],
+            [Document(content="y", meta={"file_id": "b"})],
+        ],
+        retrieved_documents=[
+            [Document(content="z", meta={"file_id": "a"})],
+            [Document(content="w", meta={"file_id": "c"})],
+        ],
+    )
+    assert result == {"individual_scores": [1.0, 0.0], "score": 0.5}
+
+
+def test_run_with_nested_meta_comparison():
+    evaluator = DocumentMAPEvaluator(document_comparison_field="meta.source.url")
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="x", meta={"source": {"url": "https://a.com"}})],
+            [Document(content="y", meta={"source": {"url": "https://b.com"}})],
+        ],
+        retrieved_documents=[
+            [Document(content="z", meta={"source": {"url": "https://a.com"}})],
+            [Document(content="w", meta={"source": {"url": "https://c.com"}})],
+        ],
+    )
+    assert result == {"individual_scores": [1.0, 0.0], "score": 0.5}
+
+
+def test_run_with_all_matching():
+    evaluator = DocumentMAPEvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        retrieved_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+    )
+
+    assert result == {"individual_scores": [1.0, 1.0], "score": 1.0}
+
+
+def test_run_with_no_matching():
+    evaluator = DocumentMAPEvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        retrieved_documents=[[Document(content="Paris")], [Document(content="London")]],
+    )
+
+    assert result == {"individual_scores": [0.0, 0.0], "score": 0.0}
+
+
+def test_run_with_partial_matching():
+    evaluator = DocumentMAPEvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+    )
+
+    assert result == {"individual_scores": [1.0, 0.0], "score": 0.5}
+
+
+def test_run_with_complex_data():
+    evaluator = DocumentMAPEvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="9th")],
+            [Document(content="classical music"), Document(content="classical")],
+            [Document(content="11th century"), Document(content="the 11th")],
+            [Document(content="Denmark, Iceland and Norway")],
+            [Document(content="10th century"), Document(content="10th")],
+        ],
+        retrieved_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="10th century"), Document(content="9th")],
+            [Document(content="classical"), Document(content="rock music"), Document(content="dubstep")],
+            [Document(content="11th"), Document(content="the 11th"), Document(content="11th century")],
+            [Document(content="Denmark"), Document(content="Norway"), Document(content="Iceland")],
+            [
+                Document(content="10th century"),
+                Document(content="the first half of the 10th century"),
+                Document(content="10th"),
+                Document(content="10th"),
+            ],
+        ],
+    )
+    assert result == {
+        "individual_scores": [
+            1.0,
+            pytest.approx(0.8333333333333333),
+            1.0,
+            pytest.approx(0.5833333333333333),
+            0.0,
+            pytest.approx(0.8055555555555555),
+        ],
+        "score": pytest.approx(0.7037037037037037),
+    }
+
+
+def test_run_with_different_lengths():
+    with pytest.raises(ValueError):
+        evaluator = DocumentMAPEvaluator()
+        evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")]],
+            retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+        )
+
+    with pytest.raises(ValueError):
+        evaluator = DocumentMAPEvaluator()
+        evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Berlin")]],
+        )

test/components/evaluators/test_document_mrr.py

@@ -0,0 +1,152 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+from haystack import Document, default_from_dict
+from haystack.components.evaluators.document_mrr import DocumentMRREvaluator
+
+
+def test_to_dict():
+    evaluator = DocumentMRREvaluator()
+    data = evaluator.to_dict()
+    assert data == {
+        "type": "haystack.components.evaluators.document_mrr.DocumentMRREvaluator",
+        "init_parameters": {"document_comparison_field": "content"},
+    }
+
+
+def test_to_dict_custom_field():
+    evaluator = DocumentMRREvaluator(document_comparison_field="id")
+    data = evaluator.to_dict()
+    assert data == {
+        "type": "haystack.components.evaluators.document_mrr.DocumentMRREvaluator",
+        "init_parameters": {"document_comparison_field": "id"},
+    }
+
+
+def test_from_dict():
+    data = {
+        "type": "haystack.components.evaluators.document_mrr.DocumentMRREvaluator",
+        "init_parameters": {"document_comparison_field": "id"},
+    }
+    evaluator = default_from_dict(DocumentMRREvaluator, data)
+    assert evaluator.document_comparison_field == "id"
+
+
+def test_run_with_id_comparison():
+    evaluator = DocumentMRREvaluator(document_comparison_field="id")
+    result = evaluator.run(
+        ground_truth_documents=[[Document(id="doc1", content="foo")], [Document(id="doc2", content="bar")]],
+        retrieved_documents=[[Document(id="doc1", content="different")], [Document(id="wrong", content="bar")]],
+    )
+    assert result == {"individual_scores": [1.0, 0.0], "score": 0.5}
+
+
+def test_run_with_meta_comparison():
+    evaluator = DocumentMRREvaluator(document_comparison_field="meta.file_id")
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="x", meta={"file_id": "a"})],
+            [Document(content="y", meta={"file_id": "b"})],
+        ],
+        retrieved_documents=[
+            [Document(content="z", meta={"file_id": "a"})],
+            [Document(content="w", meta={"file_id": "c"})],
+        ],
+    )
+    assert result == {"individual_scores": [1.0, 0.0], "score": 0.5}
+
+
+def test_run_with_nested_meta_comparison():
+    evaluator = DocumentMRREvaluator(document_comparison_field="meta.source.url")
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="x", meta={"source": {"url": "https://a.com"}})],
+            [Document(content="y", meta={"source": {"url": "https://b.com"}})],
+        ],
+        retrieved_documents=[
+            [Document(content="z", meta={"source": {"url": "https://a.com"}})],
+            [Document(content="w", meta={"source": {"url": "https://c.com"}})],
+        ],
+    )
+    assert result == {"individual_scores": [1.0, 0.0], "score": 0.5}
+
+
+def test_run_with_all_matching():
+    evaluator = DocumentMRREvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        retrieved_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+    )
+
+    assert result == {"individual_scores": [1.0, 1.0], "score": 1.0}
+
+
+def test_run_with_no_matching():
+    evaluator = DocumentMRREvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        retrieved_documents=[[Document(content="Paris")], [Document(content="London")]],
+    )
+
+    assert result == {"individual_scores": [0.0, 0.0], "score": 0.0}
+
+
+def test_run_with_partial_matching():
+    evaluator = DocumentMRREvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+    )
+
+    assert result == {"individual_scores": [1.0, 0.0], "score": 0.5}
+
+
+def test_run_with_complex_data():
+    evaluator = DocumentMRREvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="9th")],
+            [Document(content="classical music"), Document(content="classical")],
+            [Document(content="11th century"), Document(content="the 11th")],
+            [Document(content="Denmark, Iceland and Norway")],
+            [Document(content="10th century"), Document(content="10th")],
+        ],
+        retrieved_documents=[
+            [Document(content="France")],
+            [Document(content="10th century"), Document(content="9th century"), Document(content="9th")],
+            [Document(content="rock music"), Document(content="dubstep"), Document(content="classical")],
+            [Document(content="11th"), Document(content="the 11th"), Document(content="11th century")],
+            [Document(content="Denmark"), Document(content="Norway"), Document(content="Iceland")],
+            [
+                Document(content="10th century"),
+                Document(content="the first half of the 10th century"),
+                Document(content="10th"),
+                Document(content="10th"),
+            ],
+        ],
+    )
+
+    assert result == {
+        "individual_scores": [1.0, 0.5, 0.3333333333333333, 0.5, 0.0, 1.0],
+        "score": pytest.approx(0.555555555555555),
+    }
+
+
+def test_run_with_different_lengths():
+    with pytest.raises(ValueError):
+        evaluator = DocumentMRREvaluator()
+        evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")]],
+            retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+        )
+
+    with pytest.raises(ValueError):
+        evaluator = DocumentMRREvaluator()
+        evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Berlin")]],
+        )

test/components/evaluators/test_document_ndcg.py

@@ -0,0 +1,355 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+from haystack import Document, default_from_dict
+from haystack.components.evaluators.document_ndcg import DocumentNDCGEvaluator
+
+
+def test_run_with_scores():
+    evaluator = DocumentNDCGEvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[
+            [
+                Document(content="doc1", score=3),
+                Document(content="doc2", score=2),
+                Document(content="doc3", score=3),
+                Document(content="doc6", score=2),
+                Document(content="doc7", score=3),
+                Document(content="doc8", score=2),
+            ]
+        ],
+        retrieved_documents=[
+            [
+                Document(content="doc1"),
+                Document(content="doc2"),
+                Document(content="doc3"),
+                Document(content="doc4"),
+                Document(content="doc5"),
+            ]
+        ],
+    )
+    assert result["individual_scores"][0] == pytest.approx(0.6592, abs=1e-4)
+    assert result["score"] == pytest.approx(0.6592, abs=1e-4)
+
+
+def test_run_without_scores():
+    evaluator = DocumentNDCGEvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="France"), Document(content="Paris")]],
+        retrieved_documents=[[Document(content="France"), Document(content="Germany"), Document(content="Paris")]],
+    )
+    assert result["individual_scores"][0] == pytest.approx(0.9197, abs=1e-4)
+    assert result["score"] == pytest.approx(0.9197, abs=1e-4)
+
+
+def test_run_with_multiple_lists_of_docs():
+    evaluator = DocumentNDCGEvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="France"), Document(content="Paris")],
+            [
+                Document(content="doc1", score=3),
+                Document(content="doc2", score=2),
+                Document(content="doc3", score=3),
+                Document(content="doc6", score=2),
+                Document(content="doc7", score=3),
+                Document(content="doc8", score=2),
+            ],
+        ],
+        retrieved_documents=[
+            [Document(content="France"), Document(content="Germany"), Document(content="Paris")],
+            [
+                Document(content="doc1"),
+                Document(content="doc2"),
+                Document(content="doc3"),
+                Document(content="doc4"),
+                Document(content="doc5"),
+            ],
+        ],
+    )
+    assert result["individual_scores"][0] == pytest.approx(0.9197, abs=1e-4)
+    assert result["individual_scores"][1] == pytest.approx(0.6592, abs=1e-4)
+    assert result["score"] == pytest.approx(0.7895, abs=1e-4)
+
+
+def test_run_with_different_lengths():
+    evaluator = DocumentNDCGEvaluator()
+    with pytest.raises(ValueError):
+        evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")]],
+            retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+        )
+    with pytest.raises(ValueError):
+        evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Berlin")]],
+        )
+
+
+def test_run_with_mixed_documents_with_and_without_scores():
+    evaluator = DocumentNDCGEvaluator()
+    with pytest.raises(ValueError):
+        evaluator.run(
+            ground_truth_documents=[[Document(content="France", score=3), Document(content="Paris")]],
+            retrieved_documents=[[Document(content="France"), Document(content="Germany"), Document(content="Paris")]],
+        )
+
+
+def test_run_empty_retrieved():
+    evaluator = DocumentNDCGEvaluator()
+    result = evaluator.run(ground_truth_documents=[[Document(content="France")]], retrieved_documents=[[]])
+    assert result["individual_scores"] == [0.0]
+    assert result["score"] == 0.0
+
+
+def test_run_empty_ground_truth():
+    evaluator = DocumentNDCGEvaluator()
+    result = evaluator.run(ground_truth_documents=[[]], retrieved_documents=[[Document(content="France")]])
+    assert result["individual_scores"] == [0.0]
+    assert result["score"] == 0.0
+
+
+def test_run_empty_retrieved_and_empty_ground_truth():
+    evaluator = DocumentNDCGEvaluator()
+    result = evaluator.run(ground_truth_documents=[[]], retrieved_documents=[[]])
+    assert result["individual_scores"] == [0.0]
+    assert result["score"] == 0.0
+
+
+def test_run_no_retrieved():
+    evaluator = DocumentNDCGEvaluator()
+    with pytest.raises(ValueError):
+        _ = evaluator.run(ground_truth_documents=[[Document(content="France")]], retrieved_documents=[])
+
+
+def test_run_no_ground_truth():
+    evaluator = DocumentNDCGEvaluator()
+    with pytest.raises(ValueError):
+        evaluator.run(ground_truth_documents=[], retrieved_documents=[[Document(content="France")]])
+
+
+def test_run_no_retrieved_and_no_ground_truth():
+    evaluator = DocumentNDCGEvaluator()
+    with pytest.raises(ValueError):
+        evaluator.run(ground_truth_documents=[], retrieved_documents=[])
+
+
+def test_calculate_dcg_with_scores():
+    evaluator = DocumentNDCGEvaluator()
+    gt_docs = [
+        Document(content="doc1", score=3),
+        Document(content="doc2", score=2),
+        Document(content="doc3", score=3),
+        Document(content="doc4", score=0),
+        Document(content="doc5", score=1),
+        Document(content="doc6", score=2),
+    ]
+    ret_docs = [
+        Document(content="doc1"),
+        Document(content="doc2"),
+        Document(content="doc3"),
+        Document(content="doc4"),
+        Document(content="doc5"),
+        Document(content="doc6"),
+    ]
+    dcg = evaluator.calculate_dcg(gt_docs, ret_docs)
+    assert dcg == pytest.approx(6.8611, abs=1e-4)
+
+
+def test_calculate_dcg_without_scores():
+    evaluator = DocumentNDCGEvaluator()
+    gt_docs = [Document(content="doc1"), Document(content="doc2")]
+    ret_docs = [Document(content="doc2"), Document(content="doc3"), Document(content="doc1")]
+    dcg = evaluator.calculate_dcg(gt_docs, ret_docs)
+    assert dcg == pytest.approx(1.5, abs=1e-4)
+
+
+def test_calculate_dcg_empty():
+    evaluator = DocumentNDCGEvaluator()
+    gt_docs = [Document(content="doc1")]
+    ret_docs = []
+    dcg = evaluator.calculate_dcg(gt_docs, ret_docs)
+    assert dcg == 0
+
+
+def test_calculate_idcg_with_scores():
+    evaluator = DocumentNDCGEvaluator()
+    gt_docs = [
+        Document(content="doc1", score=3),
+        Document(content="doc2", score=3),
+        Document(content="doc3", score=2),
+        Document(content="doc4", score=3),
+        Document(content="doc5", score=2),
+        Document(content="doc6", score=2),
+    ]
+    idcg = evaluator.calculate_idcg(gt_docs)
+    assert idcg == pytest.approx(8.7403, abs=1e-4)
+
+
+def test_calculate_idcg_without_scores():
+    evaluator = DocumentNDCGEvaluator()
+    gt_docs = [Document(content="doc1"), Document(content="doc2"), Document(content="doc3")]
+    idcg = evaluator.calculate_idcg(gt_docs)
+    assert idcg == pytest.approx(2.1309, abs=1e-4)
+
+
+def test_calculate_idcg_empty():
+    evaluator = DocumentNDCGEvaluator()
+    gt_docs = []
+    idcg = evaluator.calculate_idcg(gt_docs)
+    assert idcg == 0
+
+
+def test_to_dict_default():
+    evaluator = DocumentNDCGEvaluator()
+    data = evaluator.to_dict()
+    assert data == {
+        "type": "haystack.components.evaluators.document_ndcg.DocumentNDCGEvaluator",
+        "init_parameters": {"document_comparison_field": "content"},
+    }
+
+
+def test_to_dict_custom_field():
+    evaluator = DocumentNDCGEvaluator(document_comparison_field="id")
+    data = evaluator.to_dict()
+    assert data == {
+        "type": "haystack.components.evaluators.document_ndcg.DocumentNDCGEvaluator",
+        "init_parameters": {"document_comparison_field": "id"},
+    }
+
+
+def test_from_dict():
+    data = {
+        "type": "haystack.components.evaluators.document_ndcg.DocumentNDCGEvaluator",
+        "init_parameters": {"document_comparison_field": "id"},
+    }
+    evaluator = default_from_dict(DocumentNDCGEvaluator, data)
+    assert evaluator.document_comparison_field == "id"
+
+
+def test_run_with_id_comparison():
+    # Documents with same content but different IDs — id comparison
+    # must match on id, not content
+    evaluator = DocumentNDCGEvaluator(document_comparison_field="id")
+    result = evaluator.run(
+        ground_truth_documents=[[Document(id="doc1", content="France"), Document(id="doc2", content="Paris")]],
+        retrieved_documents=[
+            [
+                Document(id="doc1", content="different text"),
+                Document(id="doc3", content="Germany"),
+                Document(id="doc2", content="also different"),
+            ]
+        ],
+    )
+    assert result["individual_scores"][0] == pytest.approx(0.9197, abs=1e-4)
+    assert result["score"] == pytest.approx(0.9197, abs=1e-4)
+
+
+def test_run_with_id_comparison_no_match():
+    evaluator = DocumentNDCGEvaluator(document_comparison_field="id")
+    result = evaluator.run(
+        ground_truth_documents=[[Document(id="doc1", content="France")]],
+        retrieved_documents=[[Document(id="doc99", content="France")]],
+    )
+    # Same content, different ID — should NOT match when comparing by id
+    assert result["individual_scores"] == [0.0]
+    assert result["score"] == 0.0
+
+
+def test_run_with_meta_comparison():
+    evaluator = DocumentNDCGEvaluator(document_comparison_field="meta.file_id")
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="France", meta={"file_id": "f1"}), Document(content="Paris", meta={"file_id": "f2"})]
+        ],
+        retrieved_documents=[
+            [
+                Document(content="different", meta={"file_id": "f1"}),
+                Document(content="irrelevant", meta={"file_id": "f99"}),
+                Document(content="also different", meta={"file_id": "f2"}),
+            ]
+        ],
+    )
+    assert result["individual_scores"][0] == pytest.approx(0.9197, abs=1e-4)
+    assert result["score"] == pytest.approx(0.9197, abs=1e-4)
+
+
+def test_run_with_nested_meta_comparison():
+    evaluator = DocumentNDCGEvaluator(document_comparison_field="meta.source.url")
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="x", meta={"source": {"url": "https://a.com"}})]],
+        retrieved_documents=[[Document(content="z", meta={"source": {"url": "https://a.com"}})]],
+    )
+    assert result["individual_scores"] == [1.0]
+    assert result["score"] == 1.0
+
+
+def test_run_with_meta_missing_key_treated_as_no_match():
+    # Documents missing the meta key should not match anything
+    evaluator = DocumentNDCGEvaluator(document_comparison_field="meta.file_id")
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="France", meta={"file_id": "f1"})]],
+        retrieved_documents=[[Document(content="France", meta={})]],
+    )
+    assert result["individual_scores"] == [0.0]
+    assert result["score"] == 0.0
+
+
+def test_run_with_id_comparison_with_scores():
+    # Verify that relevance scores are honoured when comparing by id
+    evaluator = DocumentNDCGEvaluator(document_comparison_field="id")
+    result = evaluator.run(
+        ground_truth_documents=[
+            [
+                Document(id="doc1", content="foo", score=3),
+                Document(id="doc2", content="bar", score=2),
+                Document(id="doc3", content="baz", score=3),
+                Document(id="doc6", content="qux", score=2),
+                Document(id="doc7", content="quux", score=3),
+                Document(id="doc8", content="corge", score=2),
+            ]
+        ],
+        retrieved_documents=[
+            [
+                Document(id="doc1", content="x"),
+                Document(id="doc2", content="y"),
+                Document(id="doc3", content="z"),
+                Document(id="doc4", content="w"),
+                Document(id="doc5", content="v"),
+            ]
+        ],
+    )
+    assert result["individual_scores"][0] == pytest.approx(0.6592, abs=1e-4)
+    assert result["score"] == pytest.approx(0.6592, abs=1e-4)
+
+
+def test_unsupported_comparison_field_raises():
+    evaluator = DocumentNDCGEvaluator(document_comparison_field="embedding")
+    with pytest.raises(ValueError, match="Unsupported document_comparison_field"):
+        evaluator.run(
+            ground_truth_documents=[[Document(content="France")]], retrieved_documents=[[Document(content="France")]]
+        )
+
+
+def test_run_with_meta_missing_key_can_still_reach_perfect_ndcg():
+    """
+    Regression test for the IDCG/DCG inflation bug: ground truth documents that
+    cannot be matched (missing the configured meta key) must be excluded from
+    IDCG too, otherwise NDCG can never reach 1.0 even for a perfect retrieval.
+    """
+    evaluator = DocumentNDCGEvaluator(document_comparison_field="meta.file_id")
+    result = evaluator.run(
+        ground_truth_documents=[
+            [
+                Document(content="France", meta={"file_id": "f1"}),
+                Document(content="unmatchable", meta={}),  # no file_id -> cannot be matched
+            ]
+        ],
+        retrieved_documents=[[Document(content="France", meta={"file_id": "f1"})]],
+    )
+    # Perfect retrieval of the one matchable document should yield NDCG of exactly 1.0
+    assert result["individual_scores"] == [1.0]
+    assert result["score"] == 1.0

test/components/evaluators/test_document_recall.py

@@ -0,0 +1,267 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+from haystack import default_from_dict
+from haystack.components.evaluators.document_recall import DocumentRecallEvaluator, RecallMode
+from haystack.dataclasses import Document
+
+
+def test_init_with_unknown_mode_string():
+    with pytest.raises(ValueError):
+        DocumentRecallEvaluator(mode="unknown_mode")
+
+
+def test_init_with_string_mode():
+    evaluator = DocumentRecallEvaluator(mode="single_hit")
+    assert evaluator.mode == RecallMode.SINGLE_HIT
+
+    evaluator = DocumentRecallEvaluator(mode="multi_hit")
+    assert evaluator.mode == RecallMode.MULTI_HIT
+
+
+def test_init_default_comparison_field():
+    evaluator = DocumentRecallEvaluator()
+    assert evaluator.document_comparison_field == "content"
+
+
+def test_run_with_id_comparison():
+    evaluator = DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT, document_comparison_field="id")
+    result = evaluator.run(
+        ground_truth_documents=[[Document(id="doc1", content="foo")], [Document(id="doc2", content="bar")]],
+        retrieved_documents=[[Document(id="doc1", content="different")], [Document(id="wrong", content="bar")]],
+    )
+    assert result == {"individual_scores": [1.0, 0.0], "score": 0.5}
+
+
+def test_run_with_meta_comparison():
+    evaluator = DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT, document_comparison_field="meta.file_id")
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="x", meta={"file_id": "a"}), Document(content="y", meta={"file_id": "b"})]
+        ],
+        retrieved_documents=[
+            [Document(content="z", meta={"file_id": "a"}), Document(content="w", meta={"file_id": "c"})]
+        ],
+    )
+    assert result == {"individual_scores": [0.5], "score": 0.5}
+
+
+def test_run_with_nested_meta_comparison():
+    evaluator = DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT, document_comparison_field="meta.source.url")
+    result = evaluator.run(
+        ground_truth_documents=[
+            [
+                Document(content="x", meta={"source": {"url": "https://a.com"}}),
+                Document(content="y", meta={"source": {"url": "https://b.com"}}),
+            ]
+        ],
+        retrieved_documents=[
+            [
+                Document(content="z", meta={"source": {"url": "https://a.com"}}),
+                Document(content="w", meta={"source": {"url": "https://c.com"}}),
+            ]
+        ],
+    )
+    assert result == {"individual_scores": [0.5], "score": 0.5}
+
+
+class TestDocumentRecallEvaluatorSingleHit:
+    @pytest.fixture
+    def evaluator(self):
+        return DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT)
+
+    def test_run_with_all_matching(self, evaluator):
+        result = evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        )
+        assert all(isinstance(individual_score, float) for individual_score in result["individual_scores"])
+        assert result == {"individual_scores": [1.0, 1.0], "score": 1.0}
+
+    def test_run_with_no_matching(self, evaluator):
+        result = evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Paris")], [Document(content="London")]],
+        )
+        assert all(isinstance(individual_score, float) for individual_score in result["individual_scores"])
+        assert result == {"individual_scores": [0.0, 0.0], "score": 0.0}
+
+    def test_run_with_partial_matching(self, evaluator):
+        result = evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+        )
+        assert all(isinstance(individual_score, float) for individual_score in result["individual_scores"])
+        assert result == {"individual_scores": [1.0, 0.0], "score": 0.5}
+
+    def test_run_with_complex_data(self, evaluator):
+        result = evaluator.run(
+            ground_truth_documents=[
+                [Document(content="France")],
+                [Document(content="9th century"), Document(content="9th")],
+                [Document(content="classical music"), Document(content="classical")],
+                [Document(content="11th century"), Document(content="the 11th")],
+                [Document(content="Denmark, Iceland and Norway")],
+                [Document(content="10th century"), Document(content="10th")],
+            ],
+            retrieved_documents=[
+                [Document(content="France")],
+                [Document(content="9th century"), Document(content="10th century"), Document(content="9th")],
+                [Document(content="classical"), Document(content="rock music"), Document(content="dubstep")],
+                [Document(content="11th"), Document(content="the 11th"), Document(content="11th century")],
+                [Document(content="Denmark"), Document(content="Norway"), Document(content="Iceland")],
+                [
+                    Document(content="10th century"),
+                    Document(content="the first half of the 10th century"),
+                    Document(content="10th"),
+                    Document(content="10th"),
+                ],
+            ],
+        )
+        assert all(isinstance(individual_score, float) for individual_score in result["individual_scores"])
+        assert result == {"individual_scores": [1, 1, 1, 1, 0, 1], "score": 0.8333333333333334}
+
+    def test_run_with_different_lengths(self, evaluator):
+        with pytest.raises(ValueError):
+            evaluator.run(
+                ground_truth_documents=[[Document(content="Berlin")]],
+                retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+            )
+
+        with pytest.raises(ValueError):
+            evaluator.run(
+                ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+                retrieved_documents=[[Document(content="Berlin")]],
+            )
+
+    def test_to_dict(self, evaluator):
+        data = evaluator.to_dict()
+        assert data == {
+            "type": "haystack.components.evaluators.document_recall.DocumentRecallEvaluator",
+            "init_parameters": {"mode": "single_hit", "document_comparison_field": "content"},
+        }
+
+    def test_from_dict(self):
+        data = {
+            "type": "haystack.components.evaluators.document_recall.DocumentRecallEvaluator",
+            "init_parameters": {"mode": "single_hit"},
+        }
+        new_evaluator = default_from_dict(DocumentRecallEvaluator, data)
+        assert new_evaluator.mode == RecallMode.SINGLE_HIT
+
+
+class TestDocumentRecallEvaluatorMultiHit:
+    @pytest.fixture
+    def evaluator(self):
+        return DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT)
+
+    def test_run_with_all_matching(self, evaluator):
+        result = evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+        )
+        assert all(isinstance(individual_score, float) for individual_score in result["individual_scores"])
+        assert result == {"individual_scores": [1.0, 1.0], "score": 1.0}
+
+    def test_run_with_no_matching(self, evaluator):
+        result = evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Paris")], [Document(content="London")]],
+        )
+        assert all(isinstance(individual_score, float) for individual_score in result["individual_scores"])
+        assert result == {"individual_scores": [0.0, 0.0], "score": 0.0}
+
+    def test_run_with_partial_matching(self, evaluator):
+        result = evaluator.run(
+            ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+            retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+        )
+        assert all(isinstance(individual_score, float) for individual_score in result["individual_scores"])
+        assert result == {"individual_scores": [1.0, 0.0], "score": 0.5}
+
+    def test_run_with_complex_data(self, evaluator):
+        result = evaluator.run(
+            ground_truth_documents=[
+                [Document(content="France")],
+                [Document(content="9th century"), Document(content="9th")],
+                [Document(content="classical music"), Document(content="classical")],
+                [Document(content="11th century"), Document(content="the 11th")],
+                [
+                    Document(content="Denmark"),
+                    Document(content="Iceland"),
+                    Document(content="Norway"),
+                    Document(content="Denmark, Iceland and Norway"),
+                ],
+                [Document(content="10th century"), Document(content="10th")],
+            ],
+            retrieved_documents=[
+                [Document(content="France")],
+                [Document(content="9th century"), Document(content="10th century"), Document(content="9th")],
+                [Document(content="classical"), Document(content="rock music"), Document(content="dubstep")],
+                [Document(content="11th"), Document(content="the 11th"), Document(content="11th century")],
+                [Document(content="Denmark"), Document(content="Norway"), Document(content="Iceland")],
+                [
+                    Document(content="10th century"),
+                    Document(content="the first half of the 10th century"),
+                    Document(content="10th"),
+                    Document(content="10th"),
+                ],
+            ],
+        )
+        assert all(isinstance(individual_score, float) for individual_score in result["individual_scores"])
+        assert result == {"individual_scores": [1.0, 1.0, 0.5, 1.0, 0.75, 1.0], "score": 0.875}
+
+    def test_run_with_different_lengths(self, evaluator):
+        with pytest.raises(ValueError):
+            evaluator.run(
+                ground_truth_documents=[[Document(content="Berlin")]],
+                retrieved_documents=[[Document(content="Berlin")], [Document(content="London")]],
+            )
+
+        with pytest.raises(ValueError):
+            evaluator.run(
+                ground_truth_documents=[[Document(content="Berlin")], [Document(content="Paris")]],
+                retrieved_documents=[[Document(content="Berlin")]],
+            )
+
+    def test_to_dict(self, evaluator):
+        data = evaluator.to_dict()
+        assert data == {
+            "type": "haystack.components.evaluators.document_recall.DocumentRecallEvaluator",
+            "init_parameters": {"mode": "multi_hit", "document_comparison_field": "content"},
+        }
+
+    def test_from_dict(self):
+        data = {
+            "type": "haystack.components.evaluators.document_recall.DocumentRecallEvaluator",
+            "init_parameters": {"mode": "multi_hit"},
+        }
+        new_evaluator = default_from_dict(DocumentRecallEvaluator, data)
+        assert new_evaluator.mode == RecallMode.MULTI_HIT
+
+    def test_empty_ground_truth_documents(self, evaluator):
+        ground_truth_documents = [[]]
+        retrieved_documents = [[Document(content="test")]]
+        score = evaluator.run(ground_truth_documents, retrieved_documents)
+        assert score == {"individual_scores": [0.0], "score": 0.0}
+
+    def test_empty_retrieved_documents(self, evaluator):
+        ground_truth_documents = [[Document(content="test")]]
+        retrieved_documents = [[]]
+        score = evaluator.run(ground_truth_documents, retrieved_documents)
+        assert score == {"individual_scores": [0.0], "score": 0.0}
+
+    def test_empty_string_ground_truth_documents(self, evaluator):
+        ground_truth_documents = [[Document(content="")]]
+        retrieved_documents = [[Document(content="test")]]
+        score = evaluator.run(ground_truth_documents, retrieved_documents)
+        assert score == {"individual_scores": [0.0], "score": 0.0}
+
+    def test_empty_string_retrieved_documents(self, evaluator):
+        ground_truth_documents = [[Document(content="test")]]
+        retrieved_documents = [[Document(content="")]]
+        score = evaluator.run(ground_truth_documents, retrieved_documents)
+        assert score == {"individual_scores": [0.0], "score": 0.0}

test/components/evaluators/test_faithfulness_evaluator.py

@@ -0,0 +1,407 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import math
+import os
+
+import pytest
+
+from haystack import Pipeline
+from haystack.components.evaluators import FaithfulnessEvaluator
+from haystack.components.generators.chat.openai import OpenAIChatGenerator
+from haystack.dataclasses.chat_message import ChatMessage
+from haystack.utils.auth import Secret
+
+
+class TestFaithfulnessEvaluator:
+    def test_init_default(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+
+        component = FaithfulnessEvaluator()
+
+        assert component.instructions == (
+            "Your task is to judge the faithfulness or groundedness of statements based "
+            "on context information. First, please extract statements from a provided predicted "
+            "answer to a question. Second, calculate a faithfulness score for each "
+            "statement made in the predicted answer. The score is 1 if the statement can be "
+            "inferred from the provided context or 0 if it cannot be inferred."
+        )
+        assert component.inputs == [
+            ("questions", list[str]),
+            ("contexts", list[list[str]]),
+            ("predicted_answers", list[str]),
+        ]
+        assert component.outputs == ["statements", "statement_scores"]
+        assert component.examples == [
+            {
+                "inputs": {
+                    "questions": "What is the capital of Germany and when was it founded?",
+                    "contexts": ["Berlin is the capital of Germany and was founded in 1244."],
+                    "predicted_answers": "The capital of Germany, Berlin, was founded in the 13th century.",
+                },
+                "outputs": {
+                    "statements": ["Berlin is the capital of Germany.", "Berlin was founded in 1244."],
+                    "statement_scores": [1, 1],
+                },
+            },
+            {
+                "inputs": {
+                    "questions": "What is the capital of France?",
+                    "contexts": ["Berlin is the capital of Germany."],
+                    "predicted_answers": "Paris",
+                },
+                "outputs": {"statements": ["Paris is the capital of France."], "statement_scores": [0]},
+            },
+            {
+                "inputs": {
+                    "questions": "What is the capital of Italy?",
+                    "contexts": ["Rome is the capital of Italy."],
+                    "predicted_answers": "Rome is the capital of Italy with more than 4 million inhabitants.",
+                },
+                "outputs": {
+                    "statements": ["Rome is the capital of Italy.", "Rome has more than 4 million inhabitants."],
+                    "statement_scores": [1, 0],
+                },
+            },
+        ]
+
+        assert isinstance(component._chat_generator, OpenAIChatGenerator)
+        assert component._chat_generator.api_key.resolve_value() == "test-api-key"
+        assert component._chat_generator.generation_kwargs == {"response_format": {"type": "json_object"}, "seed": 42}
+
+    def test_key_resolved_at_warm_up_not_init(self, monkeypatch):
+        monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+        component = FaithfulnessEvaluator()
+        with pytest.raises(ValueError, match="None of the .* environment variables are set"):
+            component.warm_up()
+
+    def test_init_with_parameters(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = FaithfulnessEvaluator(
+            examples=[
+                {
+                    "inputs": {"predicted_answers": "Damn, this is straight outta hell!!!"},
+                    "outputs": {"custom_score": 1},
+                },
+                {
+                    "inputs": {"predicted_answers": "Football is the most popular sport."},
+                    "outputs": {"custom_score": 0},
+                },
+            ]
+        )
+
+        assert component.examples == [
+            {"inputs": {"predicted_answers": "Damn, this is straight outta hell!!!"}, "outputs": {"custom_score": 1}},
+            {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"custom_score": 0}},
+        ]
+
+        assert isinstance(component._chat_generator, OpenAIChatGenerator)
+        assert component._chat_generator.api_key.resolve_value() == "test-api-key"
+        assert component._chat_generator.generation_kwargs == {"response_format": {"type": "json_object"}, "seed": 42}
+
+    def test_init_with_chat_generator(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        chat_generator = OpenAIChatGenerator(generation_kwargs={"response_format": {"type": "json_object"}, "seed": 42})
+        component = FaithfulnessEvaluator(chat_generator=chat_generator)
+
+        assert component._chat_generator is chat_generator
+
+    def test_to_dict_with_parameters(self, monkeypatch):
+        monkeypatch.setenv("ENV_VAR", "test-api-key")
+        chat_generator = OpenAIChatGenerator(
+            generation_kwargs={"response_format": {"type": "json_object"}, "seed": 42},
+            api_key=Secret.from_env_var("ENV_VAR"),
+        )
+
+        component = FaithfulnessEvaluator(
+            chat_generator=chat_generator,
+            examples=[
+                {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+            ],
+            raise_on_failure=False,
+            progress_bar=False,
+        )
+        data = component.to_dict()
+
+        assert data == {
+            "type": "haystack.components.evaluators.faithfulness.FaithfulnessEvaluator",
+            "init_parameters": {
+                "chat_generator": chat_generator.to_dict(),
+                "examples": [
+                    {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+                ],
+                "progress_bar": False,
+                "raise_on_failure": False,
+            },
+        }
+
+    def test_from_dict(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        chat_generator = OpenAIChatGenerator(generation_kwargs={"response_format": {"type": "json_object"}, "seed": 42})
+
+        data = {
+            "type": "haystack.components.evaluators.faithfulness.FaithfulnessEvaluator",
+            "init_parameters": {
+                "chat_generator": chat_generator.to_dict(),
+                "examples": [
+                    {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+                ],
+            },
+        }
+        component = FaithfulnessEvaluator.from_dict(data)
+        assert isinstance(component._chat_generator, OpenAIChatGenerator)
+        assert component._chat_generator.api_key.resolve_value() == "test-api-key"
+        assert component._chat_generator.generation_kwargs == {"response_format": {"type": "json_object"}, "seed": 42}
+        assert component.examples == [
+            {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+        ]
+
+    def test_pipeline_serde(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+
+        component = FaithfulnessEvaluator()
+        pipeline = Pipeline()
+        pipeline.add_component("evaluator", component)
+
+        serialized_pipeline = pipeline.dumps()
+        deserialized_pipeline = Pipeline.loads(serialized_pipeline)
+        assert deserialized_pipeline == pipeline
+
+    def test_run_calculates_mean_score(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = FaithfulnessEvaluator()
+
+        def chat_generator_run(self, *args, **kwargs):
+            if "Football" in kwargs["messages"][0].text:
+                return {
+                    "replies": [ChatMessage.from_assistant('{"statements": ["a", "b"], "statement_scores": [1, 0]}')]
+                }
+            return {"replies": [ChatMessage.from_assistant('{"statements": ["c", "d"], "statement_scores": [1, 1]}')]}
+
+        monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
+
+        questions = ["Which is the most popular global sport?", "Who created the Python language?"]
+        contexts = [
+            [
+                "The popularity of sports can be measured in various ways, including TV viewership, social media "
+                "presence, number of participants, and economic impact. Football is undoubtedly the world's most "
+                "popular sport with major events like the FIFA World Cup and sports personalities like Ronaldo and "
+                "Messi, drawing a followership of more than 4 billion people."
+            ],
+            [
+                "Python, created by Guido van Rossum in the late 1980s, is a high-level general-purpose programming "
+                "language. Its design philosophy emphasizes code readability, and its language constructs aim to help "
+                "programmers write clear, logical code for both small and large-scale software projects."
+            ],
+        ]
+        predicted_answers = [
+            "Football is the most popular sport with around 4 billion followers worldwide.",
+            "Python is a high-level general-purpose programming language that was created by George Lucas.",
+        ]
+        results = component.run(questions=questions, contexts=contexts, predicted_answers=predicted_answers)
+        assert results == {
+            "individual_scores": [0.5, 1],
+            "results": [
+                {"score": 0.5, "statement_scores": [1, 0], "statements": ["a", "b"]},
+                {"score": 1, "statement_scores": [1, 1], "statements": ["c", "d"]},
+            ],
+            "score": 0.75,
+            "meta": None,
+        }
+
+    def test_run_no_statements_extracted(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = FaithfulnessEvaluator()
+
+        def chat_generator_run(self, *args, **kwargs):
+            if "Football" in kwargs["messages"][0].text:
+                return {
+                    "replies": [ChatMessage.from_assistant('{"statements": ["a", "b"], "statement_scores": [1, 0]}')]
+                }
+            return {"replies": [ChatMessage.from_assistant('{"statements": [], "statement_scores": []}')]}
+
+        monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
+
+        questions = ["Which is the most popular global sport?", "Who created the Python language?"]
+        contexts = [
+            [
+                "The popularity of sports can be measured in various ways, including TV viewership, social media "
+                "presence, number of participants, and economic impact. Football is undoubtedly the world's most "
+                "popular sport with major events like the FIFA World Cup and sports personalities like Ronaldo and "
+                "Messi, drawing a followership of more than 4 billion people."
+            ],
+            [],
+        ]
+        predicted_answers = [
+            "Football is the most popular sport with around 4 billion followers worldwide.",
+            "I don't know.",
+        ]
+        results = component.run(questions=questions, contexts=contexts, predicted_answers=predicted_answers)
+        assert results == {
+            "individual_scores": [0.5, 0],
+            "results": [
+                {"score": 0.5, "statement_scores": [1, 0], "statements": ["a", "b"]},
+                {"score": 0, "statement_scores": [], "statements": []},
+            ],
+            "score": 0.25,
+            "meta": None,
+        }
+
+    def test_run_missing_parameters(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = FaithfulnessEvaluator()
+        with pytest.raises(ValueError, match="LLM evaluator expected input parameter"):
+            component.run()
+
+    def test_run_returns_nan_raise_on_failure_false(self, monkeypatch, caplog):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = FaithfulnessEvaluator(raise_on_failure=False)
+
+        def chat_generator_run(self, *args, **kwargs):
+            if "Python" in kwargs["messages"][0].text:
+                raise Exception("OpenAI API request failed.")
+            return {"replies": [ChatMessage.from_assistant('{"statements": ["c", "d"], "statement_scores": [1, 1]}')]}
+
+        monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
+
+        questions = ["Which is the most popular global sport?", "Who created the Python language?"]
+        contexts = [
+            [
+                "The popularity of sports can be measured in various ways, including TV viewership, social media "
+                "presence, number of participants, and economic impact. Football is undoubtedly the world's most "
+                "popular sport with major events like the FIFA World Cup and sports personalities like Ronaldo and "
+                "Messi, drawing a followership of more than 4 billion people."
+            ],
+            [
+                "Python, created by Guido van Rossum in the late 1980s, is a high-level general-purpose programming "
+                "language. Its design philosophy emphasizes code readability, and its language constructs aim to help "
+                "programmers write clear, logical code for both small and large-scale software projects."
+            ],
+        ]
+        predicted_answers = [
+            "Football is the most popular sport with around 4 billion followers worldwide.",
+            "Guido van Rossum.",
+        ]
+        with caplog.at_level("WARNING", logger="haystack.components.evaluators.faithfulness"):
+            results = component.run(questions=questions, contexts=contexts, predicted_answers=predicted_answers)
+
+        assert results["score"] == 1.0
+
+        assert results["individual_scores"][0] == 1.0
+        assert math.isnan(results["individual_scores"][1])
+
+        assert results["results"][0] == {"statements": ["c", "d"], "statement_scores": [1, 1], "score": 1.0}
+
+        assert results["results"][1]["statements"] == []
+        assert results["results"][1]["statement_scores"] == []
+        assert math.isnan(results["results"][1]["score"])
+
+        assert "1 query(s) failed and were excluded from the score." in caplog.text
+
+    @pytest.mark.skipif(
+        not os.environ.get("OPENAI_API_KEY", None),
+        reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.",
+    )
+    @pytest.mark.integration
+    def test_live_run(self):
+        questions = ["What is Python and who created it?"]
+        contexts = [["Python is a programming language created by Guido van Rossum."]]
+        predicted_answers = ["Python is a programming language created by George Lucas."]
+        evaluator = FaithfulnessEvaluator(chat_generator=OpenAIChatGenerator(model="gpt-4.1-nano"))
+        result = evaluator.run(questions=questions, contexts=contexts, predicted_answers=predicted_answers)
+
+        required_fields = {"individual_scores", "results", "score"}
+        assert all(field in result for field in required_fields)
+        nested_required_fields = {"score", "statement_scores", "statements"}
+        assert all(field in result["results"][0] for field in nested_required_fields)
+
+        # assert that metadata is present in the result
+        assert "meta" in result
+        assert "prompt_tokens" in result["meta"][0]["usage"]
+        assert "completion_tokens" in result["meta"][0]["usage"]
+        assert "total_tokens" in result["meta"][0]["usage"]
+
+
+class TestFaithfulnessEvaluatorAsync:
+    @pytest.mark.asyncio
+    async def test_run_async_calculates_mean_score(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = FaithfulnessEvaluator()
+
+        async def chat_generator_run_async(self, *args, **kwargs):
+            if "Football" in kwargs["messages"][0].text:
+                return {
+                    "replies": [ChatMessage.from_assistant('{"statements": ["a", "b"], "statement_scores": [1, 0]}')]
+                }
+            return {"replies": [ChatMessage.from_assistant('{"statements": ["c", "d"], "statement_scores": [1, 1]}')]}
+
+        monkeypatch.setattr(
+            "haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run_async", chat_generator_run_async
+        )
+
+        questions = ["Which is the most popular global sport?", "Who created the Python language?"]
+        contexts = [["Football is the world's most popular sport."], ["Python was created by Guido van Rossum."]]
+        predicted_answers = ["Football is the most popular sport.", "Python is a language created by George Lucas."]
+        results = await component.run_async(questions=questions, contexts=contexts, predicted_answers=predicted_answers)
+        assert results == {
+            "individual_scores": [0.5, 1.0],
+            "results": [
+                {"score": 0.5, "statement_scores": [1, 0], "statements": ["a", "b"]},
+                {"score": 1.0, "statement_scores": [1, 1], "statements": ["c", "d"]},
+            ],
+            "score": 0.75,
+            "meta": None,
+        }
+
+    @pytest.mark.asyncio
+    async def test_run_async_returns_nan_raise_on_failure_false(self, monkeypatch, caplog):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = FaithfulnessEvaluator(raise_on_failure=False)
+
+        async def chat_generator_run_async(self, *args, **kwargs):
+            if "Python" in kwargs["messages"][0].text:
+                raise Exception("OpenAI API request failed.")
+            return {"replies": [ChatMessage.from_assistant('{"statements": ["c", "d"], "statement_scores": [1, 1]}')]}
+
+        monkeypatch.setattr(
+            "haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run_async", chat_generator_run_async
+        )
+
+        questions = ["Which is the most popular global sport?", "Who created the Python language?"]
+        contexts = [["Football is popular."], ["Python was created by Guido."]]
+        predicted_answers = ["Football is popular.", "Guido van Rossum."]
+
+        with caplog.at_level("WARNING", logger="haystack.components.evaluators.faithfulness"):
+            results = await component.run_async(
+                questions=questions, contexts=contexts, predicted_answers=predicted_answers
+            )
+
+        assert results["score"] == 1.0
+        assert results["individual_scores"][0] == 1.0
+        assert math.isnan(results["individual_scores"][1])
+        assert "1 query(s) failed and were excluded from the score." in caplog.text
+
+    @pytest.mark.asyncio
+    @pytest.mark.skipif(
+        not os.environ.get("OPENAI_API_KEY", None),
+        reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.",
+    )
+    @pytest.mark.integration
+    async def test_live_run_async(self):
+        questions = ["What is Python and who created it?"]
+        contexts = [["Python is a programming language created by Guido van Rossum."]]
+        predicted_answers = ["Python is a programming language created by George Lucas."]
+        evaluator = FaithfulnessEvaluator(chat_generator=OpenAIChatGenerator(model="gpt-4.1-nano"))
+        result = await evaluator.run_async(questions=questions, contexts=contexts, predicted_answers=predicted_answers)
+
+        required_fields = {"individual_scores", "results", "score"}
+        assert all(field in result for field in required_fields)
+        nested_required_fields = {"score", "statement_scores", "statements"}
+        assert all(field in result["results"][0] for field in nested_required_fields)
+
+        # assert that metadata is present in the result
+        assert "meta" in result
+        assert "prompt_tokens" in result["meta"][0]["usage"]
+        assert "completion_tokens" in result["meta"][0]["usage"]
+        assert "total_tokens" in result["meta"][0]["usage"]

test/components/evaluators/test_llm_evaluator.py

@@ -0,0 +1,636 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from unittest.mock import AsyncMock, Mock
+
+import pytest
+
+from haystack import Pipeline
+from haystack.components.evaluators import LLMEvaluator
+from haystack.components.generators.chat.openai import OpenAIChatGenerator
+from haystack.dataclasses.chat_message import ChatMessage
+
+
+class TestLLMEvaluator:
+    def test_init_default(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("predicted_answers", list[str])],
+            outputs=["score"],
+            examples=[
+                {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+            ],
+        )
+        assert component.instructions == "test-instruction"
+        assert component.inputs == [("predicted_answers", list[str])]
+        assert component.outputs == ["score"]
+        assert component.examples == [
+            {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+        ]
+
+        assert isinstance(component._chat_generator, OpenAIChatGenerator)
+        assert component._chat_generator.generation_kwargs == {"response_format": {"type": "json_object"}, "seed": 42}
+
+    def test_key_resolved_at_warm_up_not_init(self, monkeypatch):
+        monkeypatch.delenv("OPENAI_API_KEY", raising=False)
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("predicted_answers", list[str])],
+            outputs=["score"],
+            examples=[
+                {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+            ],
+        )
+        with pytest.raises(ValueError, match="None of the .* environment variables are set"):
+            component.warm_up()
+
+    def test_init_with_chat_generator(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        chat_generator = OpenAIChatGenerator(generation_kwargs={"custom_key": "custom_value"})
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            chat_generator=chat_generator,
+            inputs=[("predicted_answers", list[str])],
+            outputs=["custom_score"],
+            examples=[
+                {"inputs": {"predicted_answers": "answer 1"}, "outputs": {"custom_score": 1}},
+                {"inputs": {"predicted_answers": "answer 2"}, "outputs": {"custom_score": 0}},
+            ],
+        )
+
+        assert component._chat_generator is chat_generator
+
+    def test_init_with_invalid_parameters(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        # Invalid inputs
+        with pytest.raises(ValueError):
+            LLMEvaluator(
+                instructions="test-instruction",
+                inputs={("predicted_answers", list[str])},
+                outputs=["score"],
+                examples=[
+                    {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+                ],
+            )
+        with pytest.raises(ValueError):
+            LLMEvaluator(
+                instructions="test-instruction",
+                inputs=[(list[str], "predicted_answers")],
+                outputs=["score"],
+                examples=[
+                    {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+                ],
+            )
+        with pytest.raises(ValueError):
+            LLMEvaluator(
+                instructions="test-instruction",
+                inputs=[list[str]],
+                outputs=["score"],
+                examples=[
+                    {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+                ],
+            )
+        with pytest.raises(ValueError):
+            LLMEvaluator(
+                instructions="test-instruction",
+                inputs={("predicted_answers", str)},
+                outputs=["score"],
+                examples=[
+                    {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+                ],
+            )
+
+        # Invalid outputs
+        with pytest.raises(ValueError):
+            LLMEvaluator(
+                instructions="test-instruction",
+                inputs=[("predicted_answers", list[str])],
+                outputs="score",
+                examples=[
+                    {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+                ],
+            )
+        with pytest.raises(ValueError):
+            LLMEvaluator(
+                instructions="test-instruction",
+                inputs=[("predicted_answers", list[str])],
+                outputs=[["score"]],
+                examples=[
+                    {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+                ],
+            )
+
+        # Invalid examples
+        with pytest.raises(ValueError):
+            LLMEvaluator(
+                instructions="test-instruction",
+                inputs=[("predicted_answers", list[str])],
+                outputs=["score"],
+                examples={
+                    "inputs": {"predicted_answers": "Damn, this is straight outta hell!!!"},
+                    "outputs": {"custom_score": 1},
+                },
+            )
+        with pytest.raises(ValueError):
+            LLMEvaluator(
+                instructions="test-instruction",
+                inputs=[("predicted_answers", list[str])],
+                outputs=["score"],
+                examples=[
+                    [
+                        {
+                            "inputs": {"predicted_answers": "Damn, this is straight outta hell!!!"},
+                            "outputs": {"custom_score": 1},
+                        }
+                    ]
+                ],
+            )
+        with pytest.raises(ValueError):
+            LLMEvaluator(
+                instructions="test-instruction",
+                inputs=[("predicted_answers", list[str])],
+                outputs=["score"],
+                examples=[
+                    {
+                        "wrong_key": {"predicted_answers": "Damn, this is straight outta hell!!!"},
+                        "outputs": {"custom_score": 1},
+                    }
+                ],
+            )
+        with pytest.raises(ValueError):
+            LLMEvaluator(
+                instructions="test-instruction",
+                inputs=[("predicted_answers", list[str])],
+                outputs=["score"],
+                examples=[
+                    {
+                        "inputs": [{"predicted_answers": "Damn, this is straight outta hell!!!"}],
+                        "outputs": [{"custom_score": 1}],
+                    }
+                ],
+            )
+        with pytest.raises(ValueError):
+            LLMEvaluator(
+                instructions="test-instruction",
+                inputs=[("predicted_answers", list[str])],
+                outputs=["score"],
+                examples=[{"inputs": {1: "Damn, this is straight outta hell!!!"}, "outputs": {2: 1}}],
+            )
+
+    def test_to_dict_default(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        chat_generator = OpenAIChatGenerator(generation_kwargs={"response_format": {"type": "json_object"}, "seed": 42})
+
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("predicted_answers", list[str])],
+            outputs=["score"],
+            examples=[
+                {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+            ],
+        )
+        data = component.to_dict()
+        assert data == {
+            "type": "haystack.components.evaluators.llm_evaluator.LLMEvaluator",
+            "init_parameters": {
+                "chat_generator": chat_generator.to_dict(),
+                "instructions": "test-instruction",
+                "inputs": [["predicted_answers", "list[str]"]],
+                "outputs": ["score"],
+                "raise_on_failure": True,
+                "progress_bar": True,
+                "examples": [
+                    {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+                ],
+            },
+        }
+
+    def test_to_dict_with_parameters(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        chat_generator = OpenAIChatGenerator(generation_kwargs={"response_format": {"type": "json_object"}, "seed": 42})
+
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("predicted_answers", list[str])],
+            outputs=["custom_score"],
+            raise_on_failure=False,
+            examples=[
+                {
+                    "inputs": {"predicted_answers": "Damn, this is straight outta hell!!!"},
+                    "outputs": {"custom_score": 1},
+                },
+                {
+                    "inputs": {"predicted_answers": "Football is the most popular sport."},
+                    "outputs": {"custom_score": 0},
+                },
+            ],
+        )
+        data = component.to_dict()
+        assert data == {
+            "type": "haystack.components.evaluators.llm_evaluator.LLMEvaluator",
+            "init_parameters": {
+                "chat_generator": chat_generator.to_dict(),
+                "instructions": "test-instruction",
+                "inputs": [["predicted_answers", "list[str]"]],
+                "outputs": ["custom_score"],
+                "raise_on_failure": False,
+                "progress_bar": True,
+                "examples": [
+                    {
+                        "inputs": {"predicted_answers": "Damn, this is straight outta hell!!!"},
+                        "outputs": {"custom_score": 1},
+                    },
+                    {
+                        "inputs": {"predicted_answers": "Football is the most popular sport."},
+                        "outputs": {"custom_score": 0},
+                    },
+                ],
+            },
+        }
+
+    def test_from_dict(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        chat_generator = OpenAIChatGenerator(generation_kwargs={"response_format": {"type": "json_object"}, "seed": 42})
+
+        data = {
+            "type": "haystack.components.evaluators.llm_evaluator.LLMEvaluator",
+            "init_parameters": {
+                "chat_generator": chat_generator.to_dict(),
+                "instructions": "test-instruction",
+                "inputs": [["predicted_answers", "list[str]"]],
+                "outputs": ["score"],
+                "examples": [
+                    {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+                ],
+            },
+        }
+
+        component = LLMEvaluator.from_dict(data)
+        assert isinstance(component._chat_generator, OpenAIChatGenerator)
+        assert component._chat_generator.generation_kwargs == {"response_format": {"type": "json_object"}, "seed": 42}
+        assert component.instructions == "test-instruction"
+        assert component.inputs == [("predicted_answers", list[str])]
+        assert component.outputs == ["score"]
+        assert component.examples == [
+            {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+        ]
+
+    def test_pipeline_serde(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        pipeline = Pipeline()
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("questions", list[str]), ("predicted_answers", list[list[str]])],
+            outputs=["score"],
+            examples=[
+                {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+            ],
+        )
+        pipeline.add_component("evaluator", component)
+        serialized_pipeline = pipeline.dumps()
+        deserialized_pipeline = Pipeline.loads(serialized_pipeline)
+        assert deserialized_pipeline == pipeline
+
+    def test_run_with_different_lengths(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("questions", list[str]), ("predicted_answers", list[list[str]])],
+            outputs=["score"],
+            examples=[
+                {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+            ],
+        )
+
+        def chat_generator_run(self, *args, **kwargs):
+            return {"replies": [ChatMessage.from_assistant('{"score": 0.5}')]}
+
+        monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
+
+        with pytest.raises(ValueError):
+            component.run(questions=["What is the capital of Germany?"], predicted_answers=[["Berlin"], ["Paris"]])
+
+        with pytest.raises(ValueError):
+            component.run(
+                questions=["What is the capital of Germany?", "What is the capital of France?"],
+                predicted_answers=[["Berlin"]],
+            )
+
+    def test_run_returns_parsed_result(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("questions", list[str]), ("predicted_answers", list[list[str]])],
+            outputs=["score"],
+            examples=[
+                {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+            ],
+        )
+
+        def chat_generator_run(self, *args, **kwargs):
+            return {"replies": [ChatMessage.from_assistant('{"score": 0.5}')]}
+
+        monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
+
+        results = component.run(questions=["What is the capital of Germany?"], predicted_answers=["Berlin"])
+        assert results == {"results": [{"score": 0.5}], "meta": None}
+
+    def test_prepare_template(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("predicted_answers", list[str])],
+            outputs=["score"],
+            examples=[
+                {"inputs": {"predicted_answers": "Damn, this is straight outta hell!!!"}, "outputs": {"score": 1}},
+                {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}},
+            ],
+        )
+        template = component.prepare_template()
+        assert (
+            template
+            == "Instructions:\ntest-instruction\n\nGenerate the response in JSON format with the following keys:"
+            '\n["score"]\nConsider the instructions and the examples below to determine those values.\n\n'
+            'Examples:\nInputs:\n{"predicted_answers": "Damn, this is straight outta hell!!!"}\nOutputs:'
+            '\n{"score": 1}\nInputs:\n{"predicted_answers": "Football is the most popular sport."}\nOutputs:'
+            '\n{"score": 0}\n\nInputs:\n{"predicted_answers": {{ predicted_answers }}}\nOutputs:\n'
+        )
+
+    def test_invalid_input_parameters(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("predicted_answers", list[str])],
+            outputs=["score"],
+            examples=[
+                {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+            ],
+        )
+        # None of the expected parameters are received
+        with pytest.raises(ValueError):
+            component.validate_input_parameters(
+                expected={"predicted_answers": list[str]}, received={"questions": list[str]}
+            )
+
+        # Only one but not all the expected parameters are received
+        with pytest.raises(ValueError):
+            component.validate_input_parameters(
+                expected={"predicted_answers": list[str], "questions": list[str]}, received={"questions": list[str]}
+            )
+
+        # Received inputs are not lists
+        with pytest.raises(ValueError):
+            component.validate_input_parameters(expected={"questions": list[str]}, received={"questions": str})
+
+    def test_invalid_outputs(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("predicted_answers", list[str])],
+            outputs=["score"],
+            examples=[
+                {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+            ],
+        )
+
+        def chat_generator_run(self, *args, **kwargs):
+            return {"replies": [ChatMessage.from_assistant('{"score": 1.0}')]}
+
+        monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
+
+        # Test missing key "another_expected_output"
+        component.outputs = ["score", "another_expected_output"]
+        with pytest.raises(ValueError, match="Missing expected keys"):
+            component.run(predicted_answers=["answer"])
+
+        # Test wrong key
+        def chat_generator_run_wrong_key(self, *args, **kwargs):
+            return {"replies": [ChatMessage.from_assistant('{"wrong_name": 1.0}')]}
+
+        monkeypatch.setattr(
+            "haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run_wrong_key
+        )
+
+        component.outputs = ["score"]
+        with pytest.raises(ValueError, match="Missing expected keys"):
+            component.run(predicted_answers=["answer"])
+
+    def test_output_invalid_json_raise_on_failure_false(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("predicted_answers", list[str])],
+            outputs=["score"],
+            examples=[
+                {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+            ],
+            raise_on_failure=False,
+        )
+
+        def chat_generator_run(self, *args, **kwargs):
+            return {"replies": [ChatMessage.from_assistant("some_invalid_json_output")]}
+
+        monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
+
+        result = component.run(predicted_answers=["answer"])
+        assert result["results"] == [None]
+
+    def test_output_invalid_json_raise_on_failure_true(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("predicted_answers", list[str])],
+            outputs=["score"],
+            examples=[
+                {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+            ],
+            raise_on_failure=True,
+        )
+
+        def chat_generator_run(self, *args, **kwargs):
+            return {"replies": [ChatMessage.from_assistant("some_invalid_json_output")]}
+
+        monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
+
+        with pytest.raises(
+            ValueError
+        ):  # json_utils/LLMEvaluator might raise JSONDecodeError which inherits from ValueError or wrapped
+            component.run(predicted_answers=["answer"])
+
+
+class TestLLMEvaluatorAsync:
+    @pytest.mark.asyncio
+    async def test_run_async_returns_parsed_result(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("questions", list[str]), ("predicted_answers", list[str])],
+            outputs=["score"],
+            examples=[
+                {
+                    "inputs": {
+                        "questions": "What is the value of any non-zero number raised to the power of zero?",
+                        "predicted_answers": "Zero",
+                    },
+                    "outputs": {"score": 0},
+                }
+            ],
+        )
+
+        async def chat_generator_run_async(self, *args, **kwargs):
+            return {"replies": [ChatMessage.from_assistant('{"score": 1}')]}
+
+        monkeypatch.setattr(
+            "haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run_async", chat_generator_run_async
+        )
+
+        results = await component.run_async(
+            questions=["What is the perimeter of a circle called?"], predicted_answers=["Circumference"]
+        )
+        assert results == {"results": [{"score": 1}], "meta": None}
+
+    @pytest.mark.asyncio
+    async def test_run_async_fallback_to_thread_with_sync_generator(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+
+        class SyncOnlyGenerator:
+            def run(self, messages):
+                return {"replies": [ChatMessage.from_assistant('{"score": 0}')]}
+
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("questions", list[str]), ("predicted_answers", list[str])],
+            outputs=["score"],
+            examples=[
+                {
+                    "inputs": {
+                        "questions": "What is the top sport?",
+                        "predicted_answers": "Football is the most popular sport.",
+                    },
+                    "outputs": {"score": 1},
+                }
+            ],
+            chat_generator=SyncOnlyGenerator(),
+        )
+
+        results = await component.run_async(questions=["question"], predicted_answers=["answer"])
+        assert results == {"results": [{"score": 0}], "meta": None}
+
+    @pytest.mark.asyncio
+    async def test_run_async_raise_on_failure_false(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("questions", list[str]), ("predicted_answers", list[str])],
+            outputs=["score"],
+            examples=[
+                {
+                    "inputs": {
+                        "questions": "What is the value of any non-zero number raised to the power of zero?",
+                        "predicted_answers": "One",
+                    },
+                    "outputs": {"score": 1},
+                }
+            ],
+            raise_on_failure=False,
+        )
+
+        async def chat_generator_run_async(self, *args, **kwargs):
+            raise Exception("API error")
+
+        monkeypatch.setattr(
+            "haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run_async", chat_generator_run_async
+        )
+
+        result = await component.run_async(questions=["question"], predicted_answers=["answer"])
+        assert result["results"] == [None]
+
+    @pytest.mark.asyncio
+    async def test_run_async_raise_on_failure_true(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("questions", list[str]), ("predicted_answers", list[str])],
+            outputs=["score"],
+            examples=[
+                {
+                    "inputs": {
+                        "questions": "What is the smallest unit of data in a computer?",
+                        "predicted_answers": "Bit",
+                    },
+                    "outputs": {"score": 1},
+                }
+            ],
+        )
+
+        async def chat_generator_run_async(self, *args, **kwargs):
+            raise Exception("API error")
+
+        monkeypatch.setattr(
+            "haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run_async", chat_generator_run_async
+        )
+
+        with pytest.raises(ValueError):
+            await component.run_async(questions=["question"], predicted_answers=["answer"])
+
+
+class TestComponentLifecycle:
+    @staticmethod
+    def _make_evaluator(chat_generator):
+        return LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("predicted_answers", list[str])],
+            outputs=["score"],
+            examples=[
+                {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+            ],
+            chat_generator=chat_generator,
+        )
+
+    def test_warm_up_delegates_to_chat_generator(self):
+        chat_generator = Mock(spec=["run", "warm_up"])
+        evaluator = self._make_evaluator(chat_generator)
+        evaluator.warm_up()
+        chat_generator.warm_up.assert_called_once()
+
+    async def test_warm_up_async_delegates_to_chat_generator(self):
+        chat_generator = Mock(spec=["run", "warm_up_async"])
+        chat_generator.warm_up_async = AsyncMock()
+        evaluator = self._make_evaluator(chat_generator)
+        await evaluator.warm_up_async()
+        chat_generator.warm_up_async.assert_awaited_once()
+
+    async def test_warm_up_async_falls_back_to_sync_warm_up(self):
+        chat_generator = Mock(spec=["run", "warm_up"])
+        evaluator = self._make_evaluator(chat_generator)
+        await evaluator.warm_up_async()
+        chat_generator.warm_up.assert_called_once()
+
+    def test_close_delegates_to_chat_generator(self):
+        chat_generator = Mock(spec=["run", "close"])
+        evaluator = self._make_evaluator(chat_generator)
+        evaluator.close()
+        chat_generator.close.assert_called_once()
+
+    async def test_close_async_delegates_to_chat_generator(self):
+        chat_generator = Mock(spec=["run", "close_async"])
+        chat_generator.close_async = AsyncMock()
+        evaluator = self._make_evaluator(chat_generator)
+        await evaluator.close_async()
+        chat_generator.close_async.assert_awaited_once()
+
+    async def test_close_async_falls_back_to_sync_close(self):
+        chat_generator = Mock(spec=["run", "close"])
+        evaluator = self._make_evaluator(chat_generator)
+        await evaluator.close_async()
+        chat_generator.close.assert_called_once()
+
+    async def test_lifecycle_is_safe_when_chat_generator_lacks_methods(self):
+        chat_generator = Mock(spec=["run"])
+        evaluator = self._make_evaluator(chat_generator)
+        evaluator.warm_up()
+        await evaluator.warm_up_async()
+        evaluator.close()
+        await evaluator.close_async()

test/components/evaluators/test_sas_evaluator.py

@@ -0,0 +1,128 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+from haystack.components.evaluators.sas_evaluator import SASEvaluator
+from haystack.utils.device import ComponentDevice
+
+
+class TestSASEvaluator:
+    def test_init_default(self, monkeypatch):
+        monkeypatch.setenv("HF_API_TOKEN", "fake-token")
+        evaluator = SASEvaluator()
+
+        assert evaluator._model == "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"
+        assert evaluator._batch_size == 32
+        assert evaluator._device is None
+        assert evaluator._token.resolve_value() == "fake-token"
+
+    def test_to_dict(self, monkeypatch):
+        monkeypatch.setenv("HF_API_TOKEN", "fake-token")
+
+        evaluator = SASEvaluator(device=ComponentDevice.from_str("cuda:0"))
+
+        expected_dict = {
+            "type": "haystack.components.evaluators.sas_evaluator.SASEvaluator",
+            "init_parameters": {
+                "model": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",
+                "batch_size": 32,
+                "device": {"type": "single", "device": "cuda:0"},
+                "token": {"type": "env_var", "env_vars": ["HF_API_TOKEN", "HF_TOKEN"], "strict": False},
+            },
+        }
+        assert evaluator.to_dict() == expected_dict
+
+    def test_from_dict(self, monkeypatch):
+        monkeypatch.setenv("HF_API_TOKEN", "fake-token")
+        evaluator = SASEvaluator.from_dict(
+            {
+                "type": "haystack.components.evaluators.sas_evaluator.SASEvaluator",
+                "init_parameters": {
+                    "model": "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",
+                    "batch_size": 32,
+                    "device": {"type": "single", "device": "cuda:0"},
+                    "token": {"type": "env_var", "env_vars": ["HF_API_TOKEN", "HF_TOKEN"], "strict": False},
+                },
+            }
+        )
+
+        assert evaluator._model == "sentence-transformers/paraphrase-multilingual-mpnet-base-v2"
+        assert evaluator._batch_size == 32
+        assert evaluator._device.to_torch_str() == "cuda:0"
+        assert evaluator._token.resolve_value() == "fake-token"
+
+    def test_run_with_empty_inputs(self):
+        evaluator = SASEvaluator()
+        result = evaluator.run(ground_truth_answers=[], predicted_answers=[])
+        assert len(result) == 2
+        assert result["score"] == 0.0
+        assert result["individual_scores"] == [0.0]
+
+    def test_run_with_different_lengths(self):
+        evaluator = SASEvaluator()
+        ground_truths = [
+            "A construction budget of US $2.3 billion",
+            "The Eiffel Tower, completed in 1889, symbolizes Paris's cultural magnificence.",
+        ]
+        predictions = [
+            "A construction budget of US $2.3 billion",
+            "The Eiffel Tower, completed in 1889, symbolizes Paris's cultural magnificence.",
+            "The Meiji Restoration in 1868 transformed Japan into a modernized world power.",
+        ]
+        with pytest.raises(ValueError):
+            evaluator.run(ground_truth_answers=ground_truths, predicted_answers=predictions)
+
+    def test_run_with_none_in_predictions(self):
+        evaluator = SASEvaluator()
+        ground_truths = [
+            "A construction budget of US $2.3 billion",
+            "The Eiffel Tower, completed in 1889, symbolizes Paris's cultural magnificence.",
+            "The Meiji Restoration in 1868 transformed Japan into a modernized world power.",
+        ]
+        predictions = [
+            "A construction budget of US $2.3 billion",
+            None,
+            "The Meiji Restoration in 1868 transformed Japan into a modernized world power.",
+        ]
+        with pytest.raises(ValueError):
+            evaluator.run(ground_truth_answers=ground_truths, predicted_answers=predictions)
+
+    @pytest.mark.integration
+    @pytest.mark.slow
+    def test_run_with_bi_encoder_model(self, del_hf_env_vars):
+        evaluator = SASEvaluator("sentence-transformers-testing/stsb-bert-tiny-safetensors")
+        ground_truths = [
+            "US $2.3 billion",
+            "Paris's cultural magnificence is symbolized by the Eiffel Tower",
+            "Japan was transformed into a modernized world power after the Meiji Restoration.",
+        ]
+        predictions = [
+            "A construction budget of US $2.3 billion",
+            "The Eiffel Tower, completed in 1889, symbolizes Paris's cultural magnificence.",
+            "The Meiji Restoration in 1868 transformed Japan into a modernized world power.",
+        ]
+        result = evaluator.run(ground_truth_answers=ground_truths, predicted_answers=predictions)
+        assert len(result) == 2
+        assert result["score"] == pytest.approx(0.912335)
+        assert result["individual_scores"] == pytest.approx([0.855047, 0.907907, 0.974050], abs=1e-5)
+
+    @pytest.mark.integration
+    @pytest.mark.slow
+    def test_run_with_cross_encoder_model(self, del_hf_env_vars):
+        evaluator = SASEvaluator(model="cross-encoder-testing/reranker-bert-tiny-gooaq-bce")
+        ground_truths = [
+            "A construction budget of US $2.3 billion",
+            "The Eiffel Tower, completed in 1889, symbolizes Paris's cultural magnificence.",
+            "The Meiji Restoration in 1868 transformed Japan into a modernized world power.",
+        ]
+        predictions = [
+            "A construction budget of US $2.3 billion",
+            "The Eiffel Tower, completed in 1889, symbolizes Paris's cultural magnificence.",
+            "The Meiji Restoration in 1868 transformed Japan into a modernized world power.",
+        ]
+        result = evaluator.run(ground_truth_answers=ground_truths, predicted_answers=predictions)
+        assert len(result) == 2
+        assert result["score"] == pytest.approx(0.938108, abs=1e-5)
+        assert result["individual_scores"] == pytest.approx([0.930112, 0.9431504, 0.9410622], abs=1e-5)