chore: import upstream snapshot with attribution

haystack/components/evaluators/__init__.py

@@ -0,0 +1,34 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {
+    "answer_exact_match": ["AnswerExactMatchEvaluator"],
+    "context_relevance": ["ContextRelevanceEvaluator"],
+    "document_map": ["DocumentMAPEvaluator"],
+    "document_mrr": ["DocumentMRREvaluator"],
+    "document_ndcg": ["DocumentNDCGEvaluator"],
+    "document_recall": ["DocumentRecallEvaluator"],
+    "faithfulness": ["FaithfulnessEvaluator"],
+    "llm_evaluator": ["LLMEvaluator"],
+    "sas_evaluator": ["SASEvaluator"],
+}
+
+if TYPE_CHECKING:
+    from .answer_exact_match import AnswerExactMatchEvaluator as AnswerExactMatchEvaluator
+    from .context_relevance import ContextRelevanceEvaluator as ContextRelevanceEvaluator
+    from .document_map import DocumentMAPEvaluator as DocumentMAPEvaluator
+    from .document_mrr import DocumentMRREvaluator as DocumentMRREvaluator
+    from .document_ndcg import DocumentNDCGEvaluator as DocumentNDCGEvaluator
+    from .document_recall import DocumentRecallEvaluator as DocumentRecallEvaluator
+    from .faithfulness import FaithfulnessEvaluator as FaithfulnessEvaluator
+    from .llm_evaluator import LLMEvaluator as LLMEvaluator
+    from .sas_evaluator import SASEvaluator as SASEvaluator
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/evaluators/answer_exact_match.py

@@ -0,0 +1,69 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from haystack.core.component import component
+
+
+@component
+class AnswerExactMatchEvaluator:
+    """
+    An answer exact match evaluator class.
+
+    The evaluator that checks if the predicted answers matches any of the ground truth answers exactly.
+    The result is a number from 0.0 to 1.0, it represents the proportion of predicted answers
+    that matched one of the ground truth answers.
+    There can be multiple ground truth answers and multiple predicted answers as input.
+
+
+    Usage example:
+    ```python
+    from haystack.components.evaluators import AnswerExactMatchEvaluator
+
+    evaluator = AnswerExactMatchEvaluator()
+    result = evaluator.run(
+        ground_truth_answers=["Berlin", "Paris"],
+        predicted_answers=["Berlin", "Lyon"],
+    )
+
+    print(result["individual_scores"])
+    # [1, 0]
+    print(result["score"])
+    # 0.5
+    ```
+    """
+
+    @component.output_types(individual_scores=list[int], score=float)
+    def run(self, ground_truth_answers: list[str], predicted_answers: list[str]) -> dict[str, Any]:
+        """
+        Run the AnswerExactMatchEvaluator on the given inputs.
+
+        The `ground_truth_answers` and `retrieved_answers` must have the same length.
+
+        :param ground_truth_answers:
+            A list of expected answers.
+        :param predicted_answers:
+            A list of predicted answers.
+        :returns:
+            A dictionary with the following outputs:
+            - `individual_scores` - A list of 0s and 1s, where 1 means that the predicted answer matched one of the
+                ground truth.
+            - `score` - A number from 0.0 to 1.0 that represents the proportion of questions where any predicted
+                         answer matched one of the ground truth answers.
+        """
+        if not len(ground_truth_answers) == len(predicted_answers):
+            raise ValueError("The length of ground_truth_answers and predicted_answers must be the same.")
+
+        matches = []
+        for truth, extracted in zip(ground_truth_answers, predicted_answers, strict=True):
+            if truth == extracted:
+                matches.append(1)
+            else:
+                matches.append(0)
+
+        # The proportion of questions where any predicted answer matched one of the ground truth answers
+        average = sum(matches) / len(predicted_answers)
+
+        return {"individual_scores": matches, "score": average}

haystack/components/evaluators/context_relevance.py

@@ -0,0 +1,257 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import math
+from statistics import mean
+from typing import Any
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.components.evaluators.llm_evaluator import LLMEvaluator
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.core.serialization import component_to_dict
+from haystack.utils import deserialize_chatgenerator_inplace
+
+logger = logging.getLogger(__name__)
+
+# Private global variable for default examples to include in the prompt if the user does not provide any examples
+_DEFAULT_EXAMPLES = [
+    {
+        "inputs": {
+            "questions": "What is the capital of Germany?",
+            "contexts": ["Berlin is the capital of Germany. Berlin and was founded in 1244."],
+        },
+        "outputs": {"relevant_statements": ["Berlin is the capital of Germany."]},
+    },
+    {
+        "inputs": {
+            "questions": "What is the capital of France?",
+            "contexts": [
+                "Berlin is the capital of Germany and was founded in 1244.",
+                "Europe is a continent with 44 countries.",
+                "Madrid is the capital of Spain.",
+            ],
+        },
+        "outputs": {"relevant_statements": []},
+    },
+    {
+        "inputs": {"questions": "What is the capital of Italy?", "contexts": ["Rome is the capital of Italy."]},
+        "outputs": {"relevant_statements": ["Rome is the capital of Italy."]},
+    },
+]
+
+
+@component
+class ContextRelevanceEvaluator(LLMEvaluator):
+    """
+    Evaluator that checks if a provided context is relevant to the question.
+
+    An LLM breaks up a context into multiple statements and checks whether each statement
+    is relevant for answering a question.
+    The score for each context is either binary score of 1 or 0, where 1 indicates that the context is relevant
+    to the question and 0 indicates that the context is not relevant.
+    The evaluator also provides the relevant statements from the context and an average score over all the provided
+    input questions contexts pairs.
+
+    Usage example:
+    ```python
+    from haystack.components.evaluators import ContextRelevanceEvaluator
+
+    questions = ["Who created the Python language?", "Why does Java needs a JVM?", "Is C++ better than Python?"]
+    contexts = [
+        [(
+            "Python, created by Guido van Rossum in the late 1980s, is a high-level general-purpose programming "
+            "language. Its design philosophy emphasizes code readability, and its language constructs aim to help "
+            "programmers write clear, logical code for both small and large-scale software projects."
+        )],
+        [(
+            "Java is a high-level, class-based, object-oriented programming language that is designed to have as few "
+            "implementation dependencies as possible. The JVM has two primary functions: to allow Java programs to run"
+            "on any device or operating system (known as the 'write once, run anywhere' principle), and to manage and"
+            "optimize program memory."
+        )],
+        [(
+            "C++ is a general-purpose programming language created by Bjarne Stroustrup as an extension of the C "
+            "programming language."
+        )],
+    ]
+
+    evaluator = ContextRelevanceEvaluator()
+    result = evaluator.run(questions=questions, contexts=contexts)
+    print(result["score"])
+    # 0.67
+    print(result["individual_scores"])
+    # [1,1,0]
+    print(result["results"])
+    # [{
+    #   'relevant_statements': ['Python, created by Guido van Rossum in the late 1980s.'],
+    #    'score': 1.0
+    #  },
+    #  {
+    #   'relevant_statements': ['The JVM has two primary functions: to allow Java programs to run on any device or
+    #                           operating system (known as the "write once, run anywhere" principle), and to manage and
+    #                           optimize program memory'],
+    #   'score': 1.0
+    #  },
+    #  {
+    #   'relevant_statements': [],
+    #   'score': 0.0
+    #  }]
+    ```
+    """
+
+    def __init__(
+        self,
+        examples: list[dict[str, Any]] | None = None,
+        progress_bar: bool = True,
+        raise_on_failure: bool = True,
+        chat_generator: ChatGenerator | None = None,
+    ) -> None:
+        """
+        Creates an instance of ContextRelevanceEvaluator.
+
+        If no LLM is specified using the `chat_generator` parameter, the component will use OpenAI in JSON mode.
+
+        :param examples:
+            Optional few-shot examples conforming to the expected input and output format of ContextRelevanceEvaluator.
+            Default examples will be used if none are provided.
+            Each example must be a dictionary with keys "inputs" and "outputs".
+            "inputs" must be a dictionary with keys "questions" and "contexts".
+            "outputs" must be a dictionary with "relevant_statements".
+            Expected format:
+            ```python
+            [{
+                "inputs": {
+                    "questions": "What is the capital of Italy?", "contexts": ["Rome is the capital of Italy."],
+                },
+                "outputs": {
+                    "relevant_statements": ["Rome is the capital of Italy."],
+                },
+            }]
+            ```
+        :param progress_bar:
+            Whether to show a progress bar during the evaluation.
+        :param raise_on_failure:
+            Whether to raise an exception if the API call fails.
+        :param chat_generator:
+            a ChatGenerator instance which represents the LLM.
+            In order for the component to work, the LLM should be configured to return a JSON object. For example,
+            when using the OpenAIChatGenerator, you should pass `{"response_format": {"type": "json_object"}}` in the
+            `generation_kwargs`.
+        """
+
+        self.instructions = (
+            "Please extract only sentences from the provided context which are absolutely relevant and "
+            "required to answer the following question. If no relevant sentences are found, or if you "
+            "believe the question cannot be answered from the given context, return an empty list, example: []"
+        )
+        self.inputs = [("questions", list[str]), ("contexts", list[list[str]])]
+        self.outputs = ["relevant_statements"]
+        self.examples = examples or _DEFAULT_EXAMPLES
+
+        super(ContextRelevanceEvaluator, self).__init__(  # noqa: UP008
+            instructions=self.instructions,
+            inputs=self.inputs,
+            outputs=self.outputs,
+            examples=self.examples,
+            chat_generator=chat_generator,
+            raise_on_failure=raise_on_failure,
+            progress_bar=progress_bar,
+        )
+
+    @component.output_types(score=float, results=list[dict[str, Any]])
+    def run(self, **inputs: Any) -> dict[str, Any]:
+        """
+        Run the LLM evaluator.
+
+        :param questions:
+            A list of questions.
+        :param contexts:
+            A list of lists of contexts. Each list of contexts corresponds to one question.
+        :returns:
+            A dictionary with the following outputs:
+                - `score`: Mean context relevance score over all the provided input questions.
+                - `results`: A list of dictionaries with `relevant_statements` and `score` for each input context.
+        """
+        result = super(ContextRelevanceEvaluator, self).run(**inputs)  # noqa: UP008
+        # Post-process the raw results to calculate relevance metrics and scores
+        return self._postprocess_results(result)
+
+    @component.output_types(score=float, results=list[dict[str, Any]])
+    async def run_async(self, **inputs: Any) -> dict[str, Any]:
+        """
+        Run the LLM evaluator asynchronously.
+
+        :param questions:
+            A list of questions.
+        :param contexts:
+            A list of lists of contexts. Each list of contexts corresponds to one question.
+        :returns:
+            A dictionary with the following outputs:
+                - `score`: Mean context relevance score over all the provided input questions.
+                - `results`: A list of dictionaries with `relevant_statements` and `score` for each input context.
+        """
+        result = await super(ContextRelevanceEvaluator, self).run_async(**inputs)  # noqa: UP008
+        # Post-process the raw results to calculate relevance metrics and scores
+        return self._postprocess_results(result)
+
+    def _postprocess_results(self, result: dict[str, Any]) -> dict[str, Any]:
+        """
+        Post-processes raw LLM evaluator outputs to compute context relevance scores.
+
+        Calculates binary scores based on whether relevant statements were found,
+        averages the scores across all successful queries, and updates the result payload.
+
+        :param result:
+            The raw evaluation dictionary from the base LLM evaluator.
+        :returns:
+            The updated dictionary containing final scores and tracking metrics.
+        """
+        for idx, res in enumerate(result["results"]):
+            if res is None:
+                result["results"][idx] = {"relevant_statements": [], "score": float("nan")}
+                continue
+            if len(res["relevant_statements"]) > 0:
+                res["score"] = 1
+            else:
+                res["score"] = 0
+
+        # calculate average context relevance score over all queries
+        scores = [res["score"] for res in result["results"]]
+        valid_scores = [s for s in scores if not math.isnan(s)]
+        skipped = len(scores) - len(valid_scores)
+        if skipped:
+            logger.warning("{skipped} query(s) failed and were excluded from the score.", skipped=skipped)
+        result["score"] = mean(valid_scores) if valid_scores else float("nan")
+        result["individual_scores"] = scores  # useful for the EvaluationRunResult
+
+        return result
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            A dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            chat_generator=component_to_dict(obj=self._chat_generator, name="chat_generator"),
+            examples=self.examples,
+            progress_bar=self.progress_bar,
+            raise_on_failure=self.raise_on_failure,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "ContextRelevanceEvaluator":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data:
+            The dictionary representation of this component.
+        :returns:
+            The deserialized component instance.
+        """
+        if data["init_parameters"].get("chat_generator"):
+            deserialize_chatgenerator_inplace(data["init_parameters"], key="chat_generator")
+        return default_from_dict(cls, data)

haystack/components/evaluators/document_map.py

@@ -0,0 +1,136 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from haystack import Document, component, default_to_dict
+
+
+@component
+class DocumentMAPEvaluator:
+    """
+    A Mean Average Precision (MAP) evaluator for documents.
+
+    Evaluator that calculates the mean average precision of the retrieved documents, a metric
+    that measures how high retrieved documents are ranked.
+    Each question can have multiple ground truth documents and multiple retrieved documents.
+
+    `DocumentMAPEvaluator` doesn't normalize its inputs, the `DocumentCleaner` component
+    should be used to clean and normalize the documents before passing them to this evaluator.
+
+    Usage example:
+    ```python
+    from haystack import Document
+    from haystack.components.evaluators import DocumentMAPEvaluator
+
+    evaluator = DocumentMAPEvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="9th")],
+        ],
+        retrieved_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="10th century"), Document(content="9th")],
+        ],
+    )
+
+    print(result["individual_scores"])
+    # [1.0, 0.8333333333333333]
+    print(result["score"])
+    # 0.9166666666666666
+    ```
+    """
+
+    def __init__(self, document_comparison_field: str = "content") -> None:
+        """
+        Create a DocumentMAPEvaluator component.
+
+        :param document_comparison_field:
+            The Document field to use for comparison. Possible options:
+            - `"content"`: uses `doc.content`
+            - `"id"`: uses `doc.id`
+            - A `meta.` prefix followed by a key name: uses `doc.meta["<key>"]`
+              (e.g. `"meta.file_id"`, `"meta.page_number"`)
+              Nested keys are supported (e.g. `"meta.source.url"`).
+        """
+        self.document_comparison_field = document_comparison_field
+
+    def _get_comparison_value(self, doc: Document) -> Any:
+        """
+        Extract the comparison value from a document based on the configured field.
+        """
+        if self.document_comparison_field == "content":
+            return doc.content
+        if self.document_comparison_field == "id":
+            return doc.id
+        if self.document_comparison_field.startswith("meta."):
+            parts = self.document_comparison_field[5:].split(".")
+            value = doc.meta
+            for part in parts:
+                if not isinstance(value, dict) or part not in value:
+                    return None
+                value = value[part]
+            return value
+        msg = (
+            f"Unsupported document_comparison_field: '{self.document_comparison_field}'. "
+            "Use 'content', 'id', or 'meta.<key>'."
+        )
+        raise ValueError(msg)
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(self, document_comparison_field=self.document_comparison_field)
+
+    # Refer to https://www.pinecone.io/learn/offline-evaluation/ for the algorithm.
+    @component.output_types(score=float, individual_scores=list[float])
+    def run(
+        self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
+    ) -> dict[str, Any]:
+        """
+        Run the DocumentMAPEvaluator on the given inputs.
+
+        All lists must have the same length.
+
+        :param ground_truth_documents:
+            A list of expected documents for each question.
+        :param retrieved_documents:
+            A list of retrieved documents for each question.
+        :returns:
+            A dictionary with the following outputs:
+            - `score` - The average of calculated scores.
+            - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents how high retrieved documents
+                are ranked.
+        """
+        if len(ground_truth_documents) != len(retrieved_documents):
+            msg = "The length of ground_truth_documents and retrieved_documents must be the same."
+            raise ValueError(msg)
+
+        individual_scores = []
+
+        for ground_truth, retrieved in zip(ground_truth_documents, retrieved_documents, strict=True):
+            average_precision = 0.0
+            average_precision_numerator = 0.0
+            relevant_documents = 0
+
+            ground_truth_values = [val for doc in ground_truth if (val := self._get_comparison_value(doc)) is not None]
+            for rank, retrieved_document in enumerate(retrieved):
+                retrieved_value = self._get_comparison_value(retrieved_document)
+                if retrieved_value is None:
+                    continue
+
+                if retrieved_value in ground_truth_values:
+                    relevant_documents += 1
+                    average_precision_numerator += relevant_documents / (rank + 1)
+            if relevant_documents > 0:
+                average_precision = average_precision_numerator / relevant_documents
+            individual_scores.append(average_precision)
+
+        score = sum(individual_scores) / len(ground_truth_documents)
+        return {"score": score, "individual_scores": individual_scores}

haystack/components/evaluators/document_mrr.py

@@ -0,0 +1,130 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from haystack import Document, component, default_to_dict
+
+
+@component
+class DocumentMRREvaluator:
+    """
+    Evaluator that calculates the mean reciprocal rank of the retrieved documents.
+
+    MRR measures how high the first retrieved document is ranked.
+    Each question can have multiple ground truth documents and multiple retrieved documents.
+
+    `DocumentMRREvaluator` doesn't normalize its inputs, the `DocumentCleaner` component
+    should be used to clean and normalize the documents before passing them to this evaluator.
+
+    Usage example:
+    ```python
+    from haystack import Document
+    from haystack.components.evaluators import DocumentMRREvaluator
+
+    evaluator = DocumentMRREvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="9th")],
+        ],
+        retrieved_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="10th century"), Document(content="9th")],
+        ],
+    )
+    print(result["individual_scores"])
+    # [1.0, 1.0]
+    print(result["score"])
+    # 1.0
+    ```
+    """
+
+    def __init__(self, document_comparison_field: str = "content") -> None:
+        """
+        Create a DocumentMRREvaluator component.
+
+        :param document_comparison_field:
+            The Document field to use for comparison. Possible options:
+            - `"content"`: uses `doc.content`
+            - `"id"`: uses `doc.id`
+            - A `meta.` prefix followed by a key name: uses `doc.meta["<key>"]`
+              (e.g. `"meta.file_id"`, `"meta.page_number"`)
+              Nested keys are supported (e.g. `"meta.source.url"`).
+        """
+        self.document_comparison_field = document_comparison_field
+
+    def _get_comparison_value(self, doc: Document) -> Any:
+        """
+        Extract the comparison value from a document based on the configured field.
+        """
+        if self.document_comparison_field == "content":
+            return doc.content
+        if self.document_comparison_field == "id":
+            return doc.id
+        if self.document_comparison_field.startswith("meta."):
+            parts = self.document_comparison_field[5:].split(".")
+            value = doc.meta
+            for part in parts:
+                if not isinstance(value, dict) or part not in value:
+                    return None
+                value = value[part]
+            return value
+        msg = (
+            f"Unsupported document_comparison_field: '{self.document_comparison_field}'. "
+            "Use 'content', 'id', or 'meta.<key>'."
+        )
+        raise ValueError(msg)
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(self, document_comparison_field=self.document_comparison_field)
+
+    # Refer to https://www.pinecone.io/learn/offline-evaluation/ for the algorithm.
+    @component.output_types(score=float, individual_scores=list[float])
+    def run(
+        self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
+    ) -> dict[str, Any]:
+        """
+        Run the DocumentMRREvaluator on the given inputs.
+
+        `ground_truth_documents` and `retrieved_documents` must have the same length.
+
+        :param ground_truth_documents:
+            A list of expected documents for each question.
+        :param retrieved_documents:
+            A list of retrieved documents for each question.
+        :returns:
+            A dictionary with the following outputs:
+            - `score` - The average of calculated scores.
+            - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents how high the first retrieved
+                document is ranked.
+        """
+        if len(ground_truth_documents) != len(retrieved_documents):
+            msg = "The length of ground_truth_documents and retrieved_documents must be the same."
+            raise ValueError(msg)
+
+        individual_scores = []
+
+        for ground_truth, retrieved in zip(ground_truth_documents, retrieved_documents, strict=True):
+            reciprocal_rank = 0.0
+
+            ground_truth_values = [val for doc in ground_truth if (val := self._get_comparison_value(doc)) is not None]
+            for rank, retrieved_document in enumerate(retrieved):
+                retrieved_value = self._get_comparison_value(retrieved_document)
+                if retrieved_value is None:
+                    continue
+                if retrieved_value in ground_truth_values:
+                    reciprocal_rank = 1 / (rank + 1)
+                    break
+            individual_scores.append(reciprocal_rank)
+
+        score = sum(individual_scores) / len(ground_truth_documents)
+
+        return {"score": score, "individual_scores": individual_scores}

haystack/components/evaluators/document_ndcg.py

@@ -0,0 +1,193 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from math import log2
+from typing import Any
+
+from haystack import Document, component, default_to_dict
+
+
+@component
+class DocumentNDCGEvaluator:
+    """
+    Evaluator that calculates the normalized discounted cumulative gain (NDCG) of retrieved documents.
+
+    Each question can have multiple ground truth documents and multiple retrieved documents.
+    If the ground truth documents have relevance scores, the NDCG calculation uses these scores.
+    Otherwise, it assumes binary relevance of all ground truth documents.
+
+    Usage example:
+    ```python
+    from haystack import Document
+    from haystack.components.evaluators import DocumentNDCGEvaluator
+
+    evaluator = DocumentNDCGEvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[[Document(content="France", score=1.0), Document(content="Paris", score=0.5)]],
+        retrieved_documents=[[Document(content="France"), Document(content="Germany"), Document(content="Paris")]],
+    )
+    print(result["individual_scores"])
+    # [0.8869]
+    print(result["score"])
+    # 0.8869
+    ```
+    """
+
+    def __init__(self, document_comparison_field: str = "content") -> None:
+        """
+        Create a DocumentNDCGEvaluator component.
+
+        :param document_comparison_field:
+            The Document field to use for comparison. Possible options:
+            - `"content"`: uses `doc.content`
+            - `"id"`: uses `doc.id`
+            - A `meta.` prefix followed by a key name: uses `doc.meta["<key>"]`
+              (e.g. `"meta.file_id"`, `"meta.page_number"`)
+              Nested keys are supported (e.g. `"meta.source.url"`).
+        """
+        self.document_comparison_field = document_comparison_field
+
+    def _get_comparison_value(self, doc: Document) -> Any:
+        """
+        Extract the comparison value from a document based on the configured field.
+        """
+        if self.document_comparison_field == "content":
+            return doc.content
+        if self.document_comparison_field == "id":
+            return doc.id
+        if self.document_comparison_field.startswith("meta."):
+            parts = self.document_comparison_field[5:].split(".")
+            value = doc.meta
+            for part in parts:
+                if not isinstance(value, dict) or part not in value:
+                    return None
+                value = value[part]
+            return value
+        msg = (
+            f"Unsupported document_comparison_field: '{self.document_comparison_field}'. "
+            "Use 'content', 'id', or 'meta.<key>'."
+        )
+        raise ValueError(msg)
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(self, document_comparison_field=self.document_comparison_field)
+
+    @component.output_types(score=float, individual_scores=list[float])
+    def run(
+        self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
+    ) -> dict[str, Any]:
+        """
+        Run the DocumentNDCGEvaluator on the given inputs.
+
+        `ground_truth_documents` and `retrieved_documents` must have the same length.
+        The list items within `ground_truth_documents` and `retrieved_documents` can differ in length.
+
+        :param ground_truth_documents:
+            Lists of expected documents, one list per question. Binary relevance is used if documents have no scores.
+        :param retrieved_documents:
+            Lists of retrieved documents, one list per question.
+        :returns:
+            A dictionary with the following outputs:
+            - `score` - The average of calculated scores.
+            - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents the NDCG for each question.
+        """
+        self.validate_inputs(ground_truth_documents, retrieved_documents)
+
+        individual_scores = []
+
+        for gt_docs, ret_docs in zip(ground_truth_documents, retrieved_documents, strict=True):
+            dcg = self.calculate_dcg(gt_docs, ret_docs)
+            idcg = self.calculate_idcg(gt_docs)
+            ndcg = dcg / idcg if idcg > 0 else 0
+            individual_scores.append(ndcg)
+
+        score = sum(individual_scores) / len(ground_truth_documents)
+
+        return {"score": score, "individual_scores": individual_scores}
+
+    @staticmethod
+    def validate_inputs(gt_docs: list[list[Document]], ret_docs: list[list[Document]]) -> None:
+        """
+        Validate the input parameters.
+
+        :param gt_docs:
+            The ground_truth_documents to validate.
+        :param ret_docs:
+            The retrieved_documents to validate.
+
+        :raises ValueError:
+            If the ground_truth_documents or the retrieved_documents are an empty list.
+            If the length of ground_truth_documents and retrieved_documents differs.
+            If any list of documents in ground_truth_documents contains a mix of documents with and without a score.
+        """
+        if len(gt_docs) == 0 or len(ret_docs) == 0:
+            msg = "ground_truth_documents and retrieved_documents must be provided."
+            raise ValueError(msg)
+
+        if len(gt_docs) != len(ret_docs):
+            msg = "The length of ground_truth_documents and retrieved_documents must be the same."
+            raise ValueError(msg)
+
+        for docs in gt_docs:
+            if any(doc.score is not None for doc in docs) and any(doc.score is None for doc in docs):
+                msg = "Either none or all documents in each list of ground_truth_documents must have a score."
+                raise ValueError(msg)
+
+    def calculate_dcg(self, gt_docs: list[Document], ret_docs: list[Document]) -> float:
+        """
+        Calculate the discounted cumulative gain (DCG) of the retrieved documents.
+
+        :param gt_docs:
+            The ground truth documents.
+        :param ret_docs:
+            The retrieved documents.
+        :returns:
+            The discounted cumulative gain (DCG) of the retrieved
+            documents based on the ground truth documents.
+        """
+        dcg = 0.0
+        # Build lookup from comparison value -> relevance score, skipping documents
+        # whose comparison value cannot be determined (e.g. missing meta key)
+        relevant_value_to_score: dict[Any, float] = {}
+        for doc in gt_docs:
+            value = self._get_comparison_value(doc)
+            if value is not None:
+                relevant_value_to_score[value] = doc.score if doc.score is not None else 1
+
+        for i, doc in enumerate(ret_docs):
+            value = self._get_comparison_value(doc)
+            if value is not None and value in relevant_value_to_score:
+                dcg += relevant_value_to_score[value] / log2(i + 2)  # i + 2 because i is 0-indexed
+        return dcg
+
+    def calculate_idcg(self, gt_docs: list[Document]) -> float:
+        """
+        Calculate the ideal discounted cumulative gain (IDCG) of the ground truth documents.
+
+        Ground truth documents whose comparison value cannot be determined (e.g. missing meta key)
+        are excluded, since they can never be matched in `calculate_dcg` either. Including them here
+        would inflate the IDCG and make it impossible for NDCG to reach 1.0 for a perfect retrieval.
+
+        :param gt_docs:
+            The ground truth documents.
+        :returns:
+            The ideal discounted cumulative gain (IDCG) of the ground truth documents.
+        """
+        # Filter out documents that cannot be matched, consistent with calculate_dcg
+        matchable_docs = [doc for doc in gt_docs if self._get_comparison_value(doc) is not None]
+
+        idcg = 0.0
+        for i, doc in enumerate(
+            sorted(matchable_docs, key=lambda x: x.score if x.score is not None else 1, reverse=True)
+        ):
+            # If the document has a score, use it; otherwise, use 1 for binary relevance.
+            relevance = doc.score if doc.score is not None else 1
+            idcg += relevance / log2(i + 2)  # i + 2 because i is 0-indexed
+        return idcg

haystack/components/evaluators/document_recall.py

@@ -0,0 +1,181 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from enum import Enum
+from typing import Any
+
+from haystack import component, default_to_dict, logging
+from haystack.dataclasses import Document
+
+logger = logging.getLogger(__name__)
+
+
+class RecallMode(Enum):
+    """
+    Enum for the mode to use for calculating the recall score.
+    """
+
+    # Score is based on whether any document is retrieved.
+    SINGLE_HIT = "single_hit"
+    # Score is based on how many documents were retrieved.
+    MULTI_HIT = "multi_hit"
+
+    def __str__(self) -> str:
+        return self.value
+
+    @staticmethod
+    def from_str(string: str) -> "RecallMode":
+        """
+        Convert a string to a RecallMode enum.
+        """
+        enum_map = {e.value: e for e in RecallMode}
+        mode = enum_map.get(string)
+        if mode is None:
+            msg = f"Unknown recall mode '{string}'. Supported modes are: {list(enum_map.keys())}"
+            raise ValueError(msg)
+        return mode
+
+
+@component
+class DocumentRecallEvaluator:
+    """
+    Evaluator that calculates the Recall score for a list of documents.
+
+    Returns both a list of scores for each question and the average.
+    There can be multiple ground truth documents and multiple predicted documents as input.
+
+    Usage example:
+    ```python
+    from haystack import Document
+    from haystack.components.evaluators import DocumentRecallEvaluator
+
+    evaluator = DocumentRecallEvaluator()
+    result = evaluator.run(
+        ground_truth_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="9th")],
+        ],
+        retrieved_documents=[
+            [Document(content="France")],
+            [Document(content="9th century"), Document(content="10th century"), Document(content="9th")],
+        ],
+    )
+    print(result["individual_scores"])
+    # [1.0, 1.0]
+    print(result["score"])
+    # 1.0
+    ```
+    """
+
+    def __init__(
+        self, mode: str | RecallMode = RecallMode.SINGLE_HIT, document_comparison_field: str = "content"
+    ) -> None:
+        """
+        Create a DocumentRecallEvaluator component.
+
+        :param mode:
+            Mode to use for calculating the recall score.
+        :param document_comparison_field:
+            The Document field to use for comparison. Possible options:
+            - `"content"`: uses `doc.content`
+            - `"id"`: uses `doc.id`
+            - A `meta.` prefix followed by a key name: uses `doc.meta["<key>"]`
+              (e.g. `"meta.file_id"`, `"meta.page_number"`)
+              Nested keys are supported (e.g. `"meta.source.url"`).
+        """
+        if isinstance(mode, str):
+            mode = RecallMode.from_str(mode)
+
+        self.mode = mode
+        self.document_comparison_field = document_comparison_field
+
+    def _get_comparison_value(self, doc: Document) -> Any:
+        """
+        Extract the comparison value from a document based on the configured field.
+        """
+        if self.document_comparison_field == "content":
+            return doc.content
+        if self.document_comparison_field == "id":
+            return doc.id
+        if self.document_comparison_field.startswith("meta."):
+            parts = self.document_comparison_field[5:].split(".")
+            value = doc.meta
+            for part in parts:
+                if not isinstance(value, dict) or part not in value:
+                    return None
+                value = value[part]
+            return value
+        msg = (
+            f"Unsupported document_comparison_field: '{self.document_comparison_field}'. "
+            "Use 'content', 'id', or 'meta.<key>'."
+        )
+        raise ValueError(msg)
+
+    def _recall_single_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float:
+        unique_truths = {self._get_comparison_value(g) for g in ground_truth_documents}
+        unique_retrievals = {self._get_comparison_value(p) for p in retrieved_documents}
+        retrieved_ground_truths = unique_truths.intersection(unique_retrievals)
+
+        return float(len(retrieved_ground_truths) > 0)
+
+    def _recall_multi_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float:
+        unique_truths = {self._get_comparison_value(g) for g in ground_truth_documents}
+        unique_retrievals = {self._get_comparison_value(p) for p in retrieved_documents}
+        retrieved_ground_truths = unique_truths.intersection(unique_retrievals)
+
+        if not unique_truths or unique_truths <= {"", None}:
+            logger.warning(
+                "There are no ground truth documents or none of them contain a valid comparison value. "
+                "Score will be set to 0."
+            )
+            return 0.0
+
+        if not unique_retrievals or unique_retrievals <= {"", None}:
+            logger.warning(
+                "There are no retrieved documents or none of them contain a valid comparison value. "
+                "Score will be set to 0."
+            )
+            return 0.0
+
+        return len(retrieved_ground_truths) / len(unique_truths)
+
+    @component.output_types(score=float, individual_scores=list[float])
+    def run(
+        self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
+    ) -> dict[str, Any]:
+        """
+        Run the DocumentRecallEvaluator on the given inputs.
+
+        `ground_truth_documents` and `retrieved_documents` must have the same length.
+
+        :param ground_truth_documents:
+            A list of expected documents for each question.
+        :param retrieved_documents:
+            A list of retrieved documents for each question.
+        A dictionary with the following outputs:
+            - `score` - The average of calculated scores.
+            - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents the proportion of matching
+                documents retrieved. If the mode is `single_hit`, the individual scores are 0 or 1.
+        """
+        if len(ground_truth_documents) != len(retrieved_documents):
+            msg = "The length of ground_truth_documents and retrieved_documents must be the same."
+            raise ValueError(msg)
+
+        if self.mode == RecallMode.SINGLE_HIT:
+            mode_function = self._recall_single_hit
+        elif self.mode == RecallMode.MULTI_HIT:
+            mode_function = self._recall_multi_hit
+
+        scores = [mode_function(gt, ret) for gt, ret in zip(ground_truth_documents, retrieved_documents, strict=True)]
+
+        return {"score": sum(scores) / len(retrieved_documents), "individual_scores": scores}
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(self, mode=str(self.mode), document_comparison_field=self.document_comparison_field)

haystack/components/evaluators/faithfulness.py

@@ -0,0 +1,255 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import math
+from typing import Any
+
+from numpy import mean as np_mean
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.components.evaluators.llm_evaluator import LLMEvaluator
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.core.serialization import component_to_dict
+from haystack.utils import deserialize_chatgenerator_inplace
+
+logger = logging.getLogger(__name__)
+
+# Default examples to include in the prompt if the user does not provide any examples
+_DEFAULT_EXAMPLES = [
+    {
+        "inputs": {
+            "questions": "What is the capital of Germany and when was it founded?",
+            "contexts": ["Berlin is the capital of Germany and was founded in 1244."],
+            "predicted_answers": "The capital of Germany, Berlin, was founded in the 13th century.",
+        },
+        "outputs": {
+            "statements": ["Berlin is the capital of Germany.", "Berlin was founded in 1244."],
+            "statement_scores": [1, 1],
+        },
+    },
+    {
+        "inputs": {
+            "questions": "What is the capital of France?",
+            "contexts": ["Berlin is the capital of Germany."],
+            "predicted_answers": "Paris",
+        },
+        "outputs": {"statements": ["Paris is the capital of France."], "statement_scores": [0]},
+    },
+    {
+        "inputs": {
+            "questions": "What is the capital of Italy?",
+            "contexts": ["Rome is the capital of Italy."],
+            "predicted_answers": "Rome is the capital of Italy with more than 4 million inhabitants.",
+        },
+        "outputs": {
+            "statements": ["Rome is the capital of Italy.", "Rome has more than 4 million inhabitants."],
+            "statement_scores": [1, 0],
+        },
+    },
+]
+
+
+@component
+class FaithfulnessEvaluator(LLMEvaluator):
+    """
+    Evaluator that checks if a generated answer can be inferred from the provided contexts.
+
+    An LLM separates the answer into multiple statements and checks whether the statement can be inferred from the
+    context or not. The final score for the full answer is a number from 0.0 to 1.0. It represents the proportion of
+    statements that can be inferred from the provided contexts.
+
+    Usage example:
+    ```python
+    from haystack.components.evaluators import FaithfulnessEvaluator
+
+    questions = ["Who created the Python language?"]
+    contexts = [
+        [(
+            "Python, created by Guido van Rossum in the late 1980s, is a high-level general-purpose programming "
+            "language. Its design philosophy emphasizes code readability, and its language constructs aim to help "
+            "programmers write clear, logical code for both small and large-scale software projects."
+        )],
+    ]
+    predicted_answers = [
+        "Python is a high-level general-purpose programming language that was created by George Lucas."
+    ]
+    evaluator = FaithfulnessEvaluator()
+    result = evaluator.run(questions=questions, contexts=contexts, predicted_answers=predicted_answers)
+
+    print(result["individual_scores"])
+    # [0.5]
+    print(result["score"])
+    # 0.5
+    print(result["results"])
+    # [{'statements': ['Python is a high-level general-purpose programming language.',
+    # 'Python was created by George Lucas.'], 'statement_scores': [1, 0], 'score': 0.5}]
+    ```
+    """
+
+    def __init__(
+        self,
+        examples: list[dict[str, Any]] | None = None,
+        progress_bar: bool = True,
+        raise_on_failure: bool = True,
+        chat_generator: ChatGenerator | None = None,
+    ) -> None:
+        """
+        Creates an instance of FaithfulnessEvaluator.
+
+        If no LLM is specified using the `chat_generator` parameter, the component will use OpenAI in JSON mode.
+
+        :param examples:
+            Optional few-shot examples conforming to the expected input and output format of FaithfulnessEvaluator.
+            Default examples will be used if none are provided.
+            Each example must be a dictionary with keys "inputs" and "outputs".
+            "inputs" must be a dictionary with keys "questions", "contexts", and "predicted_answers".
+            "outputs" must be a dictionary with "statements" and "statement_scores".
+            Expected format:
+            ```python
+            [{
+                "inputs": {
+                    "questions": "What is the capital of Italy?", "contexts": ["Rome is the capital of Italy."],
+                    "predicted_answers": "Rome is the capital of Italy with more than 4 million inhabitants.",
+                },
+                "outputs": {
+                    "statements": ["Rome is the capital of Italy.", "Rome has more than 4 million inhabitants."],
+                    "statement_scores": [1, 0],
+                },
+            }]
+            ```
+        :param progress_bar:
+            Whether to show a progress bar during the evaluation.
+        :param raise_on_failure:
+            Whether to raise an exception if the API call fails.
+        :param chat_generator:
+            a ChatGenerator instance which represents the LLM.
+            In order for the component to work, the LLM should be configured to return a JSON object. For example,
+            when using the OpenAIChatGenerator, you should pass `{"response_format": {"type": "json_object"}}` in the
+            `generation_kwargs`.
+        """
+        self.instructions = (
+            "Your task is to judge the faithfulness or groundedness of statements based "
+            "on context information. First, please extract statements from a provided "
+            "predicted answer to a question. Second, calculate a faithfulness score for each "
+            "statement made in the predicted answer. The score is 1 if the statement can be "
+            "inferred from the provided context or 0 if it cannot be inferred."
+        )
+        self.inputs = [("questions", list[str]), ("contexts", list[list[str]]), ("predicted_answers", list[str])]
+        self.outputs = ["statements", "statement_scores"]
+        self.examples = examples or _DEFAULT_EXAMPLES
+
+        super(FaithfulnessEvaluator, self).__init__(  # noqa: UP008
+            instructions=self.instructions,
+            inputs=self.inputs,
+            outputs=self.outputs,
+            examples=self.examples,
+            chat_generator=chat_generator,
+            raise_on_failure=raise_on_failure,
+            progress_bar=progress_bar,
+        )
+
+    @component.output_types(individual_scores=list[float], score=float, results=list[dict[str, Any]])
+    def run(self, **inputs: Any) -> dict[str, Any]:
+        """
+        Run the LLM evaluator.
+
+        :param questions:
+            A list of questions.
+        :param contexts:
+            A nested list of contexts that correspond to the questions.
+        :param predicted_answers:
+            A list of predicted answers.
+        :returns:
+            A dictionary with the following outputs:
+                - `score`: Mean faithfulness score over all the provided input answers.
+                - `individual_scores`: A list of faithfulness scores for each input answer.
+                - `results`: A list of dictionaries with `statements` and `statement_scores` for each input answer.
+        """
+        result = super(FaithfulnessEvaluator, self).run(**inputs)  # noqa: UP008
+        # Post-process the raw results to calculate relevance metrics and scores
+        return self._postprocess_results(result)
+
+    @component.output_types(individual_scores=list[float], score=float, results=list[dict[str, Any]])
+    async def run_async(self, **inputs: Any) -> dict[str, Any]:
+        """
+        Run the LLM evaluator asynchronously.
+
+        :param questions:
+            A list of questions.
+        :param contexts:
+            A nested list of contexts that correspond to the questions.
+        :param predicted_answers:
+            A list of predicted answers.
+        :returns:
+            A dictionary with the following outputs:
+                - `score`: Mean faithfulness score over all the provided input answers.
+                - `individual_scores`: A list of faithfulness scores for each input answer.
+                - `results`: A list of dictionaries with `statements` and `statement_scores` for each input answer.
+        """
+        result = await super(FaithfulnessEvaluator, self).run_async(**inputs)  # noqa: UP008
+        # Post-process the raw results to calculate relevance metrics and scores
+        return self._postprocess_results(result)
+
+    def _postprocess_results(self, result: dict[str, Any]) -> dict[str, Any]:
+        """
+        Post-processes raw LLM evaluator outputs to compute faithfulness scores.
+
+        Calculates statement-level score averages, computes the overall mean faithfulness
+        score across successful queries, and updates the result payload.
+
+        :param result:
+            The raw evaluation dictionary from the base LLM evaluator.
+        :returns:
+            The updated dictionary containing final scores and tracking metrics.
+        """
+
+        # calculate average statement faithfulness score per query
+        for idx, res in enumerate(result["results"]):
+            if res is None:
+                result["results"][idx] = {"statements": [], "statement_scores": [], "score": float("nan")}
+                continue
+            if not res["statements"]:
+                res["score"] = 0
+            else:
+                res["score"] = np_mean(res["statement_scores"])
+
+        # calculate average answer faithfulness score over all queries
+        scores = [res["score"] for res in result["results"]]
+        valid_scores = [s for s in scores if not math.isnan(s)]
+        skipped = len(scores) - len(valid_scores)
+        if skipped:
+            logger.warning("{skipped} query(s) failed and were excluded from the score.", skipped=skipped)
+        result["score"] = np_mean(valid_scores) if valid_scores else float("nan")
+        result["individual_scores"] = scores
+
+        return result
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            A dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            chat_generator=component_to_dict(obj=self._chat_generator, name="chat_generator"),
+            examples=self.examples,
+            progress_bar=self.progress_bar,
+            raise_on_failure=self.raise_on_failure,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "FaithfulnessEvaluator":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data:
+            The dictionary representation of this component.
+        :returns:
+            The deserialized component instance.
+        """
+        if data["init_parameters"].get("chat_generator"):
+            deserialize_chatgenerator_inplace(data["init_parameters"], key="chat_generator")
+        return default_from_dict(cls, data)

haystack/components/evaluators/llm_evaluator.py

@@ -0,0 +1,462 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import asyncio
+import json
+from typing import Any
+
+from tqdm import tqdm
+from tqdm.asyncio import tqdm as async_tqdm
+
+from haystack import component, default_from_dict, default_to_dict, logging
+from haystack.components.builders import PromptBuilder
+from haystack.components.generators.chat.openai import OpenAIChatGenerator
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.core.serialization import component_to_dict
+from haystack.dataclasses.chat_message import ChatMessage
+from haystack.utils import deserialize_chatgenerator_inplace, deserialize_type, serialize_type
+from haystack.utils.misc import _parse_dict_from_json
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class LLMEvaluator:
+    """
+    Uses an LLM to evaluate inputs based on a prompt containing instructions and examples.
+
+    The default API requires an OpenAI API key to be provided as an environment variable "OPENAI_API_KEY".
+    The inputs are lists that are user-defined depending on the desired metric.
+    The output is a dictionary with a key `results` containing a list of evaluation results.
+    Each result is a dictionary with user-defined keys and values of either 0 for FALSE or 1 for TRUE respectively.
+
+    Usage example:
+    ```python
+    from haystack.components.evaluators import LLMEvaluator
+    evaluator = LLMEvaluator(
+        instructions="Is this answer problematic for children?",
+        inputs=[("predicted_answers", list[str])],
+        outputs=["score"],
+        examples=[
+            {"inputs": {"predicted_answers": "Damn, this is straight outta hell!!!"}, "outputs": {"score": 1}},
+            {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}},
+        ],
+    )
+    predicted_answers = [
+        "Football is the most popular sport with around 4 billion followers worldwide",
+        "Python language was created by Guido van Rossum.",
+    ]
+    results = evaluator.run(predicted_answers=predicted_answers)
+    print(results)
+    # {'results': [{'score': 0}, {'score': 0}]}
+    ```
+    """
+
+    def __init__(
+        self,
+        instructions: str,
+        inputs: list[tuple[str, type[list]]],
+        outputs: list[str],
+        examples: list[dict[str, Any]],
+        progress_bar: bool = True,
+        *,
+        raise_on_failure: bool = True,
+        chat_generator: ChatGenerator | None = None,
+    ) -> None:
+        """
+        Creates an instance of LLMEvaluator.
+
+        If no LLM is specified using the `chat_generator` parameter, the component will use OpenAI in JSON mode.
+
+        :param instructions:
+            The prompt instructions to use for evaluation.
+            Should be a question about the inputs that can be answered with yes or no.
+        :param inputs:
+            The inputs that the component expects as incoming connections and that it evaluates.
+            Each input is a tuple of an input name and input type. Input types must be lists.
+        :param outputs:
+            Output names of the evaluation results. They correspond to keys in the output dictionary.
+        :param examples:
+            Few-shot examples conforming to the expected input and output format as defined in the `inputs` and
+             `outputs` parameters.
+            Each example is a dictionary with keys "inputs" and "outputs"
+            They contain the input and output as dictionaries respectively.
+        :param raise_on_failure:
+            If True, the component will raise an exception on an unsuccessful API call.
+        :param progress_bar:
+            Whether to show a progress bar during the evaluation.
+        :param chat_generator:
+            a ChatGenerator instance which represents the LLM.
+            In order for the component to work, the LLM should be configured to return a JSON object. For example,
+            when using the OpenAIChatGenerator, you should pass `{"response_format": {"type": "json_object"}}` in the
+            `generation_kwargs`.
+        """
+        self.validate_init_parameters(inputs, outputs, examples)
+        component.set_input_types(self, **dict(inputs))
+
+        self.raise_on_failure = raise_on_failure
+        self.instructions = instructions
+        self.inputs = inputs
+        self.outputs = outputs
+        self.examples = examples
+        self.progress_bar = progress_bar
+
+        template = self.prepare_template()
+        self.builder = PromptBuilder(template=template)
+
+        if chat_generator is not None:
+            self._chat_generator = chat_generator
+        else:
+            generation_kwargs = {"response_format": {"type": "json_object"}, "seed": 42}
+            self._chat_generator = OpenAIChatGenerator(generation_kwargs=generation_kwargs)
+
+    def warm_up(self) -> None:
+        """
+        Warm up the underlying chat generator.
+        """
+        if hasattr(self._chat_generator, "warm_up"):
+            self._chat_generator.warm_up()
+
+    async def warm_up_async(self) -> None:
+        """
+        Warm up the underlying chat generator on the serving event loop.
+        """
+        if hasattr(self._chat_generator, "warm_up_async"):
+            await self._chat_generator.warm_up_async()
+        elif hasattr(self._chat_generator, "warm_up"):
+            self._chat_generator.warm_up()
+
+    def close(self) -> None:
+        """
+        Release the underlying chat generator's resources.
+        """
+        if hasattr(self._chat_generator, "close"):
+            self._chat_generator.close()
+
+    async def close_async(self) -> None:
+        """
+        Release the underlying chat generator's async resources.
+        """
+        if hasattr(self._chat_generator, "close_async"):
+            await self._chat_generator.close_async()
+        elif hasattr(self._chat_generator, "close"):
+            self._chat_generator.close()
+
+    @staticmethod
+    def validate_init_parameters(
+        inputs: list[tuple[str, type[list]]], outputs: list[str], examples: list[dict[str, Any]]
+    ) -> None:
+        """
+        Validate the init parameters.
+
+        :param inputs:
+            The inputs to validate.
+        :param outputs:
+            The outputs to validate.
+        :param examples:
+            The examples to validate.
+
+        :raises ValueError:
+            If the inputs are not a list of tuples with a string and a type of list.
+            If the outputs are not a list of strings.
+            If the examples are not a list of dictionaries.
+            If any example does not have keys "inputs" and "outputs" with values that are dictionaries with string keys.
+        """
+        # Validate inputs
+        if (
+            not isinstance(inputs, list)
+            or not all(isinstance(_input, tuple) for _input in inputs)
+            or not all(isinstance(_input[0], str) and _input[1] is not list and len(_input) == 2 for _input in inputs)
+        ):
+            msg = (
+                f"LLM evaluator expects inputs to be a list of tuples. Each tuple must contain an input name and "
+                f"type of list but received {inputs}."
+            )
+            raise ValueError(msg)
+
+        # Validate outputs
+        if not isinstance(outputs, list) or not all(isinstance(output, str) for output in outputs):
+            msg = f"LLM evaluator expects outputs to be a list of str but received {outputs}."
+            raise ValueError(msg)
+
+        # Validate examples are lists of dicts
+        if not isinstance(examples, list) or not all(isinstance(example, dict) for example in examples):
+            msg = f"LLM evaluator expects examples to be a list of dictionaries but received {examples}."
+            raise ValueError(msg)
+
+        # Validate each example
+        for example in examples:
+            if (
+                {"inputs", "outputs"} != example.keys()
+                or not all(isinstance(example[param], dict) for param in ["inputs", "outputs"])
+                or not all(isinstance(key, str) for param in ["inputs", "outputs"] for key in example[param])
+            ):
+                msg = (
+                    f"LLM evaluator expects each example to have keys `inputs` and `outputs` with values that are "
+                    f"dictionaries with str keys but received {example}."
+                )
+                raise ValueError(msg)
+
+    @component.output_types(results=list[dict[str, Any]])
+    def run(self, **inputs: Any) -> dict[str, Any]:
+        """
+        Run the LLM evaluator.
+
+        :param inputs:
+            The input values to evaluate. The keys are the input names and the values are lists of input values.
+        :returns:
+            A dictionary with a `results` entry that contains a list of results.
+            Each result is a dictionary containing the keys as defined in the `outputs` parameter of the LLMEvaluator
+            and the evaluation results as the values. If an exception occurs for a particular input value, the result
+            will be `None` for that entry.
+            If the API is "openai" and the response contains a "meta" key, the metadata from OpenAI will be included
+            in the output dictionary, under the key "meta".
+        :raises ValueError:
+            Only in the case that  `raise_on_failure` is set to True and the received inputs are not lists or have
+            different lengths, or if the output is not a valid JSON or doesn't contain the expected keys.
+        """
+        self.warm_up()
+
+        self.validate_input_parameters(dict(self.inputs), inputs)
+
+        # inputs is a dictionary with keys being input names and values being a list of input values
+        # We need to iterate through the lists in parallel for all keys of the dictionary
+        input_names, values = inputs.keys(), list(zip(*inputs.values(), strict=True))
+        list_of_input_names_to_values = [dict(zip(input_names, v, strict=True)) for v in values]
+
+        results: list[dict[str, Any] | None] = []
+        metadata = []
+        errors = 0
+        for input_names_to_values in tqdm(list_of_input_names_to_values, disable=not self.progress_bar):
+            prompt = self.builder.run(**input_names_to_values)
+            messages = [ChatMessage.from_user(prompt["prompt"])]
+            try:
+                result = self._chat_generator.run(messages=messages)
+            except Exception as e:
+                if self.raise_on_failure:
+                    raise ValueError(f"Error while generating response for prompt: {prompt}. Error: {e}") from e
+                logger.warning("Error while generating response for prompt: {prompt}. Error: {e}", prompt=prompt, e=e)
+                results.append(None)
+                errors += 1
+                continue
+
+            parsed_result = _parse_dict_from_json(
+                result["replies"][0].text, expected_keys=self.outputs, raise_on_failure=self.raise_on_failure
+            )
+            if parsed_result is None:
+                results.append(None)
+                errors += 1
+            else:
+                results.append(parsed_result)
+
+            if result["replies"][0].meta:
+                metadata.append(result["replies"][0].meta)
+
+        if errors > 0:
+            logger.warning(
+                "LLM evaluator failed for {errors} out of {len(list_of_input_names_to_values)} inputs.",
+                errors=errors,
+                len=len(list_of_input_names_to_values),
+            )
+
+        return {"results": results, "meta": metadata or None}
+
+    @component.output_types(results=list[dict[str, Any]])
+    async def run_async(self, **inputs: Any) -> dict[str, Any]:
+        """
+        Run the LLM evaluator asynchronously
+
+        :param inputs:
+            The input values to evaluate. The keys are the input names and the values are lists of input values.
+        :returns:
+            A dictionary with a `results` entry that contains a list of results.
+            Each result is a dictionary containing the keys as defined in the `outputs` parameter of the LLMEvaluator
+            and the evaluation results as the values. If an exception occurs for a particular input value, the result
+            will be `None` for that entry.
+            If the API is "openai" and the response contains a "meta" key, the metadata from OpenAI will be included
+            in the output dictionary, under the key "meta".
+        :raises TypeError:
+            If the chat generator does not support async execution.
+        :raises ValueError:
+            Only in the case that  `raise_on_failure` is set to True and the received inputs are not lists or have
+            different lengths, or if the output is not a valid JSON or doesn't contain the expected keys.
+        """
+
+        await self.warm_up_async()
+
+        self.validate_input_parameters(dict(self.inputs), inputs)
+
+        # inputs is a dictionary with keys being input names and values being a list of input values
+        # We need to iterate through the lists in parallel for all keys of the dictionary
+        input_names, values = inputs.keys(), list(zip(*inputs.values(), strict=True))
+        list_of_input_names_to_values = [dict(zip(input_names, v, strict=True)) for v in values]
+
+        results: list[dict[str, Any] | None] = []
+        metadata = []
+        errors = 0
+
+        generator_has_async = hasattr(self._chat_generator, "run_async")
+        for input_names_to_values in async_tqdm(list_of_input_names_to_values, disable=not self.progress_bar):
+            prompt = self.builder.run(**input_names_to_values)
+            messages = [ChatMessage.from_user(prompt["prompt"])]
+            try:
+                if generator_has_async:
+                    result = await self._chat_generator.run_async(messages=messages)  # type: ignore[attr-defined]
+                else:
+                    logger.debug(
+                        "{generator_type} does not implement 'run_async'."
+                        " Running the synchronous 'run' method in a thread to avoid blocking the event loop.",
+                        generator_type=type(self._chat_generator).__name__,
+                    )
+                    result = await asyncio.to_thread(self._chat_generator.run, messages=messages)
+            except Exception as e:
+                if self.raise_on_failure:
+                    raise ValueError(f"Error while generating response for prompt: {prompt}. Error: {e}") from e
+                logger.warning("Error while generating response for prompt: {prompt}. Error: {e}", prompt=prompt, e=e)
+                results.append(None)
+                errors += 1
+                continue
+
+            parsed_result = _parse_dict_from_json(
+                result["replies"][0].text, expected_keys=self.outputs, raise_on_failure=self.raise_on_failure
+            )
+            if parsed_result is None:
+                results.append(None)
+                errors += 1
+            else:
+                results.append(parsed_result)
+
+            if result["replies"][0].meta:
+                metadata.append(result["replies"][0].meta)
+
+        if errors > 0:
+            logger.warning(
+                "LLM evaluator failed for {errors} out of {len(list_of_input_names_to_values)} inputs.",
+                errors=errors,
+                len=len(list_of_input_names_to_values),
+            )
+
+        return {"results": results, "meta": metadata or None}
+
+    def prepare_template(self) -> str:
+        """
+        Prepare the prompt template.
+
+        Combine instructions, inputs, outputs, and examples into one prompt template with the following format:
+        Instructions:
+        `<instructions>`
+
+        Generate the response in JSON format with the following keys:
+        `<list of output keys>`
+        Consider the instructions and the examples below to determine those values.
+
+        Examples:
+        `<examples>`
+
+        Inputs:
+        `<inputs>`
+        Outputs:
+
+        :returns:
+            The prompt template.
+        """
+        inputs_section = (
+            "{" + ", ".join([f'"{input_socket[0]}": {{{{ {input_socket[0]} }}}}' for input_socket in self.inputs]) + "}"
+        )
+
+        examples_section = "\n".join(
+            [
+                "Inputs:\n" + json.dumps(example["inputs"]) + "\nOutputs:\n" + json.dumps(example["outputs"])
+                for example in self.examples
+            ]
+        )
+        return (
+            f"Instructions:\n"
+            f"{self.instructions}\n\n"
+            f"Generate the response in JSON format with the following keys:\n"
+            f"{json.dumps(self.outputs)}\n"
+            f"Consider the instructions and the examples below to determine those values.\n\n"
+            f"Examples:\n"
+            f"{examples_section}\n\n"
+            f"Inputs:\n"
+            f"{inputs_section}\n"
+            f"Outputs:\n"
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            The serialized component as a dictionary.
+        """
+        # Since we cannot currently serialize tuples, convert the inputs to a list.
+        inputs = [[name, serialize_type(type_)] for name, type_ in self.inputs]
+        return default_to_dict(
+            self,
+            instructions=self.instructions,
+            inputs=inputs,
+            outputs=self.outputs,
+            examples=self.examples,
+            chat_generator=component_to_dict(obj=self._chat_generator, name="chat_generator"),
+            progress_bar=self.progress_bar,
+            raise_on_failure=self.raise_on_failure,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "LLMEvaluator":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data:
+            The dictionary representation of this component.
+        :returns:
+            The deserialized component instance.
+        """
+        data["init_parameters"]["inputs"] = [
+            (name, deserialize_type(type_)) for name, type_ in data["init_parameters"]["inputs"]
+        ]
+
+        if data["init_parameters"].get("chat_generator"):
+            deserialize_chatgenerator_inplace(data["init_parameters"], key="chat_generator")
+
+        return default_from_dict(cls, data)
+
+    @staticmethod
+    def validate_input_parameters(expected: dict[str, Any], received: dict[str, Any]) -> None:
+        """
+        Validate the input parameters.
+
+        :param expected:
+            The expected input parameters.
+        :param received:
+            The received input parameters.
+
+        :raises ValueError:
+            If not all expected inputs are present in the received inputs
+            If the received inputs are not lists or have different lengths
+        """
+        # Validate that all expected inputs are present in the received inputs
+        for param in expected:
+            if param not in received:
+                msg = f"LLM evaluator expected input parameter '{param}' but received only {received.keys()}."
+                raise ValueError(msg)
+
+        # Validate that all received inputs are lists
+        if not all(isinstance(_input, list) for _input in received.values()):
+            msg = (
+                "LLM evaluator expects all input values to be lists but received "
+                f"{[type(_input) for _input in received.values()]}."
+            )
+            raise ValueError(msg)
+
+        # Validate that all received inputs are of the same length
+        inputs = received.values()
+        length = len(next(iter(inputs)))
+        if not all(len(_input) == length for _input in inputs):
+            msg = (
+                f"LLM evaluator expects all input lists to have the same length but received {inputs} with lengths "
+                f"{[len(_input) for _input in inputs]}."
+            )
+            raise ValueError(msg)

haystack/components/evaluators/sas_evaluator.py

@@ -0,0 +1,188 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Any
+
+from numpy import mean as np_mean
+
+from haystack import component, default_from_dict, default_to_dict
+from haystack.lazy_imports import LazyImport
+from haystack.utils import ComponentDevice, expit
+from haystack.utils.auth import Secret
+
+with LazyImport(message="Run 'pip install \"sentence-transformers>=5.0.0\"'") as sas_import:
+    from sentence_transformers import CrossEncoder, SentenceTransformer, util
+    from transformers import AutoConfig
+
+
+@component
+class SASEvaluator:
+    """
+    SASEvaluator computes the Semantic Answer Similarity (SAS) between a list of predictions and a one of ground truths.
+
+    It's usually used in Retrieval Augmented Generation (RAG) pipelines to evaluate the quality of the generated
+    answers. The SAS is computed using a pre-trained model from the Hugging Face model hub. The model can be either a
+    Bi-Encoder or a Cross-Encoder. The choice of the model is based on the `model` parameter.
+
+    Usage example:
+    ```python
+    from haystack.components.evaluators.sas_evaluator import SASEvaluator
+
+    evaluator = SASEvaluator(model="cross-encoder/ms-marco-MiniLM-L-6-v2")
+    ground_truths = [
+        "A construction budget of US $2.3 billion",
+        "The Eiffel Tower, completed in 1889, symbolizes Paris's cultural magnificence.",
+        "The Meiji Restoration in 1868 transformed Japan into a modernized world power.",
+    ]
+    predictions = [
+        "A construction budget of US $2.3 billion",
+        "The Eiffel Tower, completed in 1889, symbolizes Paris's cultural magnificence.",
+        "The Meiji Restoration in 1868 transformed Japan into a modernized world power.",
+    ]
+    result = evaluator.run(
+        ground_truth_answers=ground_truths, predicted_answers=predictions
+    )
+
+    print(result["score"])
+    # 0.9999673763910929
+
+    print(result["individual_scores"])
+    # [0.9999765157699585, 0.999968409538269, 0.9999572038650513]
+    ```
+    """
+
+    def __init__(
+        self,
+        model: str = "sentence-transformers/paraphrase-multilingual-mpnet-base-v2",
+        batch_size: int = 32,
+        device: ComponentDevice | None = None,
+        token: Secret = Secret.from_env_var(["HF_API_TOKEN", "HF_TOKEN"], strict=False),
+    ) -> None:
+        """
+        Creates a new instance of SASEvaluator.
+
+        :param model:
+            SentenceTransformers semantic textual similarity model, should be path or string pointing to a downloadable
+            model.
+        :param batch_size:
+            Number of prediction-label pairs to encode at once.
+        :param device:
+            The device on which the model is loaded. If `None`, the default device is automatically selected.
+        :param token:
+            The Hugging Face token for HTTP bearer authorization.
+            You can find your HF token in your [account settings](https://huggingface.co/settings/tokens)
+        """
+        sas_import.check()
+
+        self._model = model
+        self._batch_size = batch_size
+        self._device = device
+        self._token = token
+        self._similarity_model: SentenceTransformer | CrossEncoder | None = None
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serialize this component to a dictionary.
+
+        :returns:
+            The serialized component as a dictionary.
+        """
+        return default_to_dict(
+            self, model=self._model, batch_size=self._batch_size, device=self._device, token=self._token
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "SASEvaluator":
+        """
+        Deserialize this component from a dictionary.
+
+        :param data:
+            The dictionary representation of this component.
+        :returns:
+            The deserialized component instance.
+        """
+        return default_from_dict(cls, data)
+
+    def warm_up(self) -> None:
+        """
+        Initializes the component.
+        """
+        if self._similarity_model:
+            return
+
+        token = self._token.resolve_value() if self._token else None
+        config = AutoConfig.from_pretrained(self._model, use_auth_token=token)
+        cross_encoder_used = False
+        if config.architectures:
+            cross_encoder_used = any(arch.endswith("ForSequenceClassification") for arch in config.architectures)
+        device = ComponentDevice.resolve_device(self._device).to_torch_str()
+        # Based on the Model string we can load either Bi-Encoders or Cross Encoders.
+        # Similarity computation changes for both approaches
+        if cross_encoder_used:
+            self._similarity_model = CrossEncoder(self._model, device=device, token=token)
+        else:
+            self._similarity_model = SentenceTransformer(self._model, device=device, token=token)
+
+    @component.output_types(score=float, individual_scores=list[float])
+    def run(self, ground_truth_answers: list[str], predicted_answers: list[str]) -> dict[str, float | list[float]]:
+        """
+        SASEvaluator component run method.
+
+        Run the SASEvaluator to compute the Semantic Answer Similarity (SAS) between a list of predicted answers
+        and a list of ground truth answers. Both must be list of strings of same length.
+
+        :param ground_truth_answers:
+            A list of expected answers for each question.
+        :param predicted_answers:
+            A list of generated answers for each question.
+        :returns:
+            A dictionary with the following outputs:
+                - `score`: Mean SAS score over all the predictions/ground-truth pairs.
+                - `individual_scores`: A list of similarity scores for each prediction/ground-truth pair.
+        """
+        if len(ground_truth_answers) != len(predicted_answers):
+            raise ValueError("The number of predictions and labels must be the same.")
+
+        if any(answer is None for answer in predicted_answers):
+            raise ValueError("Predicted answers must not contain None values.")
+
+        if len(predicted_answers) == 0:
+            return {"score": 0.0, "individual_scores": [0.0]}
+
+        if not self._similarity_model:
+            self.warm_up()
+
+        if isinstance(self._similarity_model, CrossEncoder):
+            # For Cross Encoders we create a list of pairs of predictions and labels
+            sentence_pairs = list(zip(predicted_answers, ground_truth_answers, strict=True))
+            similarity_scores = self._similarity_model.predict(
+                sentence_pairs, batch_size=self._batch_size, convert_to_numpy=True
+            )
+
+            # All Cross Encoders do not return a set of logits scores that are normalized
+            # We normalize scores if they are larger than 1
+            if (similarity_scores > 1).any():
+                similarity_scores = expit(similarity_scores)
+
+            # Convert scores to list of floats from numpy array
+            similarity_scores = similarity_scores.tolist()
+
+        elif isinstance(self._similarity_model, SentenceTransformer):
+            # For Bi-encoders we create embeddings separately for predictions and labels
+            predictions_embeddings = self._similarity_model.encode(
+                predicted_answers, batch_size=self._batch_size, convert_to_tensor=True
+            )
+            label_embeddings = self._similarity_model.encode(
+                ground_truth_answers, batch_size=self._batch_size, convert_to_tensor=True
+            )
+
+            # Compute cosine-similarities
+            similarity_scores = [
+                float(util.cos_sim(pred_embedding, label_embedding).cpu().squeeze().numpy())
+                for pred_embedding, label_embedding in zip(predictions_embeddings, label_embeddings, strict=True)
+            ]
+
+        sas_score = np_mean(similarity_scores)
+
+        return {"score": sas_score, "individual_scores": similarity_scores}