chore: import upstream snapshot with attribution

e2e/pipelines/test_evaluation_pipeline.py

@@ -0,0 +1,291 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+
+import pytest
+
+from haystack import Document, Pipeline
+from haystack.components.builders import AnswerBuilder, ChatPromptBuilder
+from haystack.components.embedders import OpenAIDocumentEmbedder, OpenAITextEmbedder
+from haystack.components.evaluators import (
+    ContextRelevanceEvaluator,
+    DocumentMAPEvaluator,
+    DocumentMRREvaluator,
+    DocumentRecallEvaluator,
+    FaithfulnessEvaluator,
+    SASEvaluator,
+)
+from haystack.components.evaluators.document_recall import RecallMode
+from haystack.components.generators.chat import OpenAIChatGenerator
+from haystack.components.retrievers import InMemoryEmbeddingRetriever
+from haystack.components.writers import DocumentWriter
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.document_stores.types import DuplicatePolicy
+from haystack.evaluation import EvaluationRunResult
+from haystack.dataclasses import ChatMessage
+
+EMBEDDINGS_MODEL = "text-embedding-3-small"
+
+
+def indexing_pipeline(documents: list[Document]):
+    """Indexing the documents"""
+    document_store = InMemoryDocumentStore()
+    doc_writer = DocumentWriter(document_store=document_store, policy=DuplicatePolicy.SKIP)
+    doc_embedder = OpenAIDocumentEmbedder(model=EMBEDDINGS_MODEL, progress_bar=False)
+    ingestion_pipe = Pipeline()
+    ingestion_pipe.add_component(instance=doc_embedder, name="doc_embedder")
+    ingestion_pipe.add_component(instance=doc_writer, name="doc_writer")
+    ingestion_pipe.connect("doc_embedder.documents", "doc_writer.documents")
+    ingestion_pipe.run({"doc_embedder": {"documents": documents}})
+    return document_store
+
+
+def rag_pipeline(document_store: InMemoryDocumentStore, top_k: int):
+    """RAG pipeline"""
+    template = [
+        ChatMessage.from_system(
+            text="You have to answer the following question based on the given context information only."
+        ),
+        ChatMessage.from_user(
+            text="""Context:
+            {% for document in documents %}
+                {{ document.content }}
+            {% endfor %}
+
+            Question: {{question}}"""
+        ),
+    ]
+    rag = Pipeline()
+    rag.add_component("embedder", OpenAITextEmbedder(model=EMBEDDINGS_MODEL))
+    rag.add_component("retriever", InMemoryEmbeddingRetriever(document_store, top_k=top_k))
+    rag.add_component("prompt_builder", ChatPromptBuilder(template=template))
+    rag.add_component("generator", OpenAIChatGenerator(model="gpt-4o-mini"))
+    rag.add_component("answer_builder", AnswerBuilder())
+    rag.connect("embedder", "retriever.query_embedding")
+    rag.connect("retriever", "prompt_builder.documents")
+    rag.connect("prompt_builder", "generator")
+    rag.connect("generator.replies", "answer_builder.replies")
+    rag.connect("retriever", "answer_builder.documents")
+
+    return rag
+
+
+def evaluation_pipeline():
+    """
+    Create an evaluation pipeline with the following evaluators:
+
+    - DocumentMRREvaluator
+    - FaithfulnessEvaluator
+    - SASEvaluator
+    - DocumentMAPEvaluator
+    - DocumentRecallEvaluator
+    - ContextRelevanceEvaluator
+    """
+    eval_pipeline = Pipeline()
+    eval_pipeline.add_component("doc_mrr", DocumentMRREvaluator())
+    eval_pipeline.add_component("groundedness", FaithfulnessEvaluator())
+    eval_pipeline.add_component("sas", SASEvaluator())
+    eval_pipeline.add_component("doc_map", DocumentMAPEvaluator())
+    eval_pipeline.add_component("doc_recall_single_hit", DocumentRecallEvaluator(mode=RecallMode.SINGLE_HIT))
+    eval_pipeline.add_component("doc_recall_multi_hit", DocumentRecallEvaluator(mode=RecallMode.MULTI_HIT))
+    eval_pipeline.add_component("relevance", ContextRelevanceEvaluator())
+
+    return eval_pipeline
+
+
+def built_eval_input(questions, truth_docs, truth_answers, retrieved_docs, contexts, pred_answers):
+    """Helper function to build the input for the evaluation pipeline"""
+    return {
+        "doc_mrr": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs},
+        "groundedness": {"questions": questions, "contexts": contexts, "predicted_answers": pred_answers},
+        "sas": {"predicted_answers": pred_answers, "ground_truth_answers": truth_answers},
+        "doc_map": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs},
+        "doc_recall_single_hit": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs},
+        "doc_recall_multi_hit": {"ground_truth_documents": truth_docs, "retrieved_documents": retrieved_docs},
+        "relevance": {"questions": questions, "contexts": contexts},
+    }
+
+
+def run_rag_pipeline(documents, evaluation_questions, rag_pipeline_a):
+    """
+    Run the RAG pipeline and return the contexts, predicted answers, retrieved documents and ground truth documents
+    """
+
+    truth_docs = []
+    retrieved_docs = []
+    contexts = []
+    predicted_answers = []
+
+    for q in evaluation_questions:
+        response = rag_pipeline_a.run(
+            {
+                "embedder": {"text": q["question"]},
+                "prompt_builder": {"question": q["question"]},
+                "answer_builder": {"query": q["question"]},
+            }
+        )
+        truth_docs.append([doc for doc in documents if doc.meta["name"] in q["ground_truth_doc"] and doc.content])
+        retrieved_docs.append(response["answer_builder"]["answers"][0].documents)
+        contexts.append([doc.content for doc in response["answer_builder"]["answers"][0].documents])
+        predicted_answers.append(response["answer_builder"]["answers"][0].data)
+
+    return contexts, predicted_answers, retrieved_docs, truth_docs
+
+
+def built_input_for_results_eval(rag_results):
+    """Helper function to build the input for the results evaluation"""
+    return {
+        "Mean Reciprocal Rank": {
+            "individual_scores": rag_results["doc_mrr"]["individual_scores"],
+            "score": rag_results["doc_mrr"]["score"],
+        },
+        "Semantic Answer Similarity": {
+            "individual_scores": rag_results["sas"]["individual_scores"],
+            "score": rag_results["sas"]["score"],
+        },
+        "Faithfulness": {
+            "individual_scores": rag_results["groundedness"]["individual_scores"],
+            "score": rag_results["groundedness"]["score"],
+        },
+        "Document MAP": {
+            "individual_scores": rag_results["doc_map"]["individual_scores"],
+            "score": rag_results["doc_map"]["score"],
+        },
+        "Document Recall Single Hit": {
+            "individual_scores": rag_results["doc_recall_single_hit"]["individual_scores"],
+            "score": rag_results["doc_recall_single_hit"]["score"],
+        },
+        "Document Recall Multi Hit": {
+            "individual_scores": rag_results["doc_recall_multi_hit"]["individual_scores"],
+            "score": rag_results["doc_recall_multi_hit"]["score"],
+        },
+        "Contextual Relevance": {
+            "individual_scores": rag_results["relevance"]["individual_scores"],
+            "score": rag_results["relevance"]["score"],
+        },
+    }
+
+
+@pytest.mark.skipif(
+    not os.environ.get("OPENAI_API_KEY", None),
+    reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.",
+)
+def test_evaluation_pipeline(samples_path):
+    """Test an evaluation pipeline"""
+    eval_questions = [
+        {
+            "question": 'What falls within the term "cultural anthropology"?',
+            "answer": "the ideology and analytical stance of cultural relativism",
+            "ground_truth_doc": ["Culture.txt"],
+        },
+        {
+            "question": "Who was the spiritual guide during the Protestant Reformation?",
+            "answer": "Martin Bucer",
+            "ground_truth_doc": ["Strasbourg.txt"],
+        },
+        {
+            "question": "What is materialism?",
+            "answer": "a form of philosophical monism",
+            "ground_truth_doc": ["Materialism.txt"],
+        },
+    ]
+
+    questions = [q["question"] for q in eval_questions]
+    truth_answers = [q["answer"] for q in eval_questions]
+
+    # indexing documents
+    docs = []
+    full_path = os.path.join(str(samples_path) + "/test_documents/")
+    for article in os.listdir(full_path):
+        with open(f"{full_path}/{article}", "r") as f:
+            for text in f.read().split("\n"):
+                if doc := Document(content=text, meta={"name": article}) if text else None:
+                    docs.append(doc)
+    doc_store = indexing_pipeline(docs)
+
+    # running the RAG pipeline A + evaluation pipeline
+    rag_pipeline_a = rag_pipeline(doc_store, top_k=2)
+    contexts_a, pred_answers_a, retrieved_docs_a, truth_docs = run_rag_pipeline(docs, eval_questions, rag_pipeline_a)
+    eval_pipeline = evaluation_pipeline()
+    eval_input = built_eval_input(questions, truth_docs, truth_answers, retrieved_docs_a, contexts_a, pred_answers_a)
+    results_rag_a = eval_pipeline.run(eval_input)
+
+    # running the evaluation EvaluationRunResult
+    inputs_a = {
+        "question": questions,
+        "contexts": contexts_a,
+        "answer": truth_answers,
+        "predicted_answer": pred_answers_a,
+    }
+    results_a = built_input_for_results_eval(results_rag_a)
+    evaluation_result_a = EvaluationRunResult(run_name="rag_pipeline_a", results=results_a, inputs=inputs_a)
+    aggregated_score_report_json = evaluation_result_a.aggregated_report()
+
+    # assert the score report has all the metrics
+    assert len(aggregated_score_report_json["metrics"]) == 7
+    assert list(aggregated_score_report_json.keys()) == ["metrics", "score"]
+    assert list(aggregated_score_report_json["metrics"]) == [
+        "Mean Reciprocal Rank",
+        "Semantic Answer Similarity",
+        "Faithfulness",
+        "Document MAP",
+        "Document Recall Single Hit",
+        "Document Recall Multi Hit",
+        "Contextual Relevance",
+    ]
+
+    # assert the evaluation result has all the metrics, inputs and questions
+    detailed_report_json = evaluation_result_a.detailed_report()
+    assert list(detailed_report_json.keys()) == [
+        "question",
+        "contexts",
+        "answer",
+        "predicted_answer",
+        "Mean Reciprocal Rank",
+        "Semantic Answer Similarity",
+        "Faithfulness",
+        "Document MAP",
+        "Document Recall Single Hit",
+        "Document Recall Multi Hit",
+        "Contextual Relevance",
+    ]
+
+    # running the RAG pipeline B
+    rag_pipeline_b = rag_pipeline(doc_store, top_k=4)
+    contexts_b, pred_answers_b, retrieved_docs_b, truth_docs = run_rag_pipeline(docs, eval_questions, rag_pipeline_b)
+    eval_input = built_eval_input(questions, truth_docs, truth_answers, retrieved_docs_b, contexts_b, pred_answers_b)
+    results_rag_b = eval_pipeline.run(eval_input)
+
+    inputs_b = {
+        "question": questions,
+        "contexts": contexts_b,
+        "answer": truth_answers,
+        "predicted_answer": pred_answers_b,
+    }
+    results_b = built_input_for_results_eval(results_rag_b)
+    evaluation_result_b = EvaluationRunResult(run_name="rag_pipeline_b", results=results_b, inputs=inputs_b)
+    comparative_json = evaluation_result_a.comparative_detailed_report(evaluation_result_b)
+
+    # assert the comparative score report has all the metrics, inputs and questions
+    assert list(comparative_json.keys()) == [
+        "question",
+        "contexts",
+        "answer",
+        "predicted_answer",
+        "rag_pipeline_a_Mean Reciprocal Rank",
+        "rag_pipeline_a_Semantic Answer Similarity",
+        "rag_pipeline_a_Faithfulness",
+        "rag_pipeline_a_Document MAP",
+        "rag_pipeline_a_Document Recall Single Hit",
+        "rag_pipeline_a_Document Recall Multi Hit",
+        "rag_pipeline_a_Contextual Relevance",
+        "rag_pipeline_b_Mean Reciprocal Rank",
+        "rag_pipeline_b_Semantic Answer Similarity",
+        "rag_pipeline_b_Faithfulness",
+        "rag_pipeline_b_Document MAP",
+        "rag_pipeline_b_Document Recall Single Hit",
+        "rag_pipeline_b_Document Recall Multi Hit",
+        "rag_pipeline_b_Contextual Relevance",
+    ]

e2e/pipelines/test_pdf_content_extraction_pipeline.py

@@ -0,0 +1,90 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+
+import pytest
+
+from haystack import Pipeline
+from haystack.components.converters.pypdf import PyPDFToDocument
+from haystack.components.joiners import DocumentJoiner
+from haystack.components.preprocessors.document_splitter import DocumentSplitter
+from haystack.components.writers.document_writer import DocumentWriter
+from haystack.document_stores.in_memory import InMemoryDocumentStore
+from haystack.components.extractors.image.llm_document_content_extractor import LLMDocumentContentExtractor
+from haystack.components.generators.chat.openai import OpenAIChatGenerator
+from haystack.components.routers.document_length_router import DocumentLengthRouter
+
+
+@pytest.mark.skipif(
+    not os.environ.get("OPENAI_API_KEY", None),
+    reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.",
+)
+def test_pdf_content_extraction_pipeline():
+    """
+    Test a pipeline that processes PDFs with the following steps:
+    1. Convert PDFs to documents
+    2. Split documents by page
+    3. Route documents by length (short vs long)
+    4. Extract content from short documents using LLM
+    5. Join documents back together
+    6. Write to document store
+    """
+    document_store = InMemoryDocumentStore()
+
+    pdf_converter = PyPDFToDocument(store_full_path=True)
+    pdf_splitter = DocumentSplitter(split_by="page", split_length=1, skip_empty_documents=False)
+    doc_length_router = DocumentLengthRouter(threshold=10)
+    content_extractor = LLMDocumentContentExtractor(chat_generator=OpenAIChatGenerator(model="gpt-4o-mini"))
+    final_doc_joiner = DocumentJoiner(sort_by_score=False)
+    document_writer = DocumentWriter(document_store=document_store)
+
+    # Create and configure pipeline
+    indexing_pipe = Pipeline()
+    indexing_pipe.add_component("pdf_converter", pdf_converter)
+    indexing_pipe.add_component("pdf_splitter", pdf_splitter)
+    indexing_pipe.add_component("doc_length_router", doc_length_router)
+    indexing_pipe.add_component("content_extractor", content_extractor)
+    indexing_pipe.add_component("final_doc_joiner", final_doc_joiner)
+    indexing_pipe.add_component("document_writer", document_writer)
+
+    # Connect components
+    indexing_pipe.connect("pdf_converter.documents", "pdf_splitter.documents")
+    indexing_pipe.connect("pdf_splitter.documents", "doc_length_router.documents")
+    # The short PDF pages will be enriched/captioned
+    indexing_pipe.connect("doc_length_router.short_documents", "content_extractor.documents")
+    indexing_pipe.connect("doc_length_router.long_documents", "final_doc_joiner.documents")
+    indexing_pipe.connect("content_extractor.documents", "final_doc_joiner.documents")
+    indexing_pipe.connect("final_doc_joiner.documents", "document_writer.documents")
+
+    # Test with both text-searchable and non-text-searchable PDFs
+    test_files = [
+        "test/test_files/pdf/sample_pdf_1.pdf",  # a PDF with 4 pages
+        "test/test_files/pdf/non_text_searchable.pdf",  # a non-text searchable PDF with 1 page
+    ]
+
+    # Run the indexing pipeline
+    indexing_result = indexing_pipe.run(data={"sources": test_files})
+
+    assert indexing_result is not None
+    assert "document_writer" in indexing_result
+
+    indexed_documents = document_store.filter_documents()
+
+    # We expect documents from both PDFs
+    # sample_pdf_1.pdf has 4 pages, non_text_searchable.pdf has 1 page
+    assert len(indexed_documents) == 5
+
+    file_paths = {doc.meta["file_path"] for doc in indexed_documents}
+    assert file_paths == set(test_files)
+
+    for doc in indexed_documents:
+        assert hasattr(doc, "content")
+        assert hasattr(doc, "meta")
+        assert "file_path" in doc.meta
+        assert "page_number" in doc.meta
+
+    for doc in indexed_documents:
+        assert isinstance(doc.meta["page_number"], int)
+        assert doc.meta["page_number"] >= 1