chore: import upstream snapshot with attribution
CodeQL / Analyze (python) (push) Has been cancelled
Update Platform Components Table / update (push) Has been cancelled
Docker image release / Build base image (push) Has been cancelled
Sync docs with Docusaurus / sync (push) Has been cancelled
Tests / Check if changed (push) Has been cancelled
Tests / format (push) Has been cancelled
Tests / check-imports (push) Has been cancelled
Tests / Unit / macos-latest (push) Has been cancelled
Tests / Unit / ubuntu-latest (push) Has been cancelled
Tests / Unit / windows-latest (push) Has been cancelled
Tests / mypy (push) Has been cancelled
Tests / Integration / ubuntu-latest (push) Has been cancelled
Tests / Integration / macos-latest (push) Has been cancelled
Tests / Integration / windows-latest (push) Has been cancelled
Tests / notify-slack-on-failure (push) Has been cancelled
Tests / Mark tests as completed (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Update Platform Components Table / update (push) Has been cancelled
Docker image release / Build base image (push) Has been cancelled
Sync docs with Docusaurus / sync (push) Has been cancelled
Tests / Check if changed (push) Has been cancelled
Tests / format (push) Has been cancelled
Tests / check-imports (push) Has been cancelled
Tests / Unit / macos-latest (push) Has been cancelled
Tests / Unit / ubuntu-latest (push) Has been cancelled
Tests / Unit / windows-latest (push) Has been cancelled
Tests / mypy (push) Has been cancelled
Tests / Integration / ubuntu-latest (push) Has been cancelled
Tests / Integration / macos-latest (push) Has been cancelled
Tests / Integration / windows-latest (push) Has been cancelled
Tests / notify-slack-on-failure (push) Has been cancelled
Tests / Mark tests as completed (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,56 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from haystack.core.pipeline import Pipeline
|
||||
from haystack.core.pipeline.breakpoint import HAYSTACK_PIPELINE_SNAPSHOT_SAVE_ENABLED, load_pipeline_snapshot
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def enable_snapshot_saving(monkeypatch):
|
||||
"""Enable snapshot file saving for these integration tests."""
|
||||
monkeypatch.setenv(HAYSTACK_PIPELINE_SNAPSHOT_SAVE_ENABLED, "true")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def output_directory(tmp_path):
|
||||
"""Provide a temporary directory for snapshot files."""
|
||||
return tmp_path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def load_and_resume_pipeline_snapshot():
|
||||
"""Fixture that returns a function to load and resume a pipeline from a snapshot."""
|
||||
|
||||
def _resume(pipeline: Pipeline, output_directory: Path, component_name: str, data: dict | None = None) -> dict:
|
||||
"""
|
||||
Utility function to load and resume pipeline snapshot from a breakpoint file.
|
||||
|
||||
:param pipeline: The pipeline instance to resume
|
||||
:param output_directory: Directory containing the breakpoint files
|
||||
:param component_name: Component name to look for in breakpoint files
|
||||
:param data: Data to pass to the pipeline run (defaults to empty dict)
|
||||
|
||||
:returns:
|
||||
Dict containing the pipeline run results
|
||||
|
||||
:raises:
|
||||
ValueError: If no breakpoint file is found for the given component
|
||||
"""
|
||||
data = data or {}
|
||||
all_files = list(output_directory.glob("*"))
|
||||
|
||||
for full_path in all_files:
|
||||
f_name = Path(full_path).name
|
||||
if str(f_name).startswith(component_name):
|
||||
pipeline_snapshot = load_pipeline_snapshot(full_path)
|
||||
return pipeline.run(data=data, pipeline_snapshot=pipeline_snapshot)
|
||||
|
||||
msg = f"No files found for {component_name} in {output_directory}."
|
||||
raise ValueError(msg)
|
||||
|
||||
return _resume
|
||||
@@ -0,0 +1,83 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import pytest
|
||||
|
||||
from haystack import component
|
||||
from haystack.components.builders.answer_builder import AnswerBuilder
|
||||
from haystack.components.joiners import AnswerJoiner
|
||||
from haystack.core.errors import BreakpointException
|
||||
from haystack.core.pipeline.pipeline import Pipeline
|
||||
from haystack.dataclasses import ChatMessage
|
||||
from haystack.dataclasses.breakpoints import Breakpoint
|
||||
|
||||
|
||||
@component
|
||||
class FakeChatGenerator:
|
||||
def __init__(self, content: str, model_name: str):
|
||||
self.content = content
|
||||
self.model_name = model_name
|
||||
|
||||
@component.output_types(replies=list[ChatMessage])
|
||||
def run(self, messages: list[ChatMessage]) -> dict[str, list[ChatMessage]]:
|
||||
return {"replies": [ChatMessage.from_assistant(self.content)]}
|
||||
|
||||
|
||||
class TestPipelineBreakpoints:
|
||||
@pytest.fixture
|
||||
def answer_join_pipeline(self):
|
||||
"""Creates a pipeline with fake components."""
|
||||
pipeline = Pipeline()
|
||||
pipeline.add_component("gpt-4o", FakeChatGenerator("GPT-4 response", "gpt-4o"))
|
||||
pipeline.add_component("gpt-3", FakeChatGenerator("GPT-3 response", "gpt-3.5-turbo"))
|
||||
pipeline.add_component("answer_builder_a", AnswerBuilder())
|
||||
pipeline.add_component("answer_builder_b", AnswerBuilder())
|
||||
pipeline.add_component("answer_joiner", AnswerJoiner())
|
||||
pipeline.connect("gpt-4o.replies", "answer_builder_a")
|
||||
pipeline.connect("gpt-3.replies", "answer_builder_b")
|
||||
pipeline.connect("answer_builder_a.answers", "answer_joiner")
|
||||
pipeline.connect("answer_builder_b.answers", "answer_joiner")
|
||||
|
||||
return pipeline
|
||||
|
||||
BREAKPOINT_COMPONENTS = ["gpt-4o", "gpt-3", "answer_builder_a", "answer_builder_b", "answer_joiner"]
|
||||
|
||||
@pytest.mark.parametrize("component", BREAKPOINT_COMPONENTS, ids=BREAKPOINT_COMPONENTS)
|
||||
@pytest.mark.integration
|
||||
def test_pipeline_breakpoints_answer_joiner(
|
||||
self, answer_join_pipeline, output_directory, component, load_and_resume_pipeline_snapshot
|
||||
):
|
||||
"""
|
||||
Test that an answer joiner pipeline can be executed with breakpoints at each component.
|
||||
"""
|
||||
query = "What's Natural Language Processing?"
|
||||
messages = [
|
||||
ChatMessage.from_system("You are a helpful, respectful and honest assistant. Be super concise."),
|
||||
ChatMessage.from_user(query),
|
||||
]
|
||||
data = {
|
||||
"gpt-4o": {"messages": messages},
|
||||
"gpt-3": {"messages": messages},
|
||||
"answer_builder_a": {"query": query},
|
||||
"answer_builder_b": {"query": query},
|
||||
}
|
||||
|
||||
# Create a Breakpoint on-the-fly using the shared output directory
|
||||
break_point = Breakpoint(component_name=component, visit_count=0, snapshot_file_path=str(output_directory))
|
||||
|
||||
try:
|
||||
_ = answer_join_pipeline.run(data, break_point=break_point)
|
||||
except BreakpointException:
|
||||
pass
|
||||
|
||||
result = load_and_resume_pipeline_snapshot(
|
||||
pipeline=answer_join_pipeline,
|
||||
output_directory=output_directory,
|
||||
component_name=break_point.component_name,
|
||||
data=data,
|
||||
)
|
||||
assert result["answer_joiner"]
|
||||
assert len(result["answer_joiner"]["answers"]) == 2
|
||||
assert "GPT-4 response" in [a.data for a in result["answer_joiner"]["answers"]]
|
||||
assert "GPT-3 response" in [a.data for a in result["answer_joiner"]["answers"]]
|
||||
@@ -0,0 +1,90 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from haystack import component
|
||||
from haystack.components.converters import OutputAdapter
|
||||
from haystack.components.joiners import BranchJoiner
|
||||
from haystack.components.validators import JsonSchemaValidator
|
||||
from haystack.core.errors import BreakpointException
|
||||
from haystack.core.pipeline.pipeline import Pipeline
|
||||
from haystack.dataclasses import ChatMessage
|
||||
from haystack.dataclasses.breakpoints import Breakpoint
|
||||
|
||||
|
||||
@component
|
||||
class FakeChatGenerator:
|
||||
def __init__(self, content: str):
|
||||
self.content = content
|
||||
|
||||
@component.output_types(replies=list[ChatMessage])
|
||||
def run(
|
||||
self, messages: list[ChatMessage], generation_kwargs: dict | None = None, **kwargs: Any
|
||||
) -> dict[str, list[ChatMessage]]:
|
||||
return {"replies": [ChatMessage.from_assistant(self.content)]}
|
||||
|
||||
|
||||
class TestPipelineBreakpoints:
|
||||
@pytest.fixture
|
||||
def branch_joiner_pipeline(self):
|
||||
person_schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"first_name": {"type": "string", "pattern": "^[A-Z][a-z]+$"},
|
||||
"last_name": {"type": "string", "pattern": "^[A-Z][a-z]+$"},
|
||||
"nationality": {"type": "string", "enum": ["Italian", "Portuguese", "American"]},
|
||||
},
|
||||
"required": ["first_name", "last_name", "nationality"],
|
||||
}
|
||||
|
||||
content = '{"first_name": "Peter", "last_name": "Parker", "nationality": "American"}'
|
||||
|
||||
pipe = Pipeline()
|
||||
pipe.add_component("joiner", BranchJoiner(list[ChatMessage]))
|
||||
pipe.add_component("fc_llm", FakeChatGenerator(content))
|
||||
pipe.add_component("validator", JsonSchemaValidator(json_schema=person_schema))
|
||||
pipe.add_component("adapter", OutputAdapter("{{chat_message}}", list[ChatMessage], unsafe=True))
|
||||
|
||||
pipe.connect("adapter", "joiner")
|
||||
pipe.connect("joiner", "fc_llm")
|
||||
pipe.connect("fc_llm.replies", "validator.messages")
|
||||
pipe.connect("validator.validation_error", "joiner")
|
||||
|
||||
return pipe
|
||||
|
||||
BREAKPOINT_COMPONENTS = ["joiner", "fc_llm", "validator", "adapter"]
|
||||
|
||||
@pytest.mark.parametrize("component", BREAKPOINT_COMPONENTS, ids=BREAKPOINT_COMPONENTS)
|
||||
@pytest.mark.integration
|
||||
def test_pipeline_breakpoints_branch_joiner(
|
||||
self, branch_joiner_pipeline, output_directory, component, load_and_resume_pipeline_snapshot
|
||||
):
|
||||
data = {
|
||||
"fc_llm": {"generation_kwargs": {"response_format": {"type": "json_object"}}},
|
||||
"adapter": {"chat_message": [ChatMessage.from_user("Create JSON from Peter Parker")]},
|
||||
}
|
||||
|
||||
# Create a Breakpoint on-the-fly using the shared output directory
|
||||
break_point = Breakpoint(component_name=component, visit_count=0, snapshot_file_path=str(output_directory))
|
||||
|
||||
try:
|
||||
_ = branch_joiner_pipeline.run(data, break_point=break_point)
|
||||
except BreakpointException:
|
||||
pass
|
||||
|
||||
result = load_and_resume_pipeline_snapshot(
|
||||
pipeline=branch_joiner_pipeline,
|
||||
output_directory=output_directory,
|
||||
component_name=break_point.component_name,
|
||||
data=data,
|
||||
)
|
||||
assert result["validator"]
|
||||
valid_json = json.loads(result["validator"]["validated"][0].text)
|
||||
assert valid_json["first_name"] == "Peter"
|
||||
assert valid_json["last_name"] == "Parker"
|
||||
assert valid_json["nationality"] == "American"
|
||||
@@ -0,0 +1,86 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from haystack import Pipeline, component
|
||||
from haystack.components.builders import ChatPromptBuilder
|
||||
from haystack.components.joiners import ListJoiner
|
||||
from haystack.core.errors import BreakpointException
|
||||
from haystack.dataclasses import ChatMessage
|
||||
from haystack.dataclasses.breakpoints import Breakpoint
|
||||
|
||||
|
||||
@component
|
||||
class FakeChatGenerator:
|
||||
def __init__(self, response: str):
|
||||
self.response = response
|
||||
|
||||
@component.output_types(replies=list[ChatMessage])
|
||||
def run(self, messages: list[ChatMessage], **kwargs: Any) -> dict[str, list[ChatMessage]]:
|
||||
return {"replies": [ChatMessage.from_assistant(self.response)]}
|
||||
|
||||
|
||||
class TestPipelineBreakpoints:
|
||||
@pytest.fixture
|
||||
def list_joiner_pipeline(self):
|
||||
user_message = [ChatMessage.from_user("Give a brief answer the following question: {{query}}")]
|
||||
|
||||
feedback_prompt = """
|
||||
You are given a question and an answer.
|
||||
Your task is to provide a score and a brief feedback on the answer.
|
||||
Question: {{query}}
|
||||
Answer: {{response}}
|
||||
"""
|
||||
feedback_message = [ChatMessage.from_system(feedback_prompt)]
|
||||
|
||||
pipe = Pipeline()
|
||||
pipe.add_component("prompt_builder", ChatPromptBuilder(template=user_message, required_variables=None))
|
||||
pipe.add_component("llm", FakeChatGenerator("Nuclear physics is the study of atomic nuclei."))
|
||||
pipe.add_component(
|
||||
"feedback_prompt_builder", ChatPromptBuilder(template=feedback_message, required_variables=None)
|
||||
)
|
||||
pipe.add_component("feedback_llm", FakeChatGenerator("Score: 8/10. Concise and accurate."))
|
||||
pipe.add_component("list_joiner", ListJoiner(list[ChatMessage]))
|
||||
|
||||
pipe.connect("prompt_builder.prompt", "llm.messages")
|
||||
pipe.connect("prompt_builder.prompt", "list_joiner")
|
||||
pipe.connect("llm.replies", "list_joiner")
|
||||
pipe.connect("llm.replies", "feedback_prompt_builder.response")
|
||||
pipe.connect("feedback_prompt_builder.prompt", "feedback_llm.messages")
|
||||
pipe.connect("feedback_llm.replies", "list_joiner")
|
||||
|
||||
return pipe
|
||||
|
||||
BREAKPOINT_COMPONENTS = ["prompt_builder", "llm", "feedback_prompt_builder", "feedback_llm", "list_joiner"]
|
||||
|
||||
@pytest.mark.parametrize("component", BREAKPOINT_COMPONENTS, ids=BREAKPOINT_COMPONENTS)
|
||||
@pytest.mark.integration
|
||||
def test_list_joiner_pipeline(
|
||||
self, list_joiner_pipeline, output_directory, component, load_and_resume_pipeline_snapshot
|
||||
):
|
||||
query = "What is nuclear physics?"
|
||||
data = {
|
||||
"prompt_builder": {"template_variables": {"query": query}},
|
||||
"feedback_prompt_builder": {"template_variables": {"query": query}},
|
||||
}
|
||||
|
||||
# Create a Breakpoint on-the-fly using the shared output directory
|
||||
break_point = Breakpoint(component_name=component, visit_count=0, snapshot_file_path=str(output_directory))
|
||||
|
||||
try:
|
||||
_ = list_joiner_pipeline.run(data, break_point=break_point)
|
||||
except BreakpointException:
|
||||
pass
|
||||
|
||||
result = load_and_resume_pipeline_snapshot(
|
||||
pipeline=list_joiner_pipeline,
|
||||
output_directory=output_directory,
|
||||
component_name=break_point.component_name,
|
||||
data=data,
|
||||
)
|
||||
assert result["list_joiner"]
|
||||
assert len(result["list_joiner"]["values"]) == 3
|
||||
@@ -0,0 +1,145 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from haystack import component
|
||||
from haystack.components.builders import ChatPromptBuilder
|
||||
from haystack.core.errors import BreakpointException
|
||||
from haystack.core.pipeline.pipeline import Pipeline
|
||||
from haystack.dataclasses import ChatMessage
|
||||
from haystack.dataclasses.breakpoints import Breakpoint
|
||||
|
||||
|
||||
@component
|
||||
class OutputValidator:
|
||||
def __init__(self, pydantic_model: Any):
|
||||
self.pydantic_model = pydantic_model
|
||||
self.iteration_counter = 0
|
||||
|
||||
@component.output_types(valid_replies=list[ChatMessage], invalid_replies=list[ChatMessage], error_message=str)
|
||||
def run(self, replies: list[ChatMessage]) -> dict[str, list[ChatMessage] | str]:
|
||||
self.iteration_counter += 1
|
||||
try:
|
||||
assert replies[0].text is not None
|
||||
output_dict = json.loads(replies[0].text)
|
||||
self.pydantic_model.model_validate(output_dict)
|
||||
return {"valid_replies": replies}
|
||||
except (ValueError, ValidationError) as e:
|
||||
return {"invalid_replies": replies, "error_message": str(e)}
|
||||
|
||||
|
||||
@component
|
||||
class FakeChatGenerator:
|
||||
def __init__(self, response: str):
|
||||
self.response = response
|
||||
|
||||
@component.output_types(replies=list[ChatMessage])
|
||||
def run(self, messages: list[ChatMessage]) -> dict[str, list[ChatMessage]]:
|
||||
return {"replies": [ChatMessage.from_assistant(self.response)]}
|
||||
|
||||
|
||||
class City(BaseModel):
|
||||
name: str
|
||||
country: str
|
||||
population: int
|
||||
|
||||
|
||||
class CitiesData(BaseModel):
|
||||
cities: list[City]
|
||||
|
||||
|
||||
class TestPipelineBreakpointsLoops:
|
||||
"""
|
||||
This class contains tests for pipelines with validation loops and breakpoints.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def validation_loop_pipeline(self):
|
||||
"""Create a pipeline with validation loops for testing."""
|
||||
prompt_template = [
|
||||
ChatMessage.from_user(
|
||||
"""
|
||||
Create a JSON object from the information present in this passage: {{passage}}.
|
||||
Only use information that is present in the passage. Follow this JSON schema, but only return the
|
||||
actual instances without any additional schema definition:
|
||||
{{schema}}
|
||||
Make sure your response is a dict and not a list.
|
||||
{% if invalid_replies and error_message %}
|
||||
You already created the following output in a previous attempt: {{invalid_replies}}
|
||||
However, this doesn't comply with the format requirements from above and triggered this
|
||||
Python exception: {{error_message}}
|
||||
Correct the output and try again. Just return the corrected output without any extra explanations.
|
||||
{% endif %}
|
||||
"""
|
||||
)
|
||||
]
|
||||
|
||||
response_json = json.dumps(
|
||||
{
|
||||
"cities": [
|
||||
{"name": "Berlin", "country": "Germany", "population": 3850809},
|
||||
{"name": "Paris", "country": "France", "population": 2161000},
|
||||
{"name": "Lisbon", "country": "Portugal", "population": 504718},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
pipeline = Pipeline(max_runs_per_component=5)
|
||||
pipeline.add_component(
|
||||
instance=ChatPromptBuilder(template=prompt_template, required_variables=["passage", "schema"]),
|
||||
name="prompt_builder",
|
||||
)
|
||||
pipeline.add_component(instance=FakeChatGenerator(response=response_json), name="llm")
|
||||
pipeline.add_component(instance=OutputValidator(pydantic_model=CitiesData), name="output_validator")
|
||||
|
||||
pipeline.connect("prompt_builder.prompt", "llm.messages")
|
||||
pipeline.connect("llm.replies", "output_validator")
|
||||
pipeline.connect("output_validator.invalid_replies", "prompt_builder.invalid_replies")
|
||||
pipeline.connect("output_validator.error_message", "prompt_builder.error_message")
|
||||
|
||||
return pipeline
|
||||
|
||||
BREAKPOINT_COMPONENTS = ["prompt_builder", "llm", "output_validator"]
|
||||
|
||||
@pytest.mark.parametrize("component", BREAKPOINT_COMPONENTS, ids=BREAKPOINT_COMPONENTS)
|
||||
@pytest.mark.integration
|
||||
def test_pipeline_breakpoints_validation_loop(
|
||||
self, validation_loop_pipeline, output_directory, component, load_and_resume_pipeline_snapshot
|
||||
):
|
||||
"""
|
||||
Test that a pipeline with validation loops can be executed with breakpoints at each component.
|
||||
"""
|
||||
data = {"prompt_builder": {"passage": "Berlin, Paris, Lisbon...", "schema": "CitiesData schema"}}
|
||||
|
||||
# Create a Breakpoint on-the-fly using the shared output directory
|
||||
break_point = Breakpoint(component_name=component, visit_count=0, snapshot_file_path=str(output_directory))
|
||||
|
||||
try:
|
||||
_ = validation_loop_pipeline.run(data, break_point=break_point)
|
||||
except BreakpointException:
|
||||
pass
|
||||
|
||||
result = load_and_resume_pipeline_snapshot(
|
||||
pipeline=validation_loop_pipeline,
|
||||
output_directory=output_directory,
|
||||
component_name=break_point.component_name,
|
||||
data=data,
|
||||
)
|
||||
|
||||
assert "output_validator" in result
|
||||
assert "valid_replies" in result["output_validator"]
|
||||
valid_reply = result["output_validator"]["valid_replies"][0].text
|
||||
valid_json = json.loads(valid_reply)
|
||||
assert "cities" in valid_json
|
||||
assert len(valid_json["cities"]) == 3
|
||||
cities_data = CitiesData.model_validate(valid_json)
|
||||
assert len(cities_data.cities) == 3
|
||||
assert cities_data.cities[0].name == "Berlin"
|
||||
assert cities_data.cities[1].name == "Paris"
|
||||
assert cities_data.cities[2].name == "Lisbon"
|
||||
@@ -0,0 +1,135 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from random import random
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from haystack import Document, Pipeline, component
|
||||
from haystack.components.builders.answer_builder import AnswerBuilder
|
||||
from haystack.components.builders.prompt_builder import PromptBuilder
|
||||
from haystack.components.joiners import DocumentJoiner
|
||||
from haystack.components.retrievers.in_memory import InMemoryBM25Retriever, InMemoryEmbeddingRetriever
|
||||
from haystack.core.errors import BreakpointException
|
||||
from haystack.dataclasses.breakpoints import Breakpoint
|
||||
from haystack.document_stores.in_memory import InMemoryDocumentStore
|
||||
|
||||
|
||||
@component
|
||||
class FakeEmbedder:
|
||||
@component.output_types(documents=list[Document], embedding=list[float])
|
||||
def run(self, text: str) -> dict[str, list[Document] | list[float]]:
|
||||
return {"embedding": [random() for _ in range(100)]}
|
||||
|
||||
|
||||
@component
|
||||
class FakeRanker:
|
||||
@component.output_types(documents=list[Document])
|
||||
def run(self, query: str, documents: list[Document], top_k: int | None = None) -> dict[str, list[Document]]:
|
||||
for i, doc in enumerate(documents):
|
||||
doc.score = 1.0 / (i + 1)
|
||||
return {"documents": sorted(documents, key=lambda x: x.score or 0, reverse=True)[:top_k]}
|
||||
|
||||
|
||||
@component
|
||||
class FakeGenerator:
|
||||
@component.output_types(replies=list[str], meta=list[dict[str, Any]])
|
||||
def run(self, prompt: str) -> dict[str, list[str | dict[str, Any]]]:
|
||||
return {"replies": ["Mark lives in Berlin."], "meta": [{"model": "fake"}]}
|
||||
|
||||
|
||||
class TestPipelineBreakpoints:
|
||||
"""
|
||||
This class contains tests for pipelines with breakpoints.
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def document_store(self):
|
||||
"""Create and populate a document store for testing."""
|
||||
documents = [
|
||||
Document(content="My name is Jean and I live in Paris."),
|
||||
Document(content="My name is Mark and I live in Berlin."),
|
||||
Document(content="My name is Giorgio and I live in Rome."),
|
||||
]
|
||||
ds = InMemoryDocumentStore()
|
||||
# Add embeddings
|
||||
for doc in documents:
|
||||
doc.embedding = [random() for _ in range(100)]
|
||||
ds.write_documents(documents)
|
||||
return ds
|
||||
|
||||
@pytest.fixture
|
||||
def hybrid_rag_pipeline(self, document_store):
|
||||
"""Create a hybrid RAG pipeline for testing."""
|
||||
prompt_template = "Documents: {% for doc in documents %}{{ doc.content }}{% endfor %} Question: {{question}}"
|
||||
|
||||
pipeline = Pipeline()
|
||||
pipeline.add_component("bm25_retriever", InMemoryBM25Retriever(document_store=document_store))
|
||||
pipeline.add_component("query_embedder", FakeEmbedder())
|
||||
pipeline.add_component("embedding_retriever", InMemoryEmbeddingRetriever(document_store=document_store))
|
||||
pipeline.add_component("doc_joiner", DocumentJoiner())
|
||||
pipeline.add_component("ranker", FakeRanker())
|
||||
pipeline.add_component("prompt_builder", PromptBuilder(template=prompt_template))
|
||||
pipeline.add_component("llm", FakeGenerator())
|
||||
pipeline.add_component("answer_builder", AnswerBuilder())
|
||||
|
||||
pipeline.connect("query_embedder.embedding", "embedding_retriever.query_embedding")
|
||||
pipeline.connect("embedding_retriever", "doc_joiner.documents")
|
||||
pipeline.connect("bm25_retriever", "doc_joiner.documents")
|
||||
pipeline.connect("doc_joiner", "ranker.documents")
|
||||
pipeline.connect("ranker", "prompt_builder.documents")
|
||||
pipeline.connect("prompt_builder", "llm")
|
||||
pipeline.connect("llm.replies", "answer_builder.replies")
|
||||
pipeline.connect("llm.meta", "answer_builder.meta")
|
||||
pipeline.connect("doc_joiner", "answer_builder.documents")
|
||||
|
||||
return pipeline
|
||||
|
||||
BREAKPOINT_COMPONENTS = [
|
||||
"bm25_retriever",
|
||||
"query_embedder",
|
||||
"embedding_retriever",
|
||||
"doc_joiner",
|
||||
"ranker",
|
||||
"prompt_builder",
|
||||
"llm",
|
||||
"answer_builder",
|
||||
]
|
||||
|
||||
@pytest.mark.parametrize("component", BREAKPOINT_COMPONENTS, ids=BREAKPOINT_COMPONENTS)
|
||||
@pytest.mark.integration
|
||||
def test_pipeline_breakpoints_hybrid_rag(
|
||||
self, hybrid_rag_pipeline, output_directory, component, load_and_resume_pipeline_snapshot
|
||||
):
|
||||
"""
|
||||
Test that a hybrid RAG pipeline can be executed with breakpoints at each component.
|
||||
"""
|
||||
question = "Where does Mark live?"
|
||||
data = {
|
||||
"query_embedder": {"text": question},
|
||||
"bm25_retriever": {"query": question},
|
||||
"ranker": {"query": question, "top_k": 5},
|
||||
"prompt_builder": {"question": question},
|
||||
"answer_builder": {"query": question},
|
||||
}
|
||||
|
||||
# Create a Breakpoint on-the-fly using the shared output directory
|
||||
break_point = Breakpoint(component_name=component, visit_count=0, snapshot_file_path=str(output_directory))
|
||||
|
||||
try:
|
||||
_ = hybrid_rag_pipeline.run(data, break_point=break_point)
|
||||
except BreakpointException:
|
||||
pass
|
||||
|
||||
result = load_and_resume_pipeline_snapshot(
|
||||
pipeline=hybrid_rag_pipeline,
|
||||
output_directory=output_directory,
|
||||
component_name=break_point.component_name,
|
||||
data=data,
|
||||
)
|
||||
assert "answer_builder" in result
|
||||
assert result["answer_builder"]["answers"][0].data == "Mark lives in Berlin."
|
||||
assert len(result["answer_builder"]["answers"]) == 1
|
||||
assert result["answer_builder"]["answers"][0].meta["model"] == "fake"
|
||||
@@ -0,0 +1,64 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import pytest
|
||||
|
||||
from haystack.components.builders.chat_prompt_builder import ChatPromptBuilder
|
||||
from haystack.components.converters import OutputAdapter
|
||||
from haystack.components.joiners import StringJoiner
|
||||
from haystack.core.errors import BreakpointException
|
||||
from haystack.core.pipeline.pipeline import Pipeline
|
||||
from haystack.dataclasses import ChatMessage
|
||||
from haystack.dataclasses.breakpoints import Breakpoint
|
||||
|
||||
|
||||
class TestPipelineBreakpoints:
|
||||
@pytest.fixture
|
||||
def string_joiner_pipeline(self):
|
||||
pipeline = Pipeline()
|
||||
pipeline.add_component(
|
||||
"prompt_builder_1", ChatPromptBuilder(template=[ChatMessage.from_user("Builder 1: {{query}}")])
|
||||
)
|
||||
pipeline.add_component(
|
||||
"prompt_builder_2", ChatPromptBuilder(template=[ChatMessage.from_user("Builder 2: {{query}}")])
|
||||
)
|
||||
pipeline.add_component("adapter_1", OutputAdapter("{{messages[0].text}}", output_type=str))
|
||||
pipeline.add_component("adapter_2", OutputAdapter("{{messages[0].text}}", output_type=str))
|
||||
pipeline.add_component("string_joiner", StringJoiner())
|
||||
|
||||
pipeline.connect("prompt_builder_1.prompt", "adapter_1.messages")
|
||||
pipeline.connect("prompt_builder_2.prompt", "adapter_2.messages")
|
||||
pipeline.connect("adapter_1", "string_joiner.strings")
|
||||
pipeline.connect("adapter_2", "string_joiner.strings")
|
||||
|
||||
return pipeline
|
||||
|
||||
BREAKPOINT_COMPONENTS = ["prompt_builder_1", "prompt_builder_2", "adapter_1", "adapter_2", "string_joiner"]
|
||||
|
||||
@pytest.mark.parametrize("component", BREAKPOINT_COMPONENTS, ids=BREAKPOINT_COMPONENTS)
|
||||
@pytest.mark.integration
|
||||
def test_string_joiner_pipeline(
|
||||
self, string_joiner_pipeline, output_directory, component, load_and_resume_pipeline_snapshot
|
||||
):
|
||||
string_1 = "What's Natural Language Processing?"
|
||||
string_2 = "What is life?"
|
||||
data = {"prompt_builder_1": {"query": string_1}, "prompt_builder_2": {"query": string_2}}
|
||||
|
||||
# Create a Breakpoint on-the-fly using the shared output directory
|
||||
break_point = Breakpoint(component_name=component, visit_count=0, snapshot_file_path=str(output_directory))
|
||||
|
||||
try:
|
||||
_ = string_joiner_pipeline.run(data, break_point=break_point)
|
||||
except BreakpointException:
|
||||
pass
|
||||
|
||||
result = load_and_resume_pipeline_snapshot(
|
||||
pipeline=string_joiner_pipeline,
|
||||
output_directory=output_directory,
|
||||
component_name=break_point.component_name,
|
||||
data=data,
|
||||
)
|
||||
assert result["string_joiner"]
|
||||
assert "Builder 1: What's Natural Language Processing?" in result["string_joiner"]["strings"]
|
||||
assert "Builder 2: What is life?" in result["string_joiner"]["strings"]
|
||||
Reference in New Issue
Block a user