chore: import upstream snapshot with attribution
Sync docs with Docusaurus / sync (push) Waiting to run
Tests / Check if changed (push) Waiting to run
Tests / format (push) Blocked by required conditions
Tests / check-imports (push) Blocked by required conditions
Tests / Unit / macos-latest (push) Blocked by required conditions
Tests / Unit / ubuntu-latest (push) Blocked by required conditions
Tests / Unit / windows-latest (push) Blocked by required conditions
Tests / mypy (push) Blocked by required conditions
Tests / Integration / ubuntu-latest (push) Blocked by required conditions
Tests / Integration / macos-latest (push) Blocked by required conditions
Tests / Integration / windows-latest (push) Blocked by required conditions
Tests / notify-slack-on-failure (push) Blocked by required conditions
Tests / Mark tests as completed (push) Blocked by required conditions
Docker image release / Build base image (push) Waiting to run
CodeQL / Analyze (python) (push) Has been cancelled
Update Platform Components Table / update (push) Has been cancelled
Sync docs with Docusaurus / sync (push) Waiting to run
Tests / Check if changed (push) Waiting to run
Tests / format (push) Blocked by required conditions
Tests / check-imports (push) Blocked by required conditions
Tests / Unit / macos-latest (push) Blocked by required conditions
Tests / Unit / ubuntu-latest (push) Blocked by required conditions
Tests / Unit / windows-latest (push) Blocked by required conditions
Tests / mypy (push) Blocked by required conditions
Tests / Integration / ubuntu-latest (push) Blocked by required conditions
Tests / Integration / macos-latest (push) Blocked by required conditions
Tests / Integration / windows-latest (push) Blocked by required conditions
Tests / notify-slack-on-failure (push) Blocked by required conditions
Tests / Mark tests as completed (push) Blocked by required conditions
Docker image release / Build base image (push) Waiting to run
CodeQL / Analyze (python) (push) Has been cancelled
Update Platform Components Table / update (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
@@ -0,0 +1,10 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
|
||||
def callable_to_deserialize(hello: str) -> str:
|
||||
"""
|
||||
A function to test callable deserialization.
|
||||
"""
|
||||
return f"{hello}, world!"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,788 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import inspect
|
||||
from typing import Any, Protocol
|
||||
|
||||
import pytest
|
||||
|
||||
from haystack.dataclasses import Document
|
||||
from haystack.document_stores.errors import DuplicateDocumentError
|
||||
from haystack.document_stores.types import DocumentStore, DuplicatePolicy
|
||||
from haystack.testing.document_store import AssertDocumentsEqualMixin, FilterableDocsFixtureMixin
|
||||
|
||||
|
||||
class AsyncDocumentStore(DocumentStore, Protocol):
|
||||
async def count_documents_async(self) -> int:
|
||||
"""
|
||||
Returns the number of documents stored.
|
||||
"""
|
||||
...
|
||||
|
||||
async def filter_documents_async(self, filters: dict[str, Any] | None = None) -> list[Document]:
|
||||
"""
|
||||
Returns the documents that match the filters provided.
|
||||
"""
|
||||
...
|
||||
|
||||
async def write_documents_async(
|
||||
self, documents: list[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE
|
||||
) -> int:
|
||||
"""
|
||||
Writes Documents into the DocumentStore.
|
||||
"""
|
||||
...
|
||||
|
||||
async def delete_documents_async(self, document_ids: list[str]) -> None:
|
||||
"""
|
||||
Deletes all documents with matching document_ids from the DocumentStore.
|
||||
"""
|
||||
...
|
||||
|
||||
|
||||
class DeleteAllAsyncTest:
|
||||
"""
|
||||
Tests for Document Store delete_all_documents_async().
|
||||
|
||||
To use it create a custom test class and override the `document_store` fixture.
|
||||
Only mix in for stores that implement delete_all_documents_async.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _delete_all_async_supports_recreate(document_store: AsyncDocumentStore) -> str | None:
|
||||
"""
|
||||
Return the recreate parameter name if delete_all_documents_async supports it, else None.
|
||||
"""
|
||||
sig = inspect.signature(document_store.delete_all_documents_async) # type:ignore[attr-defined]
|
||||
if "recreate_index" in sig.parameters:
|
||||
return "recreate_index"
|
||||
if "recreate_collection" in sig.parameters:
|
||||
return "recreate_collection"
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_all_documents_async(document_store: AsyncDocumentStore):
|
||||
"""
|
||||
Test delete_all_documents_async() normal behaviour.
|
||||
|
||||
This test verifies that delete_all_documents_async() removes all documents from the store
|
||||
and that the store remains functional after deletion.
|
||||
"""
|
||||
docs = [Document(content="first doc", id="1"), Document(content="second doc", id="2")]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 2
|
||||
|
||||
await document_store.delete_all_documents_async() # type:ignore[attr-defined]
|
||||
assert await document_store.count_documents_async() == 0
|
||||
|
||||
new_doc = Document(content="new doc after delete all", id="3")
|
||||
await document_store.write_documents_async([new_doc])
|
||||
assert await document_store.count_documents_async() == 1
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_all_documents_empty_store_async(document_store: AsyncDocumentStore):
|
||||
"""
|
||||
Test delete_all_documents_async() on an empty store.
|
||||
|
||||
This should not raise an error and should leave the store empty.
|
||||
"""
|
||||
assert await document_store.count_documents_async() == 0
|
||||
await document_store.delete_all_documents_async() # type:ignore[attr-defined]
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_all_documents_without_recreate_index_async(document_store: AsyncDocumentStore):
|
||||
"""
|
||||
Test delete_all_documents_async() with recreate_index/recreate_collection=False when supported.
|
||||
|
||||
Skipped if the store's delete_all_documents_async does not have recreate_index or recreate_collection.
|
||||
"""
|
||||
param_name = DeleteAllAsyncTest._delete_all_async_supports_recreate(document_store)
|
||||
if param_name is None:
|
||||
pytest.skip("delete_all_documents_async has no recreate_index or recreate_collection parameter")
|
||||
|
||||
docs = [Document(id="1", content="A first document"), Document(id="2", content="Second document")]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 2
|
||||
|
||||
await document_store.delete_all_documents_async(**{param_name: False}) # type:ignore[attr-defined]
|
||||
assert await document_store.count_documents_async() == 0
|
||||
|
||||
new_doc = Document(id="3", content="New document after delete all")
|
||||
await document_store.write_documents_async([new_doc])
|
||||
assert await document_store.count_documents_async() == 1
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_all_documents_with_recreate_index_async(document_store: AsyncDocumentStore):
|
||||
"""
|
||||
Test delete_all_documents_async() with recreate_index/recreate_collection=True when supported.
|
||||
|
||||
Skipped if the store's delete_all_documents_async does not have recreate_index or recreate_collection.
|
||||
"""
|
||||
param_name = DeleteAllAsyncTest._delete_all_async_supports_recreate(document_store)
|
||||
if param_name is None:
|
||||
pytest.skip("delete_all_documents_async has no recreate_index or recreate_collection parameter")
|
||||
|
||||
docs = [Document(id="1", content="A first document"), Document(id="2", content="Second document")]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 2
|
||||
|
||||
await document_store.delete_all_documents_async(**{param_name: True}) # type:ignore[attr-defined]
|
||||
assert await document_store.count_documents_async() == 0
|
||||
|
||||
new_doc = Document(id="3", content="New document after delete all with recreate")
|
||||
await document_store.write_documents_async([new_doc])
|
||||
assert await document_store.count_documents_async() == 1
|
||||
|
||||
retrieved = await document_store.filter_documents_async()
|
||||
assert len(retrieved) == 1
|
||||
assert retrieved[0].content == "New document after delete all with recreate"
|
||||
|
||||
|
||||
class CountDocumentsAsyncTest:
|
||||
"""
|
||||
Utility class to test a Document Store `count_documents_async` method.
|
||||
|
||||
To use it create a custom test class and override the `document_store` fixture to return your Document Store.
|
||||
Example usage:
|
||||
|
||||
```python
|
||||
class MyDocumentStoreTest(CountDocumentsAsyncTest):
|
||||
@pytest.fixture
|
||||
def document_store(self):
|
||||
return MyDocumentStore()
|
||||
```
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_empty_async(document_store: AsyncDocumentStore):
|
||||
"""Test count is zero for an empty document store."""
|
||||
assert await document_store.count_documents_async() == 0
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_not_empty_async(document_store: AsyncDocumentStore):
|
||||
"""Test count is greater than zero if the document store contains documents."""
|
||||
await document_store.write_documents_async(
|
||||
[Document(content="test doc 1"), Document(content="test doc 2"), Document(content="test doc 3")]
|
||||
)
|
||||
assert await document_store.count_documents_async() == 3
|
||||
|
||||
|
||||
class CountDocumentsByFilterAsyncTest:
|
||||
"""
|
||||
Tests for Document Store count_documents_by_filter_async().
|
||||
|
||||
Only mix in for stores that implement count_documents_by_filter_async.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_documents_by_filter_async_simple(document_store: AsyncDocumentStore):
|
||||
"""Test count_documents_by_filter_async() with a simple equality filter."""
|
||||
docs = [
|
||||
Document(content="Doc 1", meta={"category": "A", "status": "active"}),
|
||||
Document(content="Doc 2", meta={"category": "B", "status": "active"}),
|
||||
Document(content="Doc 3", meta={"category": "A", "status": "inactive"}),
|
||||
Document(content="Doc 4", meta={"category": "A", "status": "active"}),
|
||||
]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 4
|
||||
|
||||
count = await document_store.count_documents_by_filter_async( # type:ignore[attr-defined]
|
||||
filters={"field": "meta.category", "operator": "==", "value": "A"}
|
||||
)
|
||||
assert count == 3
|
||||
|
||||
count = await document_store.count_documents_by_filter_async( # type:ignore[attr-defined]
|
||||
filters={"field": "meta.category", "operator": "==", "value": "B"}
|
||||
)
|
||||
assert count == 1
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_documents_by_filter_async_compound(document_store: AsyncDocumentStore):
|
||||
"""Test count_documents_by_filter_async() with AND filter."""
|
||||
docs = [
|
||||
Document(content="Doc 1", meta={"category": "A", "status": "active"}),
|
||||
Document(content="Doc 2", meta={"category": "B", "status": "active"}),
|
||||
Document(content="Doc 3", meta={"category": "A", "status": "inactive"}),
|
||||
Document(content="Doc 4", meta={"category": "A", "status": "active"}),
|
||||
]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 4
|
||||
|
||||
count = await document_store.count_documents_by_filter_async( # type:ignore[attr-defined]
|
||||
filters={
|
||||
"operator": "AND",
|
||||
"conditions": [
|
||||
{"field": "meta.category", "operator": "==", "value": "A"},
|
||||
{"field": "meta.status", "operator": "==", "value": "active"},
|
||||
],
|
||||
}
|
||||
)
|
||||
assert count == 2
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_documents_by_filter_async_no_matches(document_store: AsyncDocumentStore):
|
||||
"""Test count_documents_by_filter_async() when filter matches no documents."""
|
||||
docs = [Document(content="Doc 1", meta={"category": "A"}), Document(content="Doc 2", meta={"category": "B"})]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 2
|
||||
|
||||
count = await document_store.count_documents_by_filter_async( # type:ignore[attr-defined]
|
||||
filters={"field": "meta.category", "operator": "==", "value": "Z"}
|
||||
)
|
||||
assert count == 0
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_documents_by_filter_async_empty_collection(document_store: AsyncDocumentStore):
|
||||
"""Test count_documents_by_filter_async() on an empty store."""
|
||||
assert await document_store.count_documents_async() == 0
|
||||
|
||||
count = await document_store.count_documents_by_filter_async( # type:ignore[attr-defined]
|
||||
filters={"field": "meta.category", "operator": "==", "value": "A"}
|
||||
)
|
||||
assert count == 0
|
||||
|
||||
|
||||
class CountUniqueMetadataByFilterAsyncTest:
|
||||
"""
|
||||
Tests for Document Store count_unique_metadata_by_filter_async().
|
||||
|
||||
Only mix in for stores that implement count_unique_metadata_by_filter_async.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_unique_metadata_by_filter_async_all_documents(document_store: AsyncDocumentStore):
|
||||
"""Test count_unique_metadata_by_filter_async() with no filter returns distinct counts for all docs."""
|
||||
docs = [
|
||||
Document(content="Doc 1", meta={"category": "A", "status": "active", "priority": 1}),
|
||||
Document(content="Doc 2", meta={"category": "B", "status": "active", "priority": 2}),
|
||||
Document(content="Doc 3", meta={"category": "A", "status": "inactive", "priority": 1}),
|
||||
Document(content="Doc 4", meta={"category": "A", "status": "active", "priority": 3}),
|
||||
Document(content="Doc 5", meta={"category": "C", "status": "active", "priority": 2}),
|
||||
]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 5
|
||||
|
||||
counts = await document_store.count_unique_metadata_by_filter_async( # type:ignore[attr-defined]
|
||||
filters={}, metadata_fields=["category", "status", "priority"]
|
||||
)
|
||||
assert counts["category"] == 3
|
||||
assert counts["status"] == 2
|
||||
assert counts["priority"] == 3
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_unique_metadata_by_filter_async_with_filter(document_store: AsyncDocumentStore):
|
||||
"""Test count_unique_metadata_by_filter_async() with a filter."""
|
||||
docs = [
|
||||
Document(content="Doc 1", meta={"category": "A", "status": "active", "priority": 1}),
|
||||
Document(content="Doc 2", meta={"category": "B", "status": "active", "priority": 2}),
|
||||
Document(content="Doc 3", meta={"category": "A", "status": "inactive", "priority": 1}),
|
||||
Document(content="Doc 4", meta={"category": "A", "status": "active", "priority": 3}),
|
||||
]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 4
|
||||
|
||||
counts = await document_store.count_unique_metadata_by_filter_async( # type:ignore[attr-defined]
|
||||
filters={"field": "meta.category", "operator": "==", "value": "A"}, metadata_fields=["status", "priority"]
|
||||
)
|
||||
assert counts["status"] == 2
|
||||
assert counts["priority"] == 2
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_count_unique_metadata_by_filter_async_with_multiple_filters(document_store: AsyncDocumentStore):
|
||||
"""Test counting unique metadata asynchronously with multiple filters."""
|
||||
docs = [
|
||||
Document(content="Doc 1", meta={"category": "A", "year": 2023}),
|
||||
Document(content="Doc 2", meta={"category": "A", "year": 2024}),
|
||||
Document(content="Doc 3", meta={"category": "B", "year": 2023}),
|
||||
Document(content="Doc 4", meta={"category": "B", "year": 2024}),
|
||||
]
|
||||
await document_store.write_documents_async(docs)
|
||||
|
||||
counts = await document_store.count_unique_metadata_by_filter_async( # type:ignore[attr-defined]
|
||||
filters={
|
||||
"operator": "AND",
|
||||
"conditions": [
|
||||
{"field": "meta.category", "operator": "==", "value": "B"},
|
||||
{"field": "meta.year", "operator": "==", "value": 2023},
|
||||
],
|
||||
},
|
||||
metadata_fields=["category", "year"],
|
||||
)
|
||||
assert counts == {"category": 1, "year": 1}
|
||||
|
||||
|
||||
class DeleteByFilterAsyncTest:
|
||||
"""
|
||||
Tests for Document Store delete_by_filter_async().
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _delete_by_filter_params(document_store: AsyncDocumentStore) -> dict[str, bool]:
|
||||
"""
|
||||
Return optional parameters supported by delete_by_filter_async.
|
||||
"""
|
||||
sig = inspect.signature(document_store.delete_by_filter_async) # type:ignore[attr-defined]
|
||||
return {"refresh": True} if "refresh" in sig.parameters else {}
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_by_filter_async(document_store: AsyncDocumentStore):
|
||||
"""Delete documents matching a filter and verify count and remaining docs."""
|
||||
docs = [
|
||||
Document(content="Doc 1", meta={"category": "Alpha"}),
|
||||
Document(content="Doc 2", meta={"category": "Beta"}),
|
||||
Document(content="Doc 3", meta={"category": "Alpha"}),
|
||||
]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 3
|
||||
|
||||
params = DeleteByFilterAsyncTest._delete_by_filter_params(document_store)
|
||||
deleted_count = await document_store.delete_by_filter_async( # type:ignore[attr-defined]
|
||||
filters={"field": "meta.category", "operator": "==", "value": "Alpha"}, **params
|
||||
)
|
||||
assert deleted_count == 2
|
||||
assert await document_store.count_documents_async() == 1
|
||||
|
||||
remaining_docs = await document_store.filter_documents_async()
|
||||
assert len(remaining_docs) == 1
|
||||
assert remaining_docs[0].meta["category"] == "Beta"
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_by_filter_no_matches_async(document_store: AsyncDocumentStore):
|
||||
"""Delete with a filter that matches no documents returns 0 and leaves store unchanged."""
|
||||
docs = [
|
||||
Document(content="Doc 1", meta={"category": "Alpha"}),
|
||||
Document(content="Doc 2", meta={"category": "Beta"}),
|
||||
]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 2
|
||||
|
||||
params = DeleteByFilterAsyncTest._delete_by_filter_params(document_store)
|
||||
deleted_count = await document_store.delete_by_filter_async( # type:ignore[attr-defined]
|
||||
filters={"field": "meta.category", "operator": "==", "value": "Gamma"}, **params
|
||||
)
|
||||
assert deleted_count == 0
|
||||
assert await document_store.count_documents_async() == 2
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_by_filter_advanced_filters_async(document_store: AsyncDocumentStore):
|
||||
"""Delete with AND/OR filter combinations and verify remaining documents."""
|
||||
docs = [
|
||||
Document(content="Doc 1", meta={"category": "Alpha", "year": 2023, "status": "draft"}),
|
||||
Document(content="Doc 2", meta={"category": "Alpha", "year": 2024, "status": "published"}),
|
||||
Document(content="Doc 3", meta={"category": "Beta", "year": 2023, "status": "draft"}),
|
||||
]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 3
|
||||
|
||||
params = DeleteByFilterAsyncTest._delete_by_filter_params(document_store)
|
||||
deleted_count = await document_store.delete_by_filter_async( # type:ignore[attr-defined]
|
||||
filters={
|
||||
"operator": "AND",
|
||||
"conditions": [
|
||||
{"field": "meta.category", "operator": "==", "value": "Alpha"},
|
||||
{"field": "meta.year", "operator": "==", "value": 2023},
|
||||
],
|
||||
},
|
||||
**params,
|
||||
)
|
||||
assert deleted_count == 1
|
||||
assert await document_store.count_documents_async() == 2
|
||||
|
||||
deleted_count = await document_store.delete_by_filter_async( # type:ignore[attr-defined]
|
||||
filters={
|
||||
"operator": "OR",
|
||||
"conditions": [
|
||||
{"field": "meta.category", "operator": "==", "value": "Beta"},
|
||||
{"field": "meta.status", "operator": "==", "value": "published"},
|
||||
],
|
||||
},
|
||||
**params,
|
||||
)
|
||||
assert deleted_count == 2
|
||||
assert await document_store.count_documents_async() == 0
|
||||
|
||||
|
||||
class UpdateByFilterAsyncTest:
|
||||
"""
|
||||
Tests for Document Store update_by_filter_async().
|
||||
|
||||
Only mix in for stores that implement update_by_filter_async.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_by_filter_async(document_store: AsyncDocumentStore, filterable_docs: list[Document]):
|
||||
"""Update documents matching a filter asynchronously and verify count and meta changes."""
|
||||
await document_store.write_documents_async(filterable_docs)
|
||||
expected_count = len([d for d in filterable_docs if d.meta.get("chapter") == "intro"])
|
||||
assert await document_store.count_documents_async() == len(filterable_docs)
|
||||
|
||||
sig = inspect.signature(document_store.update_by_filter_async) # type:ignore[attr-defined]
|
||||
params = {"refresh": True} if "refresh" in sig.parameters else {}
|
||||
updated_count = await document_store.update_by_filter_async( # type:ignore[attr-defined]
|
||||
filters={"field": "meta.chapter", "operator": "==", "value": "intro"}, meta={"updated": True}, **params
|
||||
)
|
||||
assert updated_count == expected_count
|
||||
|
||||
updated_docs = await document_store.filter_documents_async(
|
||||
filters={"field": "meta.updated", "operator": "==", "value": True}
|
||||
)
|
||||
assert len(updated_docs) == expected_count
|
||||
for doc in updated_docs:
|
||||
assert doc.meta["chapter"] == "intro"
|
||||
assert doc.meta["updated"] is True
|
||||
|
||||
not_updated_docs = await document_store.filter_documents_async(
|
||||
filters={"field": "meta.chapter", "operator": "==", "value": "abstract"}
|
||||
)
|
||||
for doc in not_updated_docs:
|
||||
assert doc.meta.get("updated") is not True
|
||||
|
||||
|
||||
class WriteDocumentsAsyncTest(AssertDocumentsEqualMixin):
|
||||
"""
|
||||
Utility class to test a Document Store `write_documents_async` method.
|
||||
|
||||
To use it create a custom test class and override the `document_store` fixture to return your Document Store.
|
||||
The Document Store `filter_documents_async` method must be at least partly implemented to return all stored
|
||||
Documents for these tests to work correctly.
|
||||
Example usage:
|
||||
|
||||
```python
|
||||
class MyDocumentStoreTest(WriteDocumentsAsyncTest):
|
||||
@pytest.fixture
|
||||
def document_store(self):
|
||||
return MyDocumentStore()
|
||||
```
|
||||
"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_documents_async(self, document_store: AsyncDocumentStore):
|
||||
"""
|
||||
Test write_documents_async() default behaviour.
|
||||
"""
|
||||
msg = (
|
||||
"Default write_documents_async() behaviour depends on the Document Store implementation, "
|
||||
"as we don't enforce a default behaviour when no policy is set. "
|
||||
"Override this test in your custom test class."
|
||||
)
|
||||
raise NotImplementedError(msg)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_documents_duplicate_fail_async(self, document_store: AsyncDocumentStore):
|
||||
"""Test write_documents_async() fails when writing documents with same id and `DuplicatePolicy.FAIL`."""
|
||||
doc = Document(content="test doc")
|
||||
assert await document_store.write_documents_async([doc], policy=DuplicatePolicy.FAIL) == 1
|
||||
with pytest.raises(DuplicateDocumentError):
|
||||
await document_store.write_documents_async(documents=[doc], policy=DuplicatePolicy.FAIL)
|
||||
self.assert_documents_are_equal(await document_store.filter_documents_async(), [doc])
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_documents_duplicate_skip_async(document_store: AsyncDocumentStore):
|
||||
"""Test write_documents_async() skips writing when using DuplicatePolicy.SKIP."""
|
||||
doc = Document(content="test doc")
|
||||
assert await document_store.write_documents_async([doc], policy=DuplicatePolicy.SKIP) == 1
|
||||
assert await document_store.write_documents_async(documents=[doc], policy=DuplicatePolicy.SKIP) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_documents_duplicate_overwrite_async(self, document_store: AsyncDocumentStore):
|
||||
"""Test write_documents_async() overwrites when using DuplicatePolicy.OVERWRITE."""
|
||||
doc1 = Document(id="1", content="test doc 1")
|
||||
doc2 = Document(id="1", content="test doc 2")
|
||||
|
||||
assert await document_store.write_documents_async([doc2], policy=DuplicatePolicy.OVERWRITE) == 1
|
||||
self.assert_documents_are_equal(await document_store.filter_documents_async(), [doc2])
|
||||
assert await document_store.write_documents_async(documents=[doc1], policy=DuplicatePolicy.OVERWRITE) == 1
|
||||
self.assert_documents_are_equal(await document_store.filter_documents_async(), [doc1])
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_write_documents_invalid_input_async(document_store: AsyncDocumentStore):
|
||||
"""Test write_documents_async() fails when providing unexpected input."""
|
||||
with pytest.raises(ValueError):
|
||||
await document_store.write_documents_async(["not a document for sure"]) # type: ignore
|
||||
with pytest.raises(ValueError):
|
||||
await document_store.write_documents_async("not a list actually") # type: ignore
|
||||
|
||||
|
||||
class DeleteDocumentsAsyncTest:
|
||||
"""
|
||||
Utility class to test a Document Store `delete_documents_async` method.
|
||||
|
||||
To use it create a custom test class and override the `document_store` fixture to return your Document Store.
|
||||
The Document Store `write_documents_async` and `count_documents_async` methods must be implemented for these tests
|
||||
to work correctly.
|
||||
Example usage:
|
||||
|
||||
```python
|
||||
class MyDocumentStoreTest(DeleteDocumentsAsyncTest):
|
||||
@pytest.fixture
|
||||
def document_store(self):
|
||||
return MyDocumentStore()
|
||||
```
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_documents_async(document_store: AsyncDocumentStore):
|
||||
"""Test delete_documents_async() normal behaviour."""
|
||||
doc = Document(content="test doc")
|
||||
await document_store.write_documents_async([doc])
|
||||
assert await document_store.count_documents_async() == 1
|
||||
|
||||
await document_store.delete_documents_async([doc.id])
|
||||
assert await document_store.count_documents_async() == 0
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_documents_empty_document_store_async(document_store: AsyncDocumentStore):
|
||||
"""Test delete_documents_async() doesn't fail when called using an empty Document Store."""
|
||||
await document_store.delete_documents_async(["non_existing_id"])
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_documents_non_existing_document_async(document_store: AsyncDocumentStore):
|
||||
"""Test delete_documents_async() doesn't delete any Document when called with non-existing id."""
|
||||
doc = Document(content="test doc")
|
||||
await document_store.write_documents_async([doc])
|
||||
assert await document_store.count_documents_async() == 1
|
||||
|
||||
await document_store.delete_documents_async(["non_existing_id"])
|
||||
|
||||
# No Document has been deleted
|
||||
assert await document_store.count_documents_async() == 1
|
||||
|
||||
|
||||
class GetMetadataFieldsInfoAsyncTest:
|
||||
"""
|
||||
Tests for Document Store get_metadata_fields_info_async().
|
||||
|
||||
Only mix in for stores that implement get_metadata_fields_info_async.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_metadata_fields_info_async(document_store: AsyncDocumentStore):
|
||||
"""Test get_metadata_fields_info_async() returns field names and types after writing documents."""
|
||||
docs = [
|
||||
Document(content="Doc 1", meta={"category": "A", "status": "active", "priority": 1}),
|
||||
Document(content="Doc 2", meta={"category": "B", "status": "inactive", "rating": 0.5}),
|
||||
]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 2
|
||||
|
||||
fields_info = await document_store.get_metadata_fields_info_async() # type:ignore[attr-defined]
|
||||
|
||||
assert "category" in fields_info
|
||||
assert "status" in fields_info
|
||||
assert "priority" in fields_info
|
||||
assert "rating" in fields_info
|
||||
for info in fields_info.values():
|
||||
assert isinstance(info, dict)
|
||||
assert "type" in info
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_metadata_fields_info_empty_collection_async(document_store: AsyncDocumentStore):
|
||||
"""Test get_metadata_fields_info_async() on an empty store."""
|
||||
assert await document_store.count_documents_async() == 0
|
||||
|
||||
fields_info = await document_store.get_metadata_fields_info_async() # type:ignore[attr-defined]
|
||||
assert fields_info == {}
|
||||
|
||||
|
||||
class GetMetadataFieldMinMaxAsyncTest:
|
||||
"""
|
||||
Tests for Document Store get_metadata_field_min_max_async().
|
||||
|
||||
Only mix in for stores that implement get_metadata_field_min_max_async.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_metadata_field_min_max_numeric_async(document_store: AsyncDocumentStore):
|
||||
"""Test get_metadata_field_min_max_async() with integer field."""
|
||||
docs = [
|
||||
Document(content="Doc 1", meta={"priority": 1}),
|
||||
Document(content="Doc 2", meta={"priority": 5}),
|
||||
Document(content="Doc 3", meta={"priority": 3}),
|
||||
Document(content="Doc 4", meta={"priority": 10}),
|
||||
]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 4
|
||||
|
||||
result = await document_store.get_metadata_field_min_max_async("priority") # type:ignore[attr-defined]
|
||||
assert result["min"] == 1
|
||||
assert result["max"] == 10
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_metadata_field_min_max_float_async(document_store: AsyncDocumentStore):
|
||||
"""Test get_metadata_field_min_max_async() with float field."""
|
||||
docs = [
|
||||
Document(content="Doc 1", meta={"rating": 0.6}),
|
||||
Document(content="Doc 2", meta={"rating": 0.95}),
|
||||
Document(content="Doc 3", meta={"rating": 0.8}),
|
||||
]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 3
|
||||
|
||||
result = await document_store.get_metadata_field_min_max_async("rating") # type:ignore[attr-defined]
|
||||
|
||||
assert result["min"] == pytest.approx(0.6)
|
||||
assert result["max"] == pytest.approx(0.95)
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_metadata_field_min_max_single_value_async(document_store: AsyncDocumentStore):
|
||||
"""Test get_metadata_field_min_max_async() when field has only one value."""
|
||||
docs = [Document(content="Doc 1", meta={"priority": 42})]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 1
|
||||
|
||||
result = await document_store.get_metadata_field_min_max_async("priority") # type:ignore[attr-defined]
|
||||
assert result["min"] == 42
|
||||
assert result["max"] == 42
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_metadata_field_min_max_empty_collection_async(document_store: AsyncDocumentStore):
|
||||
"""Test get_metadata_field_min_max_async() on an empty store."""
|
||||
assert await document_store.count_documents_async() == 0
|
||||
|
||||
result = await document_store.get_metadata_field_min_max_async("priority") # type:ignore[attr-defined]
|
||||
assert result["min"] is None
|
||||
assert result["max"] is None
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_metadata_field_min_max_meta_prefix_async(document_store: AsyncDocumentStore):
|
||||
"""Test get_metadata_field_min_max_async() with field names that include 'meta.' prefix."""
|
||||
docs = [
|
||||
Document(content="Doc 1", meta={"priority": 1, "age": 10}),
|
||||
Document(content="Doc 2", meta={"priority": 5, "age": 20}),
|
||||
Document(content="Doc 3", meta={"priority": 3, "age": 15}),
|
||||
Document(content="Doc 4", meta={"priority": 10, "age": 5}),
|
||||
Document(content="Doc 6", meta={"rating": 10.5}),
|
||||
Document(content="Doc 7", meta={"rating": 20.3}),
|
||||
Document(content="Doc 8", meta={"rating": 15.7}),
|
||||
Document(content="Doc 9", meta={"rating": 5.2}),
|
||||
]
|
||||
await document_store.write_documents_async(docs)
|
||||
|
||||
min_max_priority = await document_store.get_metadata_field_min_max_async("meta.priority") # type:ignore[attr-defined]
|
||||
assert min_max_priority["min"] == 1
|
||||
assert min_max_priority["max"] == 10
|
||||
|
||||
# Test with float values and "meta." prefix
|
||||
min_max_score = await document_store.get_metadata_field_min_max_async("meta.rating") # type:ignore[attr-defined]
|
||||
assert min_max_score["min"] == pytest.approx(5.2)
|
||||
assert min_max_score["max"] == pytest.approx(20.3)
|
||||
|
||||
|
||||
class GetMetadataFieldUniqueValuesAsyncTest:
|
||||
"""
|
||||
Tests for Document Store get_metadata_field_unique_values_async().
|
||||
|
||||
Only mix in for stores that implement get_metadata_field_unique_values_async.
|
||||
Expects the method to return (values_list, total_count) or (values_list, pagination_key).
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_metadata_field_unique_values_basic_async(document_store: AsyncDocumentStore):
|
||||
"""Test get_metadata_field_unique_values_async() returns unique values and total count."""
|
||||
docs = [
|
||||
Document(content="Doc 1", meta={"category": "A"}),
|
||||
Document(content="Doc 2", meta={"category": "B"}),
|
||||
Document(content="Doc 3", meta={"category": "A"}),
|
||||
Document(content="Doc 4", meta={"category": "C"}),
|
||||
Document(content="Doc 5", meta={"category": "B"}),
|
||||
]
|
||||
await document_store.write_documents_async(docs)
|
||||
assert await document_store.count_documents_async() == 5
|
||||
|
||||
sig = inspect.signature(document_store.get_metadata_field_unique_values_async) # type:ignore[attr-defined]
|
||||
params: dict = {}
|
||||
if "search_term" in sig.parameters:
|
||||
params["search_term"] = None
|
||||
if "from_" in sig.parameters:
|
||||
params["from_"] = 0
|
||||
elif "offset" in sig.parameters:
|
||||
params["offset"] = 0
|
||||
if "size" in sig.parameters:
|
||||
params["size"] = 10
|
||||
elif "limit" in sig.parameters:
|
||||
params["limit"] = 10
|
||||
|
||||
result = await document_store.get_metadata_field_unique_values_async("category", **params) # type:ignore[attr-defined]
|
||||
|
||||
values = result[0] if isinstance(result, tuple) else result
|
||||
assert isinstance(values, list)
|
||||
assert set(values) == {"A", "B", "C"}
|
||||
if isinstance(result, tuple) and len(result) >= 2 and isinstance(result[1], int):
|
||||
assert result[1] == 3
|
||||
|
||||
|
||||
class FilterDocumentsAsyncTest(AssertDocumentsEqualMixin, FilterableDocsFixtureMixin):
|
||||
"""
|
||||
Smoke tests for the async filter_documents_async() path.
|
||||
|
||||
These tests verify that the async plumbing works correctly with no filters,
|
||||
a simple equality filter, and a compound AND filter. Full filter logic correctness
|
||||
is covered by FilterDocumentsTest — the sync and async paths share the same
|
||||
filter translation layer, so only the async dispatch needs smoke-testing here.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_filters_async(document_store: AsyncDocumentStore):
|
||||
"""Verify the async path returns all documents when no filter is applied."""
|
||||
docs = [Document(content="first doc"), Document(content="second doc"), Document(content="third doc")]
|
||||
await document_store.write_documents_async(docs)
|
||||
result = await document_store.filter_documents_async()
|
||||
assert len(result) == 3
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_filter_simple_async(self, document_store: AsyncDocumentStore, filterable_docs: list[Document]):
|
||||
"""One equality filter — confirms async plumbing works with a filter."""
|
||||
await document_store.write_documents_async(filterable_docs)
|
||||
result = await document_store.filter_documents_async(
|
||||
filters={"field": "meta.number", "operator": "==", "value": 2}
|
||||
)
|
||||
self.assert_documents_are_equal(result, [d for d in filterable_docs if d.meta.get("number") == 2])
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_filter_compound_async(self, document_store: AsyncDocumentStore, filterable_docs: list[Document]):
|
||||
"""One AND filter — verifies compound filters aren't broken by the async path."""
|
||||
await document_store.write_documents_async(filterable_docs)
|
||||
result = await document_store.filter_documents_async(
|
||||
filters={
|
||||
"operator": "AND",
|
||||
"conditions": [
|
||||
{"field": "meta.number", "operator": "==", "value": 2},
|
||||
{"field": "meta.name", "operator": "==", "value": "name_0"},
|
||||
],
|
||||
}
|
||||
)
|
||||
self.assert_documents_are_equal(
|
||||
result, [d for d in filterable_docs if d.meta.get("number") == 2 and d.meta.get("name") == "name_0"]
|
||||
)
|
||||
@@ -0,0 +1,232 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any
|
||||
|
||||
from haystack.core.component import Component, component
|
||||
from haystack.core.serialization import default_from_dict, default_to_dict
|
||||
from haystack.dataclasses import Document
|
||||
from haystack.document_stores.types import DocumentStore, DuplicatePolicy
|
||||
|
||||
|
||||
def document_store_class(
|
||||
name: str,
|
||||
documents: list[Document] | None = None,
|
||||
documents_count: int | None = None,
|
||||
bases: tuple[type, ...] | None = None,
|
||||
extra_fields: dict[str, Any] | None = None,
|
||||
) -> type[DocumentStore]:
|
||||
"""
|
||||
Utility function to create a DocumentStore class with the given name and list of documents.
|
||||
|
||||
If `documents` is set but `documents_count` is not, `documents_count` will be the length
|
||||
of `documents`.
|
||||
If both are set explicitly they don't influence each other.
|
||||
|
||||
`write_documents()` and `delete_documents()` are no-op.
|
||||
You can override them using `extra_fields`.
|
||||
|
||||
### Usage
|
||||
|
||||
Create a DocumentStore class that returns no documents:
|
||||
```python
|
||||
MyFakeStore = document_store_class("MyFakeComponent")
|
||||
document_store = MyFakeStore()
|
||||
assert document_store.documents_count() == 0
|
||||
assert document_store.filter_documents() == []
|
||||
```
|
||||
|
||||
Create a DocumentStore class that returns a single document:
|
||||
```python
|
||||
doc = Document(id="fake_id", content="Fake content")
|
||||
MyFakeStore = document_store_class("MyFakeComponent", documents=[doc])
|
||||
document_store = MyFakeStore()
|
||||
assert document_store.documents_count() == 1
|
||||
assert document_store.filter_documents() == [doc]
|
||||
```
|
||||
|
||||
Create a DocumentStore class that returns no document but returns a custom count:
|
||||
```python
|
||||
MyFakeStore = document_store_class("MyFakeComponent", documents_count=100)
|
||||
document_store = MyFakeStore()
|
||||
assert document_store.documents_count() == 100
|
||||
assert document_store.filter_documents() == []
|
||||
```
|
||||
|
||||
Create a DocumentStore class that returns a document and a custom count:
|
||||
```python
|
||||
doc = Document(id="fake_id", content="Fake content")
|
||||
MyFakeStore = document_store_class("MyFakeComponent", documents=[doc], documents_count=100)
|
||||
document_store = MyFakeStore()
|
||||
assert document_store.documents_count() == 100
|
||||
assert document_store.filter_documents() == [doc]
|
||||
```
|
||||
|
||||
Create a DocumentStore class with a custom base class:
|
||||
```python
|
||||
MyFakeStore = document_store_class(
|
||||
"MyFakeStore",
|
||||
bases=(MyBaseClass,)
|
||||
)
|
||||
document_store = MyFakeStore()
|
||||
assert isinstance(store, MyBaseClass)
|
||||
```
|
||||
|
||||
Create a DocumentStore class with an extra field `my_field`:
|
||||
```python
|
||||
MyFakeStore = document_store_class(
|
||||
"MyFakeStore",
|
||||
extra_fields={"my_field": 10}
|
||||
)
|
||||
document_store = MyFakeStore()
|
||||
assert document_store.my_field == 10
|
||||
```
|
||||
"""
|
||||
if documents is not None and documents_count is None:
|
||||
documents_count = len(documents)
|
||||
elif documents_count is None:
|
||||
documents_count = 0
|
||||
|
||||
def count_documents(self) -> int | None: # noqa: ARG001
|
||||
return documents_count
|
||||
|
||||
def filter_documents(self, filters: dict[str, Any] | None = None) -> list[Document]: # noqa: ARG001
|
||||
if documents is not None:
|
||||
return documents
|
||||
return []
|
||||
|
||||
def write_documents(self, documents: list[Document], policy: DuplicatePolicy = DuplicatePolicy.FAIL) -> None: # noqa: ARG001
|
||||
return
|
||||
|
||||
def delete_documents(self, document_ids: list[str]) -> None: # noqa: ARG001
|
||||
return
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return default_to_dict(self)
|
||||
|
||||
fields = {
|
||||
"count_documents": count_documents,
|
||||
"filter_documents": filter_documents,
|
||||
"write_documents": write_documents,
|
||||
"delete_documents": delete_documents,
|
||||
"to_dict": to_dict,
|
||||
"from_dict": classmethod(default_from_dict),
|
||||
}
|
||||
|
||||
if extra_fields is not None:
|
||||
fields = {**fields, **extra_fields}
|
||||
|
||||
if bases is None:
|
||||
bases = (object,)
|
||||
|
||||
return type(name, bases, fields)
|
||||
|
||||
|
||||
def component_class(
|
||||
name: str,
|
||||
input_types: dict[str, Any] | None = None,
|
||||
output_types: dict[str, Any] | None = None,
|
||||
output: dict[str, Any] | None = None,
|
||||
bases: tuple[type, ...] | None = None,
|
||||
extra_fields: dict[str, Any] | None = None,
|
||||
) -> type[Component]:
|
||||
"""
|
||||
Utility class to create a Component class with the given name and input and output types.
|
||||
|
||||
If `output` is set but `output_types` is not, `output_types` will be set to the types of the values in `output`.
|
||||
Though if `output_types` is set but `output` is not the component's `run` method will return a dictionary
|
||||
of the same keys as `output_types` all with a value of None.
|
||||
|
||||
### Usage
|
||||
|
||||
Create a component class with default input and output types:
|
||||
```python
|
||||
MyFakeComponent = component_class_factory("MyFakeComponent")
|
||||
component = MyFakeComponent()
|
||||
output = component.run(value=1)
|
||||
assert output == {"value": None}
|
||||
```
|
||||
|
||||
Create a component class with an "value" input of type `int` and with a "value" output of `10`:
|
||||
```python
|
||||
MyFakeComponent = component_class_factory(
|
||||
"MyFakeComponent",
|
||||
input_types={"value": int},
|
||||
output={"value": 10}
|
||||
)
|
||||
component = MyFakeComponent()
|
||||
output = component.run(value=1)
|
||||
assert output == {"value": 10}
|
||||
```
|
||||
|
||||
Create a component class with a custom base class:
|
||||
```python
|
||||
MyFakeComponent = component_class_factory(
|
||||
"MyFakeComponent",
|
||||
bases=(MyBaseClass,)
|
||||
)
|
||||
component = MyFakeComponent()
|
||||
assert isinstance(component, MyBaseClass)
|
||||
```
|
||||
|
||||
Create a component class with an extra field `my_field`:
|
||||
```python
|
||||
MyFakeComponent = component_class_factory(
|
||||
"MyFakeComponent",
|
||||
extra_fields={"my_field": 10}
|
||||
)
|
||||
component = MyFakeComponent()
|
||||
assert component.my_field == 10
|
||||
```
|
||||
|
||||
Args:
|
||||
name: Name of the component class
|
||||
input_types: Dictionary of string and type that defines the inputs of the component,
|
||||
if set to None created component will expect a single input "value" of Any type.
|
||||
Defaults to None.
|
||||
output_types: Dictionary of string and type that defines the outputs of the component,
|
||||
if set to None created component will return a single output "value" of NoneType and None value.
|
||||
Defaults to None.
|
||||
output: Actual output dictionary returned by the created component run,
|
||||
is set to None it will return a dictionary of string and None values.
|
||||
Keys will be the same as the keys of output_types. Defaults to None.
|
||||
bases: Base classes for this component, if set to None only base is object. Defaults to None.
|
||||
extra_fields: Extra fields for the Component, defaults to None.
|
||||
|
||||
:return: A class definition that can be used as a component.
|
||||
"""
|
||||
if input_types is None:
|
||||
input_types = {"value": Any}
|
||||
if output_types is None and output is not None:
|
||||
output_types = {key: type(value) for key, value in output.items()}
|
||||
elif output_types is None:
|
||||
output_types = {"value": type(None)}
|
||||
|
||||
def init(self):
|
||||
component.set_input_types(self, **input_types)
|
||||
component.set_output_types(self, **output_types)
|
||||
|
||||
# Both arguments are necessary to correctly define
|
||||
# run but ruff doesn't like that we don't use them.
|
||||
# It's fine ignoring the warning here.
|
||||
def run(self, **kwargs): # noqa: ARG001
|
||||
if output is not None:
|
||||
return output
|
||||
return dict.fromkeys(output_types.keys())
|
||||
|
||||
def to_dict(self):
|
||||
return default_to_dict(self)
|
||||
|
||||
def from_dict(cls, data: dict[str, Any]):
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
fields = {"__init__": init, "run": run, "to_dict": to_dict, "from_dict": classmethod(from_dict)}
|
||||
if extra_fields is not None:
|
||||
fields = {**fields, **extra_fields}
|
||||
|
||||
if bases is None:
|
||||
bases = (object,)
|
||||
|
||||
cls = type(name, bases, fields)
|
||||
return component(cls)
|
||||
@@ -0,0 +1,40 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from haystack.testing.sample_components.accumulate import Accumulate
|
||||
from haystack.testing.sample_components.add_value import AddFixedValue
|
||||
from haystack.testing.sample_components.concatenate import Concatenate
|
||||
from haystack.testing.sample_components.double import Double
|
||||
from haystack.testing.sample_components.fstring import FString
|
||||
from haystack.testing.sample_components.future_annotations import HelloUsingFutureAnnotations
|
||||
from haystack.testing.sample_components.greet import Greet
|
||||
from haystack.testing.sample_components.hello import Hello
|
||||
from haystack.testing.sample_components.joiner import StringJoiner, StringListJoiner
|
||||
from haystack.testing.sample_components.parity import Parity
|
||||
from haystack.testing.sample_components.remainder import Remainder
|
||||
from haystack.testing.sample_components.repeat import Repeat
|
||||
from haystack.testing.sample_components.subtract import Subtract
|
||||
from haystack.testing.sample_components.sum import Sum
|
||||
from haystack.testing.sample_components.text_splitter import TextSplitter
|
||||
from haystack.testing.sample_components.threshold import Threshold
|
||||
|
||||
__all__ = [
|
||||
"Concatenate",
|
||||
"Subtract",
|
||||
"Parity",
|
||||
"Remainder",
|
||||
"Accumulate",
|
||||
"Threshold",
|
||||
"AddFixedValue",
|
||||
"Repeat",
|
||||
"Sum",
|
||||
"Greet",
|
||||
"Double",
|
||||
"StringJoiner",
|
||||
"Hello",
|
||||
"HelloUsingFutureAnnotations",
|
||||
"TextSplitter",
|
||||
"StringListJoiner",
|
||||
"FString",
|
||||
]
|
||||
@@ -0,0 +1,70 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from haystack.core.component import component
|
||||
from haystack.core.errors import ComponentDeserializationError
|
||||
from haystack.core.serialization import default_to_dict
|
||||
from haystack.utils.callable_serialization import deserialize_callable, serialize_callable
|
||||
|
||||
|
||||
def _default_function(first: int, second: int) -> int:
|
||||
return first + second
|
||||
|
||||
|
||||
@component
|
||||
class Accumulate:
|
||||
"""
|
||||
Accumulates the value flowing through the connection into an internal attribute.
|
||||
|
||||
The sum function can be customized. Example of how to deal with serialization when some of the parameters
|
||||
are not directly serializable.
|
||||
"""
|
||||
|
||||
def __init__(self, function: Callable | None = None) -> None:
|
||||
"""
|
||||
Class constructor
|
||||
|
||||
:param function:
|
||||
the function to use to accumulate the values.
|
||||
The function must take exactly two values.
|
||||
If it's a callable, it's used as it is.
|
||||
If it's a string, the component will look for it in sys.modules and
|
||||
import it at need. This is also a parameter.
|
||||
"""
|
||||
self.state = 0
|
||||
self.function: Callable = _default_function if function is None else function
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
"""Converts the component to a dictionary"""
|
||||
return default_to_dict(self, function=serialize_callable(self.function))
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "Accumulate":
|
||||
"""Loads the component from a dictionary"""
|
||||
if "type" not in data:
|
||||
raise ComponentDeserializationError("Missing 'type' in component serialization data")
|
||||
if data["type"] != f"{cls.__module__}.{cls.__name__}":
|
||||
raise ComponentDeserializationError(f"Class '{data['type']}' can't be deserialized as '{cls.__name__}'")
|
||||
|
||||
init_params = data.get("init_parameters", {})
|
||||
|
||||
# Resolve the function through `deserialize_callable` so it passes the deserialization
|
||||
# allowlist instead of importing arbitrary handles directly.
|
||||
function = init_params.get("function")
|
||||
accumulator_function = deserialize_callable(function) if function else None
|
||||
|
||||
return cls(function=accumulator_function)
|
||||
|
||||
@component.output_types(value=int)
|
||||
def run(self, value: int):
|
||||
"""
|
||||
Accumulates the value flowing through the connection into an internal attribute.
|
||||
|
||||
The sum function can be customized.
|
||||
"""
|
||||
self.state = self.function(self.state, value)
|
||||
return {"value": self.state}
|
||||
@@ -0,0 +1,24 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from haystack.core.component import component
|
||||
|
||||
|
||||
@component
|
||||
class AddFixedValue:
|
||||
"""
|
||||
Adds two values together.
|
||||
"""
|
||||
|
||||
def __init__(self, add: int = 1) -> None:
|
||||
self.add = add
|
||||
|
||||
@component.output_types(result=int)
|
||||
def run(self, value: int, add: int | None = None):
|
||||
"""
|
||||
Adds two values together.
|
||||
"""
|
||||
if add is None:
|
||||
add = self.add
|
||||
return {"result": value + add}
|
||||
@@ -0,0 +1,29 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from haystack.core.component import component
|
||||
|
||||
|
||||
@component
|
||||
class Concatenate:
|
||||
"""
|
||||
Concatenates two values
|
||||
"""
|
||||
|
||||
@component.output_types(value=list[str])
|
||||
def run(self, first: list[str] | str, second: list[str] | str):
|
||||
"""
|
||||
Concatenates two values
|
||||
"""
|
||||
if isinstance(first, str) and isinstance(second, str):
|
||||
res = [first, second]
|
||||
elif isinstance(first, list) and isinstance(second, list):
|
||||
res = first + second
|
||||
elif isinstance(first, list) and isinstance(second, str):
|
||||
res = first + [second]
|
||||
elif isinstance(first, str) and isinstance(second, list):
|
||||
res = [first] + second
|
||||
else:
|
||||
res = None
|
||||
return {"value": res}
|
||||
@@ -0,0 +1,19 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from haystack.core.component import component
|
||||
|
||||
|
||||
@component
|
||||
class Double:
|
||||
"""
|
||||
Doubles the input value.
|
||||
"""
|
||||
|
||||
@component.output_types(value=int)
|
||||
def run(self, value: int):
|
||||
"""
|
||||
Doubles the input value.
|
||||
"""
|
||||
return {"value": value * 2}
|
||||
@@ -0,0 +1,32 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from typing import Any
|
||||
|
||||
from haystack.core.component import component
|
||||
|
||||
|
||||
@component
|
||||
class FString:
|
||||
"""
|
||||
Takes a template string and a list of variables in input and returns the formatted string in output.
|
||||
"""
|
||||
|
||||
def __init__(self, template: str, variables: list[str] | None = None) -> None:
|
||||
self.template = template
|
||||
self.variables = variables or []
|
||||
if "template" in self.variables:
|
||||
raise ValueError("The variable name 'template' is reserved and cannot be used.")
|
||||
component.set_input_types(self, **dict.fromkeys(self.variables, Any))
|
||||
|
||||
@component.output_types(string=str)
|
||||
def run(self, template: str | None = None, **kwargs):
|
||||
"""
|
||||
Takes a template string and a list of variables in input and returns the formatted string in output.
|
||||
|
||||
If the template is not given, the component will use the one given at initialization.
|
||||
"""
|
||||
if not template:
|
||||
template = self.template
|
||||
return {"string": template.format(**kwargs)}
|
||||
@@ -0,0 +1,15 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from haystack import component
|
||||
|
||||
|
||||
@component
|
||||
class HelloUsingFutureAnnotations:
|
||||
@component.output_types(output=str)
|
||||
def run(self, word: str) -> dict[str, str]:
|
||||
"""Takes a string in input and returns "Hello, <string>!"in output."""
|
||||
return {"output": f"Hello, {word}!"}
|
||||
@@ -0,0 +1,48 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import logging
|
||||
|
||||
import haystack.logging as haystack_logging
|
||||
from haystack.core.component import component
|
||||
|
||||
logger = haystack_logging.getLogger(__name__)
|
||||
|
||||
|
||||
@component
|
||||
class Greet:
|
||||
"""
|
||||
Logs a greeting message without affecting the value passing on the connection.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, message: str = "\nGreeting component says: Hi! The value is {value}\n", log_level: str = "INFO"
|
||||
) -> None:
|
||||
"""
|
||||
Class constructor
|
||||
|
||||
:param message: the message to log. Can use `{value}` to embed the value.
|
||||
:param log_level: the level to log at.
|
||||
"""
|
||||
if log_level and not getattr(logging, log_level):
|
||||
raise ValueError(f"This log level does not exist: {log_level}")
|
||||
self.message = message
|
||||
self.log_level = log_level
|
||||
|
||||
@component.output_types(value=int)
|
||||
def run(self, value: int, message: str | None = None, log_level: str | None = None):
|
||||
"""
|
||||
Logs a greeting message without affecting the value passing on the connection.
|
||||
"""
|
||||
if not message:
|
||||
message = self.message
|
||||
if not log_level:
|
||||
log_level = self.log_level
|
||||
|
||||
level = getattr(logging, log_level, None)
|
||||
if not level:
|
||||
raise ValueError(f"This log level does not exist: {log_level}")
|
||||
|
||||
logger.log(level=level, msg=message.format(value=value))
|
||||
return {"value": value}
|
||||
@@ -0,0 +1,13 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from haystack.core.component import component
|
||||
|
||||
|
||||
@component
|
||||
class Hello:
|
||||
@component.output_types(output=str)
|
||||
def run(self, word: str):
|
||||
"""Takes a string in input and returns "Hello, <string>!"in output."""
|
||||
return {"output": f"Hello, {word}!"}
|
||||
@@ -0,0 +1,35 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
|
||||
from haystack.core.component import component
|
||||
from haystack.core.component.types import Variadic
|
||||
|
||||
|
||||
@component
|
||||
class StringJoiner:
|
||||
@component.output_types(output=str)
|
||||
def run(self, input_str: Variadic[str]):
|
||||
"""
|
||||
Take strings from multiple input nodes and join them into a single one returned in output.
|
||||
|
||||
Since `input_str` is Variadic, we know we'll receive a list[str].
|
||||
"""
|
||||
return {"output": " ".join(input_str)}
|
||||
|
||||
|
||||
@component
|
||||
class StringListJoiner:
|
||||
@component.output_types(output=str)
|
||||
def run(self, inputs: Variadic[list[str]]):
|
||||
"""
|
||||
Take list of strings from multiple input nodes and join them into a single one returned in output.
|
||||
|
||||
Since `input_str` is Variadic, we know we'll receive a list[list[str]].
|
||||
"""
|
||||
retval: list[str] = []
|
||||
for list_of_strings in inputs:
|
||||
retval += list_of_strings
|
||||
|
||||
return {"output": retval}
|
||||
@@ -0,0 +1,22 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from haystack.core.component import component
|
||||
|
||||
|
||||
@component
|
||||
class Parity:
|
||||
"""
|
||||
Redirects the value, unchanged, along the 'even' connection if even, or along the 'odd' one if odd.
|
||||
"""
|
||||
|
||||
@component.output_types(even=int, odd=int)
|
||||
def run(self, value: int):
|
||||
"""
|
||||
:param value: The value to check for parity
|
||||
"""
|
||||
remainder = value % 2
|
||||
if remainder:
|
||||
return {"odd": value}
|
||||
return {"even": value}
|
||||
@@ -0,0 +1,21 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from haystack.core.component import component
|
||||
|
||||
|
||||
@component
|
||||
class Remainder:
|
||||
def __init__(self, divisor: int = 3) -> None:
|
||||
if divisor == 0:
|
||||
raise ValueError("Can't divide by zero")
|
||||
self.divisor = divisor
|
||||
component.set_output_types(self, **{f"remainder_is_{val}": int for val in range(divisor)})
|
||||
|
||||
def run(self, value: int):
|
||||
"""
|
||||
:param value: the value to check the remainder of.
|
||||
"""
|
||||
remainder = value % self.divisor
|
||||
return {f"remainder_is_{remainder}": value}
|
||||
@@ -0,0 +1,19 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
|
||||
from haystack.core.component import component
|
||||
|
||||
|
||||
@component
|
||||
class Repeat:
|
||||
def __init__(self, outputs: list[str]) -> None:
|
||||
self._outputs = outputs
|
||||
component.set_output_types(self, **dict.fromkeys(outputs, int))
|
||||
|
||||
def run(self, value: int):
|
||||
"""
|
||||
:param value: the value to repeat.
|
||||
"""
|
||||
return dict.fromkeys(self._outputs, value)
|
||||
@@ -0,0 +1,22 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from haystack.core.component import component
|
||||
|
||||
|
||||
@component
|
||||
class Subtract:
|
||||
"""
|
||||
Compute the difference between two values.
|
||||
"""
|
||||
|
||||
@component.output_types(difference=int)
|
||||
def run(self, first_value: int, second_value: int):
|
||||
"""
|
||||
Run the component.
|
||||
|
||||
:param first_value: name of the connection carrying the value to subtract from.
|
||||
:param second_value: name of the connection carrying the value to subtract.
|
||||
"""
|
||||
return {"difference": first_value - second_value}
|
||||
@@ -0,0 +1,16 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from haystack.core.component import component
|
||||
from haystack.core.component.types import Variadic
|
||||
|
||||
|
||||
@component
|
||||
class Sum:
|
||||
@component.output_types(total=int)
|
||||
def run(self, values: Variadic[int]):
|
||||
"""
|
||||
:param value: the values to sum.
|
||||
"""
|
||||
return {"total": sum(values)}
|
||||
@@ -0,0 +1,14 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
|
||||
from haystack.core.component import component
|
||||
|
||||
|
||||
@component
|
||||
class TextSplitter:
|
||||
@component.output_types(output=list[str])
|
||||
def run(self, sentence: str):
|
||||
"""Takes a sentence in input and returns its words in output."""
|
||||
return {"output": sentence.split()}
|
||||
@@ -0,0 +1,34 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from haystack.core.component import component
|
||||
|
||||
|
||||
@component
|
||||
class Threshold:
|
||||
"""
|
||||
Redirects the value, along a different connection whether the value is above or below the given threshold.
|
||||
|
||||
:param threshold: the number to compare the input value against. This is also a parameter.
|
||||
"""
|
||||
|
||||
def __init__(self, threshold: int = 10) -> None:
|
||||
"""
|
||||
:param threshold: the number to compare the input value against.
|
||||
"""
|
||||
self.threshold = threshold
|
||||
|
||||
@component.output_types(above=int, below=int)
|
||||
def run(self, value: int, threshold: int | None = None):
|
||||
"""
|
||||
Redirects the value, along a different connection whether the value is above or below the given threshold.
|
||||
|
||||
:param threshold: the number to compare the input value against. This is also a parameter.
|
||||
"""
|
||||
if threshold is None:
|
||||
threshold = self.threshold
|
||||
|
||||
if value < threshold:
|
||||
return {"below": value}
|
||||
return {"above": value}
|
||||
@@ -0,0 +1,36 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
import random
|
||||
|
||||
from haystack import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def set_all_seeds(seed: int, deterministic_cudnn: bool = False) -> None:
|
||||
"""
|
||||
Setting multiple seeds to make runs reproducible.
|
||||
|
||||
Important: Enabling `deterministic_cudnn` gives you full reproducibility with CUDA,
|
||||
but might slow down your training (see https://pytorch.org/docs/stable/notes/randomness.html#cudnn) !
|
||||
|
||||
:param seed:number to use as seed
|
||||
:param deterministic_cudnn: Enable for full reproducibility when using CUDA. Caution: might slow down training.
|
||||
"""
|
||||
random.seed(seed)
|
||||
os.environ["PYTHONHASHSEED"] = str(seed)
|
||||
|
||||
try:
|
||||
import torch
|
||||
|
||||
torch.manual_seed(seed)
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
if deterministic_cudnn:
|
||||
torch.backends.cudnn.deterministic = True
|
||||
torch.backends.cudnn.benchmark = False
|
||||
|
||||
except (ImportError, ModuleNotFoundError) as exc:
|
||||
logger.info("Could not set PyTorch seed because torch is not installed. Exception: {exception}", exception=exc)
|
||||
Reference in New Issue
Block a user