chore: import upstream snapshot with attribution
Sync docs with Docusaurus / sync (push) Waiting to run
Tests / Check if changed (push) Waiting to run
Tests / format (push) Blocked by required conditions
Tests / check-imports (push) Blocked by required conditions
Tests / Unit / macos-latest (push) Blocked by required conditions
Tests / Unit / ubuntu-latest (push) Blocked by required conditions
Tests / Unit / windows-latest (push) Blocked by required conditions
Tests / mypy (push) Blocked by required conditions
Tests / Integration / ubuntu-latest (push) Blocked by required conditions
Tests / Integration / macos-latest (push) Blocked by required conditions
Tests / Integration / windows-latest (push) Blocked by required conditions
Tests / notify-slack-on-failure (push) Blocked by required conditions
Tests / Mark tests as completed (push) Blocked by required conditions
Docker image release / Build base image (push) Waiting to run
CodeQL / Analyze (python) (push) Has been cancelled
Update Platform Components Table / update (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:22:28 +08:00
commit c56bef871b
9296 changed files with 1854228 additions and 0 deletions
+3
View File
@@ -0,0 +1,3 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
@@ -0,0 +1,10 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
def callable_to_deserialize(hello: str) -> str:
"""
A function to test callable deserialization.
"""
return f"{hello}, world!"
File diff suppressed because it is too large Load Diff
+788
View File
@@ -0,0 +1,788 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
import inspect
from typing import Any, Protocol
import pytest
from haystack.dataclasses import Document
from haystack.document_stores.errors import DuplicateDocumentError
from haystack.document_stores.types import DocumentStore, DuplicatePolicy
from haystack.testing.document_store import AssertDocumentsEqualMixin, FilterableDocsFixtureMixin
class AsyncDocumentStore(DocumentStore, Protocol):
async def count_documents_async(self) -> int:
"""
Returns the number of documents stored.
"""
...
async def filter_documents_async(self, filters: dict[str, Any] | None = None) -> list[Document]:
"""
Returns the documents that match the filters provided.
"""
...
async def write_documents_async(
self, documents: list[Document], policy: DuplicatePolicy = DuplicatePolicy.NONE
) -> int:
"""
Writes Documents into the DocumentStore.
"""
...
async def delete_documents_async(self, document_ids: list[str]) -> None:
"""
Deletes all documents with matching document_ids from the DocumentStore.
"""
...
class DeleteAllAsyncTest:
"""
Tests for Document Store delete_all_documents_async().
To use it create a custom test class and override the `document_store` fixture.
Only mix in for stores that implement delete_all_documents_async.
"""
@staticmethod
def _delete_all_async_supports_recreate(document_store: AsyncDocumentStore) -> str | None:
"""
Return the recreate parameter name if delete_all_documents_async supports it, else None.
"""
sig = inspect.signature(document_store.delete_all_documents_async) # type:ignore[attr-defined]
if "recreate_index" in sig.parameters:
return "recreate_index"
if "recreate_collection" in sig.parameters:
return "recreate_collection"
return None
@staticmethod
@pytest.mark.asyncio
async def test_delete_all_documents_async(document_store: AsyncDocumentStore):
"""
Test delete_all_documents_async() normal behaviour.
This test verifies that delete_all_documents_async() removes all documents from the store
and that the store remains functional after deletion.
"""
docs = [Document(content="first doc", id="1"), Document(content="second doc", id="2")]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 2
await document_store.delete_all_documents_async() # type:ignore[attr-defined]
assert await document_store.count_documents_async() == 0
new_doc = Document(content="new doc after delete all", id="3")
await document_store.write_documents_async([new_doc])
assert await document_store.count_documents_async() == 1
@staticmethod
@pytest.mark.asyncio
async def test_delete_all_documents_empty_store_async(document_store: AsyncDocumentStore):
"""
Test delete_all_documents_async() on an empty store.
This should not raise an error and should leave the store empty.
"""
assert await document_store.count_documents_async() == 0
await document_store.delete_all_documents_async() # type:ignore[attr-defined]
@staticmethod
@pytest.mark.asyncio
async def test_delete_all_documents_without_recreate_index_async(document_store: AsyncDocumentStore):
"""
Test delete_all_documents_async() with recreate_index/recreate_collection=False when supported.
Skipped if the store's delete_all_documents_async does not have recreate_index or recreate_collection.
"""
param_name = DeleteAllAsyncTest._delete_all_async_supports_recreate(document_store)
if param_name is None:
pytest.skip("delete_all_documents_async has no recreate_index or recreate_collection parameter")
docs = [Document(id="1", content="A first document"), Document(id="2", content="Second document")]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 2
await document_store.delete_all_documents_async(**{param_name: False}) # type:ignore[attr-defined]
assert await document_store.count_documents_async() == 0
new_doc = Document(id="3", content="New document after delete all")
await document_store.write_documents_async([new_doc])
assert await document_store.count_documents_async() == 1
@staticmethod
@pytest.mark.asyncio
async def test_delete_all_documents_with_recreate_index_async(document_store: AsyncDocumentStore):
"""
Test delete_all_documents_async() with recreate_index/recreate_collection=True when supported.
Skipped if the store's delete_all_documents_async does not have recreate_index or recreate_collection.
"""
param_name = DeleteAllAsyncTest._delete_all_async_supports_recreate(document_store)
if param_name is None:
pytest.skip("delete_all_documents_async has no recreate_index or recreate_collection parameter")
docs = [Document(id="1", content="A first document"), Document(id="2", content="Second document")]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 2
await document_store.delete_all_documents_async(**{param_name: True}) # type:ignore[attr-defined]
assert await document_store.count_documents_async() == 0
new_doc = Document(id="3", content="New document after delete all with recreate")
await document_store.write_documents_async([new_doc])
assert await document_store.count_documents_async() == 1
retrieved = await document_store.filter_documents_async()
assert len(retrieved) == 1
assert retrieved[0].content == "New document after delete all with recreate"
class CountDocumentsAsyncTest:
"""
Utility class to test a Document Store `count_documents_async` method.
To use it create a custom test class and override the `document_store` fixture to return your Document Store.
Example usage:
```python
class MyDocumentStoreTest(CountDocumentsAsyncTest):
@pytest.fixture
def document_store(self):
return MyDocumentStore()
```
"""
@staticmethod
@pytest.mark.asyncio
async def test_count_empty_async(document_store: AsyncDocumentStore):
"""Test count is zero for an empty document store."""
assert await document_store.count_documents_async() == 0
@staticmethod
@pytest.mark.asyncio
async def test_count_not_empty_async(document_store: AsyncDocumentStore):
"""Test count is greater than zero if the document store contains documents."""
await document_store.write_documents_async(
[Document(content="test doc 1"), Document(content="test doc 2"), Document(content="test doc 3")]
)
assert await document_store.count_documents_async() == 3
class CountDocumentsByFilterAsyncTest:
"""
Tests for Document Store count_documents_by_filter_async().
Only mix in for stores that implement count_documents_by_filter_async.
"""
@staticmethod
@pytest.mark.asyncio
async def test_count_documents_by_filter_async_simple(document_store: AsyncDocumentStore):
"""Test count_documents_by_filter_async() with a simple equality filter."""
docs = [
Document(content="Doc 1", meta={"category": "A", "status": "active"}),
Document(content="Doc 2", meta={"category": "B", "status": "active"}),
Document(content="Doc 3", meta={"category": "A", "status": "inactive"}),
Document(content="Doc 4", meta={"category": "A", "status": "active"}),
]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 4
count = await document_store.count_documents_by_filter_async( # type:ignore[attr-defined]
filters={"field": "meta.category", "operator": "==", "value": "A"}
)
assert count == 3
count = await document_store.count_documents_by_filter_async( # type:ignore[attr-defined]
filters={"field": "meta.category", "operator": "==", "value": "B"}
)
assert count == 1
@staticmethod
@pytest.mark.asyncio
async def test_count_documents_by_filter_async_compound(document_store: AsyncDocumentStore):
"""Test count_documents_by_filter_async() with AND filter."""
docs = [
Document(content="Doc 1", meta={"category": "A", "status": "active"}),
Document(content="Doc 2", meta={"category": "B", "status": "active"}),
Document(content="Doc 3", meta={"category": "A", "status": "inactive"}),
Document(content="Doc 4", meta={"category": "A", "status": "active"}),
]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 4
count = await document_store.count_documents_by_filter_async( # type:ignore[attr-defined]
filters={
"operator": "AND",
"conditions": [
{"field": "meta.category", "operator": "==", "value": "A"},
{"field": "meta.status", "operator": "==", "value": "active"},
],
}
)
assert count == 2
@staticmethod
@pytest.mark.asyncio
async def test_count_documents_by_filter_async_no_matches(document_store: AsyncDocumentStore):
"""Test count_documents_by_filter_async() when filter matches no documents."""
docs = [Document(content="Doc 1", meta={"category": "A"}), Document(content="Doc 2", meta={"category": "B"})]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 2
count = await document_store.count_documents_by_filter_async( # type:ignore[attr-defined]
filters={"field": "meta.category", "operator": "==", "value": "Z"}
)
assert count == 0
@staticmethod
@pytest.mark.asyncio
async def test_count_documents_by_filter_async_empty_collection(document_store: AsyncDocumentStore):
"""Test count_documents_by_filter_async() on an empty store."""
assert await document_store.count_documents_async() == 0
count = await document_store.count_documents_by_filter_async( # type:ignore[attr-defined]
filters={"field": "meta.category", "operator": "==", "value": "A"}
)
assert count == 0
class CountUniqueMetadataByFilterAsyncTest:
"""
Tests for Document Store count_unique_metadata_by_filter_async().
Only mix in for stores that implement count_unique_metadata_by_filter_async.
"""
@staticmethod
@pytest.mark.asyncio
async def test_count_unique_metadata_by_filter_async_all_documents(document_store: AsyncDocumentStore):
"""Test count_unique_metadata_by_filter_async() with no filter returns distinct counts for all docs."""
docs = [
Document(content="Doc 1", meta={"category": "A", "status": "active", "priority": 1}),
Document(content="Doc 2", meta={"category": "B", "status": "active", "priority": 2}),
Document(content="Doc 3", meta={"category": "A", "status": "inactive", "priority": 1}),
Document(content="Doc 4", meta={"category": "A", "status": "active", "priority": 3}),
Document(content="Doc 5", meta={"category": "C", "status": "active", "priority": 2}),
]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 5
counts = await document_store.count_unique_metadata_by_filter_async( # type:ignore[attr-defined]
filters={}, metadata_fields=["category", "status", "priority"]
)
assert counts["category"] == 3
assert counts["status"] == 2
assert counts["priority"] == 3
@staticmethod
@pytest.mark.asyncio
async def test_count_unique_metadata_by_filter_async_with_filter(document_store: AsyncDocumentStore):
"""Test count_unique_metadata_by_filter_async() with a filter."""
docs = [
Document(content="Doc 1", meta={"category": "A", "status": "active", "priority": 1}),
Document(content="Doc 2", meta={"category": "B", "status": "active", "priority": 2}),
Document(content="Doc 3", meta={"category": "A", "status": "inactive", "priority": 1}),
Document(content="Doc 4", meta={"category": "A", "status": "active", "priority": 3}),
]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 4
counts = await document_store.count_unique_metadata_by_filter_async( # type:ignore[attr-defined]
filters={"field": "meta.category", "operator": "==", "value": "A"}, metadata_fields=["status", "priority"]
)
assert counts["status"] == 2
assert counts["priority"] == 2
@staticmethod
@pytest.mark.asyncio
async def test_count_unique_metadata_by_filter_async_with_multiple_filters(document_store: AsyncDocumentStore):
"""Test counting unique metadata asynchronously with multiple filters."""
docs = [
Document(content="Doc 1", meta={"category": "A", "year": 2023}),
Document(content="Doc 2", meta={"category": "A", "year": 2024}),
Document(content="Doc 3", meta={"category": "B", "year": 2023}),
Document(content="Doc 4", meta={"category": "B", "year": 2024}),
]
await document_store.write_documents_async(docs)
counts = await document_store.count_unique_metadata_by_filter_async( # type:ignore[attr-defined]
filters={
"operator": "AND",
"conditions": [
{"field": "meta.category", "operator": "==", "value": "B"},
{"field": "meta.year", "operator": "==", "value": 2023},
],
},
metadata_fields=["category", "year"],
)
assert counts == {"category": 1, "year": 1}
class DeleteByFilterAsyncTest:
"""
Tests for Document Store delete_by_filter_async().
"""
@staticmethod
def _delete_by_filter_params(document_store: AsyncDocumentStore) -> dict[str, bool]:
"""
Return optional parameters supported by delete_by_filter_async.
"""
sig = inspect.signature(document_store.delete_by_filter_async) # type:ignore[attr-defined]
return {"refresh": True} if "refresh" in sig.parameters else {}
@staticmethod
@pytest.mark.asyncio
async def test_delete_by_filter_async(document_store: AsyncDocumentStore):
"""Delete documents matching a filter and verify count and remaining docs."""
docs = [
Document(content="Doc 1", meta={"category": "Alpha"}),
Document(content="Doc 2", meta={"category": "Beta"}),
Document(content="Doc 3", meta={"category": "Alpha"}),
]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 3
params = DeleteByFilterAsyncTest._delete_by_filter_params(document_store)
deleted_count = await document_store.delete_by_filter_async( # type:ignore[attr-defined]
filters={"field": "meta.category", "operator": "==", "value": "Alpha"}, **params
)
assert deleted_count == 2
assert await document_store.count_documents_async() == 1
remaining_docs = await document_store.filter_documents_async()
assert len(remaining_docs) == 1
assert remaining_docs[0].meta["category"] == "Beta"
@staticmethod
@pytest.mark.asyncio
async def test_delete_by_filter_no_matches_async(document_store: AsyncDocumentStore):
"""Delete with a filter that matches no documents returns 0 and leaves store unchanged."""
docs = [
Document(content="Doc 1", meta={"category": "Alpha"}),
Document(content="Doc 2", meta={"category": "Beta"}),
]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 2
params = DeleteByFilterAsyncTest._delete_by_filter_params(document_store)
deleted_count = await document_store.delete_by_filter_async( # type:ignore[attr-defined]
filters={"field": "meta.category", "operator": "==", "value": "Gamma"}, **params
)
assert deleted_count == 0
assert await document_store.count_documents_async() == 2
@staticmethod
@pytest.mark.asyncio
async def test_delete_by_filter_advanced_filters_async(document_store: AsyncDocumentStore):
"""Delete with AND/OR filter combinations and verify remaining documents."""
docs = [
Document(content="Doc 1", meta={"category": "Alpha", "year": 2023, "status": "draft"}),
Document(content="Doc 2", meta={"category": "Alpha", "year": 2024, "status": "published"}),
Document(content="Doc 3", meta={"category": "Beta", "year": 2023, "status": "draft"}),
]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 3
params = DeleteByFilterAsyncTest._delete_by_filter_params(document_store)
deleted_count = await document_store.delete_by_filter_async( # type:ignore[attr-defined]
filters={
"operator": "AND",
"conditions": [
{"field": "meta.category", "operator": "==", "value": "Alpha"},
{"field": "meta.year", "operator": "==", "value": 2023},
],
},
**params,
)
assert deleted_count == 1
assert await document_store.count_documents_async() == 2
deleted_count = await document_store.delete_by_filter_async( # type:ignore[attr-defined]
filters={
"operator": "OR",
"conditions": [
{"field": "meta.category", "operator": "==", "value": "Beta"},
{"field": "meta.status", "operator": "==", "value": "published"},
],
},
**params,
)
assert deleted_count == 2
assert await document_store.count_documents_async() == 0
class UpdateByFilterAsyncTest:
"""
Tests for Document Store update_by_filter_async().
Only mix in for stores that implement update_by_filter_async.
"""
@staticmethod
@pytest.mark.asyncio
async def test_update_by_filter_async(document_store: AsyncDocumentStore, filterable_docs: list[Document]):
"""Update documents matching a filter asynchronously and verify count and meta changes."""
await document_store.write_documents_async(filterable_docs)
expected_count = len([d for d in filterable_docs if d.meta.get("chapter") == "intro"])
assert await document_store.count_documents_async() == len(filterable_docs)
sig = inspect.signature(document_store.update_by_filter_async) # type:ignore[attr-defined]
params = {"refresh": True} if "refresh" in sig.parameters else {}
updated_count = await document_store.update_by_filter_async( # type:ignore[attr-defined]
filters={"field": "meta.chapter", "operator": "==", "value": "intro"}, meta={"updated": True}, **params
)
assert updated_count == expected_count
updated_docs = await document_store.filter_documents_async(
filters={"field": "meta.updated", "operator": "==", "value": True}
)
assert len(updated_docs) == expected_count
for doc in updated_docs:
assert doc.meta["chapter"] == "intro"
assert doc.meta["updated"] is True
not_updated_docs = await document_store.filter_documents_async(
filters={"field": "meta.chapter", "operator": "==", "value": "abstract"}
)
for doc in not_updated_docs:
assert doc.meta.get("updated") is not True
class WriteDocumentsAsyncTest(AssertDocumentsEqualMixin):
"""
Utility class to test a Document Store `write_documents_async` method.
To use it create a custom test class and override the `document_store` fixture to return your Document Store.
The Document Store `filter_documents_async` method must be at least partly implemented to return all stored
Documents for these tests to work correctly.
Example usage:
```python
class MyDocumentStoreTest(WriteDocumentsAsyncTest):
@pytest.fixture
def document_store(self):
return MyDocumentStore()
```
"""
@pytest.mark.asyncio
async def test_write_documents_async(self, document_store: AsyncDocumentStore):
"""
Test write_documents_async() default behaviour.
"""
msg = (
"Default write_documents_async() behaviour depends on the Document Store implementation, "
"as we don't enforce a default behaviour when no policy is set. "
"Override this test in your custom test class."
)
raise NotImplementedError(msg)
@pytest.mark.asyncio
async def test_write_documents_duplicate_fail_async(self, document_store: AsyncDocumentStore):
"""Test write_documents_async() fails when writing documents with same id and `DuplicatePolicy.FAIL`."""
doc = Document(content="test doc")
assert await document_store.write_documents_async([doc], policy=DuplicatePolicy.FAIL) == 1
with pytest.raises(DuplicateDocumentError):
await document_store.write_documents_async(documents=[doc], policy=DuplicatePolicy.FAIL)
self.assert_documents_are_equal(await document_store.filter_documents_async(), [doc])
@staticmethod
@pytest.mark.asyncio
async def test_write_documents_duplicate_skip_async(document_store: AsyncDocumentStore):
"""Test write_documents_async() skips writing when using DuplicatePolicy.SKIP."""
doc = Document(content="test doc")
assert await document_store.write_documents_async([doc], policy=DuplicatePolicy.SKIP) == 1
assert await document_store.write_documents_async(documents=[doc], policy=DuplicatePolicy.SKIP) == 0
@pytest.mark.asyncio
async def test_write_documents_duplicate_overwrite_async(self, document_store: AsyncDocumentStore):
"""Test write_documents_async() overwrites when using DuplicatePolicy.OVERWRITE."""
doc1 = Document(id="1", content="test doc 1")
doc2 = Document(id="1", content="test doc 2")
assert await document_store.write_documents_async([doc2], policy=DuplicatePolicy.OVERWRITE) == 1
self.assert_documents_are_equal(await document_store.filter_documents_async(), [doc2])
assert await document_store.write_documents_async(documents=[doc1], policy=DuplicatePolicy.OVERWRITE) == 1
self.assert_documents_are_equal(await document_store.filter_documents_async(), [doc1])
@staticmethod
@pytest.mark.asyncio
async def test_write_documents_invalid_input_async(document_store: AsyncDocumentStore):
"""Test write_documents_async() fails when providing unexpected input."""
with pytest.raises(ValueError):
await document_store.write_documents_async(["not a document for sure"]) # type: ignore
with pytest.raises(ValueError):
await document_store.write_documents_async("not a list actually") # type: ignore
class DeleteDocumentsAsyncTest:
"""
Utility class to test a Document Store `delete_documents_async` method.
To use it create a custom test class and override the `document_store` fixture to return your Document Store.
The Document Store `write_documents_async` and `count_documents_async` methods must be implemented for these tests
to work correctly.
Example usage:
```python
class MyDocumentStoreTest(DeleteDocumentsAsyncTest):
@pytest.fixture
def document_store(self):
return MyDocumentStore()
```
"""
@staticmethod
@pytest.mark.asyncio
async def test_delete_documents_async(document_store: AsyncDocumentStore):
"""Test delete_documents_async() normal behaviour."""
doc = Document(content="test doc")
await document_store.write_documents_async([doc])
assert await document_store.count_documents_async() == 1
await document_store.delete_documents_async([doc.id])
assert await document_store.count_documents_async() == 0
@staticmethod
@pytest.mark.asyncio
async def test_delete_documents_empty_document_store_async(document_store: AsyncDocumentStore):
"""Test delete_documents_async() doesn't fail when called using an empty Document Store."""
await document_store.delete_documents_async(["non_existing_id"])
@staticmethod
@pytest.mark.asyncio
async def test_delete_documents_non_existing_document_async(document_store: AsyncDocumentStore):
"""Test delete_documents_async() doesn't delete any Document when called with non-existing id."""
doc = Document(content="test doc")
await document_store.write_documents_async([doc])
assert await document_store.count_documents_async() == 1
await document_store.delete_documents_async(["non_existing_id"])
# No Document has been deleted
assert await document_store.count_documents_async() == 1
class GetMetadataFieldsInfoAsyncTest:
"""
Tests for Document Store get_metadata_fields_info_async().
Only mix in for stores that implement get_metadata_fields_info_async.
"""
@staticmethod
@pytest.mark.asyncio
async def test_get_metadata_fields_info_async(document_store: AsyncDocumentStore):
"""Test get_metadata_fields_info_async() returns field names and types after writing documents."""
docs = [
Document(content="Doc 1", meta={"category": "A", "status": "active", "priority": 1}),
Document(content="Doc 2", meta={"category": "B", "status": "inactive", "rating": 0.5}),
]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 2
fields_info = await document_store.get_metadata_fields_info_async() # type:ignore[attr-defined]
assert "category" in fields_info
assert "status" in fields_info
assert "priority" in fields_info
assert "rating" in fields_info
for info in fields_info.values():
assert isinstance(info, dict)
assert "type" in info
@staticmethod
@pytest.mark.asyncio
async def test_get_metadata_fields_info_empty_collection_async(document_store: AsyncDocumentStore):
"""Test get_metadata_fields_info_async() on an empty store."""
assert await document_store.count_documents_async() == 0
fields_info = await document_store.get_metadata_fields_info_async() # type:ignore[attr-defined]
assert fields_info == {}
class GetMetadataFieldMinMaxAsyncTest:
"""
Tests for Document Store get_metadata_field_min_max_async().
Only mix in for stores that implement get_metadata_field_min_max_async.
"""
@staticmethod
@pytest.mark.asyncio
async def test_get_metadata_field_min_max_numeric_async(document_store: AsyncDocumentStore):
"""Test get_metadata_field_min_max_async() with integer field."""
docs = [
Document(content="Doc 1", meta={"priority": 1}),
Document(content="Doc 2", meta={"priority": 5}),
Document(content="Doc 3", meta={"priority": 3}),
Document(content="Doc 4", meta={"priority": 10}),
]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 4
result = await document_store.get_metadata_field_min_max_async("priority") # type:ignore[attr-defined]
assert result["min"] == 1
assert result["max"] == 10
@staticmethod
@pytest.mark.asyncio
async def test_get_metadata_field_min_max_float_async(document_store: AsyncDocumentStore):
"""Test get_metadata_field_min_max_async() with float field."""
docs = [
Document(content="Doc 1", meta={"rating": 0.6}),
Document(content="Doc 2", meta={"rating": 0.95}),
Document(content="Doc 3", meta={"rating": 0.8}),
]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 3
result = await document_store.get_metadata_field_min_max_async("rating") # type:ignore[attr-defined]
assert result["min"] == pytest.approx(0.6)
assert result["max"] == pytest.approx(0.95)
@staticmethod
@pytest.mark.asyncio
async def test_get_metadata_field_min_max_single_value_async(document_store: AsyncDocumentStore):
"""Test get_metadata_field_min_max_async() when field has only one value."""
docs = [Document(content="Doc 1", meta={"priority": 42})]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 1
result = await document_store.get_metadata_field_min_max_async("priority") # type:ignore[attr-defined]
assert result["min"] == 42
assert result["max"] == 42
@staticmethod
@pytest.mark.asyncio
async def test_get_metadata_field_min_max_empty_collection_async(document_store: AsyncDocumentStore):
"""Test get_metadata_field_min_max_async() on an empty store."""
assert await document_store.count_documents_async() == 0
result = await document_store.get_metadata_field_min_max_async("priority") # type:ignore[attr-defined]
assert result["min"] is None
assert result["max"] is None
@staticmethod
@pytest.mark.asyncio
async def test_get_metadata_field_min_max_meta_prefix_async(document_store: AsyncDocumentStore):
"""Test get_metadata_field_min_max_async() with field names that include 'meta.' prefix."""
docs = [
Document(content="Doc 1", meta={"priority": 1, "age": 10}),
Document(content="Doc 2", meta={"priority": 5, "age": 20}),
Document(content="Doc 3", meta={"priority": 3, "age": 15}),
Document(content="Doc 4", meta={"priority": 10, "age": 5}),
Document(content="Doc 6", meta={"rating": 10.5}),
Document(content="Doc 7", meta={"rating": 20.3}),
Document(content="Doc 8", meta={"rating": 15.7}),
Document(content="Doc 9", meta={"rating": 5.2}),
]
await document_store.write_documents_async(docs)
min_max_priority = await document_store.get_metadata_field_min_max_async("meta.priority") # type:ignore[attr-defined]
assert min_max_priority["min"] == 1
assert min_max_priority["max"] == 10
# Test with float values and "meta." prefix
min_max_score = await document_store.get_metadata_field_min_max_async("meta.rating") # type:ignore[attr-defined]
assert min_max_score["min"] == pytest.approx(5.2)
assert min_max_score["max"] == pytest.approx(20.3)
class GetMetadataFieldUniqueValuesAsyncTest:
"""
Tests for Document Store get_metadata_field_unique_values_async().
Only mix in for stores that implement get_metadata_field_unique_values_async.
Expects the method to return (values_list, total_count) or (values_list, pagination_key).
"""
@staticmethod
@pytest.mark.asyncio
async def test_get_metadata_field_unique_values_basic_async(document_store: AsyncDocumentStore):
"""Test get_metadata_field_unique_values_async() returns unique values and total count."""
docs = [
Document(content="Doc 1", meta={"category": "A"}),
Document(content="Doc 2", meta={"category": "B"}),
Document(content="Doc 3", meta={"category": "A"}),
Document(content="Doc 4", meta={"category": "C"}),
Document(content="Doc 5", meta={"category": "B"}),
]
await document_store.write_documents_async(docs)
assert await document_store.count_documents_async() == 5
sig = inspect.signature(document_store.get_metadata_field_unique_values_async) # type:ignore[attr-defined]
params: dict = {}
if "search_term" in sig.parameters:
params["search_term"] = None
if "from_" in sig.parameters:
params["from_"] = 0
elif "offset" in sig.parameters:
params["offset"] = 0
if "size" in sig.parameters:
params["size"] = 10
elif "limit" in sig.parameters:
params["limit"] = 10
result = await document_store.get_metadata_field_unique_values_async("category", **params) # type:ignore[attr-defined]
values = result[0] if isinstance(result, tuple) else result
assert isinstance(values, list)
assert set(values) == {"A", "B", "C"}
if isinstance(result, tuple) and len(result) >= 2 and isinstance(result[1], int):
assert result[1] == 3
class FilterDocumentsAsyncTest(AssertDocumentsEqualMixin, FilterableDocsFixtureMixin):
"""
Smoke tests for the async filter_documents_async() path.
These tests verify that the async plumbing works correctly with no filters,
a simple equality filter, and a compound AND filter. Full filter logic correctness
is covered by FilterDocumentsTest — the sync and async paths share the same
filter translation layer, so only the async dispatch needs smoke-testing here.
"""
@staticmethod
@pytest.mark.asyncio
async def test_no_filters_async(document_store: AsyncDocumentStore):
"""Verify the async path returns all documents when no filter is applied."""
docs = [Document(content="first doc"), Document(content="second doc"), Document(content="third doc")]
await document_store.write_documents_async(docs)
result = await document_store.filter_documents_async()
assert len(result) == 3
@pytest.mark.asyncio
async def test_filter_simple_async(self, document_store: AsyncDocumentStore, filterable_docs: list[Document]):
"""One equality filter — confirms async plumbing works with a filter."""
await document_store.write_documents_async(filterable_docs)
result = await document_store.filter_documents_async(
filters={"field": "meta.number", "operator": "==", "value": 2}
)
self.assert_documents_are_equal(result, [d for d in filterable_docs if d.meta.get("number") == 2])
@pytest.mark.asyncio
async def test_filter_compound_async(self, document_store: AsyncDocumentStore, filterable_docs: list[Document]):
"""One AND filter — verifies compound filters aren't broken by the async path."""
await document_store.write_documents_async(filterable_docs)
result = await document_store.filter_documents_async(
filters={
"operator": "AND",
"conditions": [
{"field": "meta.number", "operator": "==", "value": 2},
{"field": "meta.name", "operator": "==", "value": "name_0"},
],
}
)
self.assert_documents_are_equal(
result, [d for d in filterable_docs if d.meta.get("number") == 2 and d.meta.get("name") == "name_0"]
)
+232
View File
@@ -0,0 +1,232 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any
from haystack.core.component import Component, component
from haystack.core.serialization import default_from_dict, default_to_dict
from haystack.dataclasses import Document
from haystack.document_stores.types import DocumentStore, DuplicatePolicy
def document_store_class(
name: str,
documents: list[Document] | None = None,
documents_count: int | None = None,
bases: tuple[type, ...] | None = None,
extra_fields: dict[str, Any] | None = None,
) -> type[DocumentStore]:
"""
Utility function to create a DocumentStore class with the given name and list of documents.
If `documents` is set but `documents_count` is not, `documents_count` will be the length
of `documents`.
If both are set explicitly they don't influence each other.
`write_documents()` and `delete_documents()` are no-op.
You can override them using `extra_fields`.
### Usage
Create a DocumentStore class that returns no documents:
```python
MyFakeStore = document_store_class("MyFakeComponent")
document_store = MyFakeStore()
assert document_store.documents_count() == 0
assert document_store.filter_documents() == []
```
Create a DocumentStore class that returns a single document:
```python
doc = Document(id="fake_id", content="Fake content")
MyFakeStore = document_store_class("MyFakeComponent", documents=[doc])
document_store = MyFakeStore()
assert document_store.documents_count() == 1
assert document_store.filter_documents() == [doc]
```
Create a DocumentStore class that returns no document but returns a custom count:
```python
MyFakeStore = document_store_class("MyFakeComponent", documents_count=100)
document_store = MyFakeStore()
assert document_store.documents_count() == 100
assert document_store.filter_documents() == []
```
Create a DocumentStore class that returns a document and a custom count:
```python
doc = Document(id="fake_id", content="Fake content")
MyFakeStore = document_store_class("MyFakeComponent", documents=[doc], documents_count=100)
document_store = MyFakeStore()
assert document_store.documents_count() == 100
assert document_store.filter_documents() == [doc]
```
Create a DocumentStore class with a custom base class:
```python
MyFakeStore = document_store_class(
"MyFakeStore",
bases=(MyBaseClass,)
)
document_store = MyFakeStore()
assert isinstance(store, MyBaseClass)
```
Create a DocumentStore class with an extra field `my_field`:
```python
MyFakeStore = document_store_class(
"MyFakeStore",
extra_fields={"my_field": 10}
)
document_store = MyFakeStore()
assert document_store.my_field == 10
```
"""
if documents is not None and documents_count is None:
documents_count = len(documents)
elif documents_count is None:
documents_count = 0
def count_documents(self) -> int | None: # noqa: ARG001
return documents_count
def filter_documents(self, filters: dict[str, Any] | None = None) -> list[Document]: # noqa: ARG001
if documents is not None:
return documents
return []
def write_documents(self, documents: list[Document], policy: DuplicatePolicy = DuplicatePolicy.FAIL) -> None: # noqa: ARG001
return
def delete_documents(self, document_ids: list[str]) -> None: # noqa: ARG001
return
def to_dict(self) -> dict[str, Any]:
return default_to_dict(self)
fields = {
"count_documents": count_documents,
"filter_documents": filter_documents,
"write_documents": write_documents,
"delete_documents": delete_documents,
"to_dict": to_dict,
"from_dict": classmethod(default_from_dict),
}
if extra_fields is not None:
fields = {**fields, **extra_fields}
if bases is None:
bases = (object,)
return type(name, bases, fields)
def component_class(
name: str,
input_types: dict[str, Any] | None = None,
output_types: dict[str, Any] | None = None,
output: dict[str, Any] | None = None,
bases: tuple[type, ...] | None = None,
extra_fields: dict[str, Any] | None = None,
) -> type[Component]:
"""
Utility class to create a Component class with the given name and input and output types.
If `output` is set but `output_types` is not, `output_types` will be set to the types of the values in `output`.
Though if `output_types` is set but `output` is not the component's `run` method will return a dictionary
of the same keys as `output_types` all with a value of None.
### Usage
Create a component class with default input and output types:
```python
MyFakeComponent = component_class_factory("MyFakeComponent")
component = MyFakeComponent()
output = component.run(value=1)
assert output == {"value": None}
```
Create a component class with an "value" input of type `int` and with a "value" output of `10`:
```python
MyFakeComponent = component_class_factory(
"MyFakeComponent",
input_types={"value": int},
output={"value": 10}
)
component = MyFakeComponent()
output = component.run(value=1)
assert output == {"value": 10}
```
Create a component class with a custom base class:
```python
MyFakeComponent = component_class_factory(
"MyFakeComponent",
bases=(MyBaseClass,)
)
component = MyFakeComponent()
assert isinstance(component, MyBaseClass)
```
Create a component class with an extra field `my_field`:
```python
MyFakeComponent = component_class_factory(
"MyFakeComponent",
extra_fields={"my_field": 10}
)
component = MyFakeComponent()
assert component.my_field == 10
```
Args:
name: Name of the component class
input_types: Dictionary of string and type that defines the inputs of the component,
if set to None created component will expect a single input "value" of Any type.
Defaults to None.
output_types: Dictionary of string and type that defines the outputs of the component,
if set to None created component will return a single output "value" of NoneType and None value.
Defaults to None.
output: Actual output dictionary returned by the created component run,
is set to None it will return a dictionary of string and None values.
Keys will be the same as the keys of output_types. Defaults to None.
bases: Base classes for this component, if set to None only base is object. Defaults to None.
extra_fields: Extra fields for the Component, defaults to None.
:return: A class definition that can be used as a component.
"""
if input_types is None:
input_types = {"value": Any}
if output_types is None and output is not None:
output_types = {key: type(value) for key, value in output.items()}
elif output_types is None:
output_types = {"value": type(None)}
def init(self):
component.set_input_types(self, **input_types)
component.set_output_types(self, **output_types)
# Both arguments are necessary to correctly define
# run but ruff doesn't like that we don't use them.
# It's fine ignoring the warning here.
def run(self, **kwargs): # noqa: ARG001
if output is not None:
return output
return dict.fromkeys(output_types.keys())
def to_dict(self):
return default_to_dict(self)
def from_dict(cls, data: dict[str, Any]):
return default_from_dict(cls, data)
fields = {"__init__": init, "run": run, "to_dict": to_dict, "from_dict": classmethod(from_dict)}
if extra_fields is not None:
fields = {**fields, **extra_fields}
if bases is None:
bases = (object,)
cls = type(name, bases, fields)
return component(cls)
@@ -0,0 +1,40 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from haystack.testing.sample_components.accumulate import Accumulate
from haystack.testing.sample_components.add_value import AddFixedValue
from haystack.testing.sample_components.concatenate import Concatenate
from haystack.testing.sample_components.double import Double
from haystack.testing.sample_components.fstring import FString
from haystack.testing.sample_components.future_annotations import HelloUsingFutureAnnotations
from haystack.testing.sample_components.greet import Greet
from haystack.testing.sample_components.hello import Hello
from haystack.testing.sample_components.joiner import StringJoiner, StringListJoiner
from haystack.testing.sample_components.parity import Parity
from haystack.testing.sample_components.remainder import Remainder
from haystack.testing.sample_components.repeat import Repeat
from haystack.testing.sample_components.subtract import Subtract
from haystack.testing.sample_components.sum import Sum
from haystack.testing.sample_components.text_splitter import TextSplitter
from haystack.testing.sample_components.threshold import Threshold
__all__ = [
"Concatenate",
"Subtract",
"Parity",
"Remainder",
"Accumulate",
"Threshold",
"AddFixedValue",
"Repeat",
"Sum",
"Greet",
"Double",
"StringJoiner",
"Hello",
"HelloUsingFutureAnnotations",
"TextSplitter",
"StringListJoiner",
"FString",
]
@@ -0,0 +1,70 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from collections.abc import Callable
from typing import Any
from haystack.core.component import component
from haystack.core.errors import ComponentDeserializationError
from haystack.core.serialization import default_to_dict
from haystack.utils.callable_serialization import deserialize_callable, serialize_callable
def _default_function(first: int, second: int) -> int:
return first + second
@component
class Accumulate:
"""
Accumulates the value flowing through the connection into an internal attribute.
The sum function can be customized. Example of how to deal with serialization when some of the parameters
are not directly serializable.
"""
def __init__(self, function: Callable | None = None) -> None:
"""
Class constructor
:param function:
the function to use to accumulate the values.
The function must take exactly two values.
If it's a callable, it's used as it is.
If it's a string, the component will look for it in sys.modules and
import it at need. This is also a parameter.
"""
self.state = 0
self.function: Callable = _default_function if function is None else function
def to_dict(self) -> dict[str, Any]:
"""Converts the component to a dictionary"""
return default_to_dict(self, function=serialize_callable(self.function))
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "Accumulate":
"""Loads the component from a dictionary"""
if "type" not in data:
raise ComponentDeserializationError("Missing 'type' in component serialization data")
if data["type"] != f"{cls.__module__}.{cls.__name__}":
raise ComponentDeserializationError(f"Class '{data['type']}' can't be deserialized as '{cls.__name__}'")
init_params = data.get("init_parameters", {})
# Resolve the function through `deserialize_callable` so it passes the deserialization
# allowlist instead of importing arbitrary handles directly.
function = init_params.get("function")
accumulator_function = deserialize_callable(function) if function else None
return cls(function=accumulator_function)
@component.output_types(value=int)
def run(self, value: int):
"""
Accumulates the value flowing through the connection into an internal attribute.
The sum function can be customized.
"""
self.state = self.function(self.state, value)
return {"value": self.state}
@@ -0,0 +1,24 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from haystack.core.component import component
@component
class AddFixedValue:
"""
Adds two values together.
"""
def __init__(self, add: int = 1) -> None:
self.add = add
@component.output_types(result=int)
def run(self, value: int, add: int | None = None):
"""
Adds two values together.
"""
if add is None:
add = self.add
return {"result": value + add}
@@ -0,0 +1,29 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from haystack.core.component import component
@component
class Concatenate:
"""
Concatenates two values
"""
@component.output_types(value=list[str])
def run(self, first: list[str] | str, second: list[str] | str):
"""
Concatenates two values
"""
if isinstance(first, str) and isinstance(second, str):
res = [first, second]
elif isinstance(first, list) and isinstance(second, list):
res = first + second
elif isinstance(first, list) and isinstance(second, str):
res = first + [second]
elif isinstance(first, str) and isinstance(second, list):
res = [first] + second
else:
res = None
return {"value": res}
@@ -0,0 +1,19 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from haystack.core.component import component
@component
class Double:
"""
Doubles the input value.
"""
@component.output_types(value=int)
def run(self, value: int):
"""
Doubles the input value.
"""
return {"value": value * 2}
@@ -0,0 +1,32 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from typing import Any
from haystack.core.component import component
@component
class FString:
"""
Takes a template string and a list of variables in input and returns the formatted string in output.
"""
def __init__(self, template: str, variables: list[str] | None = None) -> None:
self.template = template
self.variables = variables or []
if "template" in self.variables:
raise ValueError("The variable name 'template' is reserved and cannot be used.")
component.set_input_types(self, **dict.fromkeys(self.variables, Any))
@component.output_types(string=str)
def run(self, template: str | None = None, **kwargs):
"""
Takes a template string and a list of variables in input and returns the formatted string in output.
If the template is not given, the component will use the one given at initialization.
"""
if not template:
template = self.template
return {"string": template.format(**kwargs)}
@@ -0,0 +1,15 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from __future__ import annotations
from haystack import component
@component
class HelloUsingFutureAnnotations:
@component.output_types(output=str)
def run(self, word: str) -> dict[str, str]:
"""Takes a string in input and returns "Hello, <string>!"in output."""
return {"output": f"Hello, {word}!"}
@@ -0,0 +1,48 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
import logging
import haystack.logging as haystack_logging
from haystack.core.component import component
logger = haystack_logging.getLogger(__name__)
@component
class Greet:
"""
Logs a greeting message without affecting the value passing on the connection.
"""
def __init__(
self, message: str = "\nGreeting component says: Hi! The value is {value}\n", log_level: str = "INFO"
) -> None:
"""
Class constructor
:param message: the message to log. Can use `{value}` to embed the value.
:param log_level: the level to log at.
"""
if log_level and not getattr(logging, log_level):
raise ValueError(f"This log level does not exist: {log_level}")
self.message = message
self.log_level = log_level
@component.output_types(value=int)
def run(self, value: int, message: str | None = None, log_level: str | None = None):
"""
Logs a greeting message without affecting the value passing on the connection.
"""
if not message:
message = self.message
if not log_level:
log_level = self.log_level
level = getattr(logging, log_level, None)
if not level:
raise ValueError(f"This log level does not exist: {log_level}")
logger.log(level=level, msg=message.format(value=value))
return {"value": value}
@@ -0,0 +1,13 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from haystack.core.component import component
@component
class Hello:
@component.output_types(output=str)
def run(self, word: str):
"""Takes a string in input and returns "Hello, <string>!"in output."""
return {"output": f"Hello, {word}!"}
@@ -0,0 +1,35 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from haystack.core.component import component
from haystack.core.component.types import Variadic
@component
class StringJoiner:
@component.output_types(output=str)
def run(self, input_str: Variadic[str]):
"""
Take strings from multiple input nodes and join them into a single one returned in output.
Since `input_str` is Variadic, we know we'll receive a list[str].
"""
return {"output": " ".join(input_str)}
@component
class StringListJoiner:
@component.output_types(output=str)
def run(self, inputs: Variadic[list[str]]):
"""
Take list of strings from multiple input nodes and join them into a single one returned in output.
Since `input_str` is Variadic, we know we'll receive a list[list[str]].
"""
retval: list[str] = []
for list_of_strings in inputs:
retval += list_of_strings
return {"output": retval}
@@ -0,0 +1,22 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from haystack.core.component import component
@component
class Parity:
"""
Redirects the value, unchanged, along the 'even' connection if even, or along the 'odd' one if odd.
"""
@component.output_types(even=int, odd=int)
def run(self, value: int):
"""
:param value: The value to check for parity
"""
remainder = value % 2
if remainder:
return {"odd": value}
return {"even": value}
@@ -0,0 +1,21 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from haystack.core.component import component
@component
class Remainder:
def __init__(self, divisor: int = 3) -> None:
if divisor == 0:
raise ValueError("Can't divide by zero")
self.divisor = divisor
component.set_output_types(self, **{f"remainder_is_{val}": int for val in range(divisor)})
def run(self, value: int):
"""
:param value: the value to check the remainder of.
"""
remainder = value % self.divisor
return {f"remainder_is_{remainder}": value}
@@ -0,0 +1,19 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from haystack.core.component import component
@component
class Repeat:
def __init__(self, outputs: list[str]) -> None:
self._outputs = outputs
component.set_output_types(self, **dict.fromkeys(outputs, int))
def run(self, value: int):
"""
:param value: the value to repeat.
"""
return dict.fromkeys(self._outputs, value)
@@ -0,0 +1,22 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from haystack.core.component import component
@component
class Subtract:
"""
Compute the difference between two values.
"""
@component.output_types(difference=int)
def run(self, first_value: int, second_value: int):
"""
Run the component.
:param first_value: name of the connection carrying the value to subtract from.
:param second_value: name of the connection carrying the value to subtract.
"""
return {"difference": first_value - second_value}
+16
View File
@@ -0,0 +1,16 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from haystack.core.component import component
from haystack.core.component.types import Variadic
@component
class Sum:
@component.output_types(total=int)
def run(self, values: Variadic[int]):
"""
:param value: the values to sum.
"""
return {"total": sum(values)}
@@ -0,0 +1,14 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from haystack.core.component import component
@component
class TextSplitter:
@component.output_types(output=list[str])
def run(self, sentence: str):
"""Takes a sentence in input and returns its words in output."""
return {"output": sentence.split()}
@@ -0,0 +1,34 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
from haystack.core.component import component
@component
class Threshold:
"""
Redirects the value, along a different connection whether the value is above or below the given threshold.
:param threshold: the number to compare the input value against. This is also a parameter.
"""
def __init__(self, threshold: int = 10) -> None:
"""
:param threshold: the number to compare the input value against.
"""
self.threshold = threshold
@component.output_types(above=int, below=int)
def run(self, value: int, threshold: int | None = None):
"""
Redirects the value, along a different connection whether the value is above or below the given threshold.
:param threshold: the number to compare the input value against. This is also a parameter.
"""
if threshold is None:
threshold = self.threshold
if value < threshold:
return {"below": value}
return {"above": value}
+36
View File
@@ -0,0 +1,36 @@
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
#
# SPDX-License-Identifier: Apache-2.0
import os
import random
from haystack import logging
logger = logging.getLogger(__name__)
def set_all_seeds(seed: int, deterministic_cudnn: bool = False) -> None:
"""
Setting multiple seeds to make runs reproducible.
Important: Enabling `deterministic_cudnn` gives you full reproducibility with CUDA,
but might slow down your training (see https://pytorch.org/docs/stable/notes/randomness.html#cudnn) !
:param seed:number to use as seed
:param deterministic_cudnn: Enable for full reproducibility when using CUDA. Caution: might slow down training.
"""
random.seed(seed)
os.environ["PYTHONHASHSEED"] = str(seed)
try:
import torch
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
if deterministic_cudnn:
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
except (ImportError, ModuleNotFoundError) as exc:
logger.info("Could not set PyTorch seed because torch is not installed. Exception: {exception}", exception=exc)