chore: import upstream snapshot with attribution
CodeQL / Analyze (python) (push) Has been cancelled
Update Platform Components Table / update (push) Has been cancelled
Docker image release / Build base image (push) Has been cancelled
Sync docs with Docusaurus / sync (push) Has been cancelled
Tests / Check if changed (push) Has been cancelled
Tests / format (push) Has been cancelled
Tests / check-imports (push) Has been cancelled
Tests / Unit / macos-latest (push) Has been cancelled
Tests / Unit / ubuntu-latest (push) Has been cancelled
Tests / Unit / windows-latest (push) Has been cancelled
Tests / mypy (push) Has been cancelled
Tests / Integration / ubuntu-latest (push) Has been cancelled
Tests / Integration / macos-latest (push) Has been cancelled
Tests / Integration / windows-latest (push) Has been cancelled
Tests / notify-slack-on-failure (push) Has been cancelled
Tests / Mark tests as completed (push) Has been cancelled
CodeQL / Analyze (python) (push) Has been cancelled
Update Platform Components Table / update (push) Has been cancelled
Docker image release / Build base image (push) Has been cancelled
Sync docs with Docusaurus / sync (push) Has been cancelled
Tests / Check if changed (push) Has been cancelled
Tests / format (push) Has been cancelled
Tests / check-imports (push) Has been cancelled
Tests / Unit / macos-latest (push) Has been cancelled
Tests / Unit / ubuntu-latest (push) Has been cancelled
Tests / Unit / windows-latest (push) Has been cancelled
Tests / mypy (push) Has been cancelled
Tests / Integration / ubuntu-latest (push) Has been cancelled
Tests / Integration / macos-latest (push) Has been cancelled
Tests / Integration / windows-latest (push) Has been cancelled
Tests / notify-slack-on-failure (push) Has been cancelled
Tests / Mark tests as completed (push) Has been cancelled
This commit is contained in:
@@ -0,0 +1,3 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
@@ -0,0 +1,923 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import copy
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
from jinja2.nativetypes import NativeEnvironment
|
||||
|
||||
from haystack import Pipeline
|
||||
from haystack.components.routers import ConditionalRouter
|
||||
from haystack.components.routers.conditional_router import NoRouteSelectedException
|
||||
from haystack.dataclasses import ChatMessage
|
||||
|
||||
|
||||
def custom_filter_to_sede(value):
|
||||
"""splits by hyphen and returns the first part"""
|
||||
return int(value.split("-")[0])
|
||||
|
||||
|
||||
class TestRouter:
|
||||
def test_missing_mandatory_fields(self):
|
||||
"""
|
||||
Router raises a ValueError if each route does not contain 'condition', 'output', and 'output_type' keys
|
||||
"""
|
||||
routes = [
|
||||
{"condition": "{{streams|length < 2}}", "output": "{{query}}"},
|
||||
{"condition": "{{streams|length < 2}}", "output_type": str},
|
||||
]
|
||||
with pytest.raises(ValueError):
|
||||
ConditionalRouter(routes)
|
||||
|
||||
def test_invalid_condition_field(self):
|
||||
"""
|
||||
ConditionalRouter init raises a ValueError if one of the routes contains invalid condition
|
||||
"""
|
||||
# invalid condition field
|
||||
routes = [{"condition": "{{streams|length < 2", "output": "query", "output_type": str, "output_name": "test"}]
|
||||
with pytest.raises(ValueError, match="Invalid template"):
|
||||
ConditionalRouter(routes)
|
||||
|
||||
def test_invalid_output_template_non_string(self):
|
||||
"""
|
||||
ConditionalRouter init raises a ValueError with helpful error message when output is not a string
|
||||
"""
|
||||
# output is an int instead of a string template
|
||||
routes = [
|
||||
{
|
||||
"condition": '{{ flag == "double" }}',
|
||||
"output": 2,
|
||||
"output_name": "num_additional_outputs",
|
||||
"output_type": int,
|
||||
}
|
||||
]
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
ConditionalRouter(routes)
|
||||
error_message = str(exc_info.value)
|
||||
assert "Invalid template for output" in error_message
|
||||
assert "string" in error_message
|
||||
assert "Jinja2 template" in error_message
|
||||
assert "2" in error_message
|
||||
|
||||
def test_invalid_output_template_non_string_list(self):
|
||||
"""
|
||||
ConditionalRouter init raises a ValueError with helpful error message when output in list is not a string
|
||||
"""
|
||||
# output list contains an int instead of a string template
|
||||
routes = [
|
||||
{
|
||||
"condition": '{{ flag == "double" }}',
|
||||
"output": ["{{streams}}", 2],
|
||||
"output_name": ["streams", "num"],
|
||||
"output_type": [list[int], int],
|
||||
}
|
||||
]
|
||||
with pytest.raises(ValueError) as exc_info:
|
||||
ConditionalRouter(routes)
|
||||
error_message = str(exc_info.value)
|
||||
assert "Invalid template for output" in error_message
|
||||
assert "string" in error_message
|
||||
assert "Jinja2 template" in error_message
|
||||
|
||||
def test_no_vars_in_output_route_but_with_output_name(self):
|
||||
"""
|
||||
Router can't accept a route with no variables used in the output field
|
||||
"""
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{streams|length > 2}}",
|
||||
"output": "This is a constant",
|
||||
"output_name": "enough_streams",
|
||||
"output_type": str,
|
||||
}
|
||||
]
|
||||
router = ConditionalRouter(routes)
|
||||
kwargs = {"streams": [1, 2, 3], "query": "Haystack"}
|
||||
result = router.run(**kwargs)
|
||||
assert result == {"enough_streams": "This is a constant"}
|
||||
|
||||
def test_mandatory_and_optional_fields_with_extra_fields(self):
|
||||
"""
|
||||
Router accepts a list of routes with mandatory and optional fields but not if some new field is added
|
||||
"""
|
||||
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{streams|length < 2}}",
|
||||
"output": "{{query}}",
|
||||
"output_type": str,
|
||||
"output_name": "test",
|
||||
"bla": "bla",
|
||||
},
|
||||
{"condition": "{{streams|length < 2}}", "output": "{{query}}", "output_type": str},
|
||||
]
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
ConditionalRouter(routes)
|
||||
|
||||
def test_router_initialized(self):
|
||||
routes = [
|
||||
{"condition": "{{streams|length < 2}}", "output": "{{query}}", "output_type": str, "output_name": "query"},
|
||||
{
|
||||
"condition": "{{streams|length >= 2}}",
|
||||
"output": "{{streams}}",
|
||||
"output_type": list[int],
|
||||
"output_name": "streams",
|
||||
},
|
||||
]
|
||||
router = ConditionalRouter(routes)
|
||||
|
||||
assert router.routes == routes
|
||||
assert set(router.__haystack_input__._sockets_dict.keys()) == {"query", "streams"}
|
||||
assert set(router.__haystack_output__._sockets_dict.keys()) == {"query", "streams"}
|
||||
|
||||
def test_router_evaluate_condition_expressions(self):
|
||||
router = ConditionalRouter(
|
||||
[
|
||||
{
|
||||
"condition": "{{streams|length < 2}}",
|
||||
"output": "{{query}}",
|
||||
"output_type": str,
|
||||
"output_name": "query",
|
||||
},
|
||||
{
|
||||
"condition": "{{streams|length >= 2}}",
|
||||
"output": "{{streams}}",
|
||||
"output_type": list[int],
|
||||
"output_name": "streams",
|
||||
},
|
||||
]
|
||||
)
|
||||
# first route should be selected
|
||||
kwargs = {"streams": [1, 2, 3], "query": "test"}
|
||||
result = router.run(**kwargs)
|
||||
assert result == {"streams": [1, 2, 3]}
|
||||
|
||||
# second route should be selected
|
||||
kwargs = {"streams": [1], "query": "test"}
|
||||
result = router.run(**kwargs)
|
||||
assert result == {"query": "test"}
|
||||
|
||||
def test_router_evaluate_condition_expressions_using_output_slot(self):
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{streams|length > 2}}",
|
||||
"output": "{{streams}}",
|
||||
"output_name": "enough_streams",
|
||||
"output_type": list[int],
|
||||
},
|
||||
{
|
||||
"condition": "{{streams|length <= 2}}",
|
||||
"output": "{{streams}}",
|
||||
"output_name": "insufficient_streams",
|
||||
"output_type": list[int],
|
||||
},
|
||||
]
|
||||
router = ConditionalRouter(routes)
|
||||
# enough_streams output slot will be selected with [1, 2, 3] list being outputted
|
||||
kwargs = {"streams": [1, 2, 3], "query": "Haystack"}
|
||||
result = router.run(**kwargs)
|
||||
assert result == {"enough_streams": [1, 2, 3]}
|
||||
|
||||
def test_complex_condition(self):
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{messages[-1].meta.finish_reason == 'function_call'}}",
|
||||
"output": "{{streams}}",
|
||||
"output_type": list[int],
|
||||
"output_name": "streams",
|
||||
},
|
||||
{
|
||||
"condition": "{{True}}",
|
||||
"output": "{{query}}",
|
||||
"output_type": str,
|
||||
"output_name": "query",
|
||||
}, # catch-all condition
|
||||
]
|
||||
router = ConditionalRouter(routes)
|
||||
message = mock.MagicMock()
|
||||
message.meta.finish_reason = "function_call"
|
||||
result = router.run(messages=[message], streams=[1, 2, 3], query="my query")
|
||||
assert result == {"streams": [1, 2, 3]}
|
||||
|
||||
def test_router_no_route(self):
|
||||
# should raise an exception
|
||||
router = ConditionalRouter(
|
||||
[
|
||||
{
|
||||
"condition": "{{streams|length < 2}}",
|
||||
"output": "{{query}}",
|
||||
"output_type": str,
|
||||
"output_name": "query",
|
||||
},
|
||||
{
|
||||
"condition": "{{streams|length >= 5}}",
|
||||
"output": "{{streams}}",
|
||||
"output_type": list[int],
|
||||
"output_name": "streams",
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
kwargs = {"streams": [1, 2, 3], "query": "test"}
|
||||
with pytest.raises(NoRouteSelectedException):
|
||||
router.run(**kwargs)
|
||||
|
||||
def test_router_raises_value_error_if_route_not_dictionary(self):
|
||||
"""
|
||||
Router raises a ValueError if each route is not a dictionary
|
||||
"""
|
||||
routes = [
|
||||
{"condition": "{{streams|length < 2}}", "output": "{{query}}", "output_type": str, "output_name": "query"},
|
||||
["{{streams|length >= 2}}", "streams", list[int]],
|
||||
]
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
ConditionalRouter(routes)
|
||||
|
||||
def test_router_raises_value_error_if_route_missing_keys(self):
|
||||
"""
|
||||
Router raises a ValueError if each route does not contain 'condition', 'output', and 'output_type' keys
|
||||
"""
|
||||
routes = [
|
||||
{"condition": "{{streams|length < 2}}", "output": "{{query}}"},
|
||||
{"condition": "{{streams|length < 2}}", "output_type": str},
|
||||
]
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
ConditionalRouter(routes)
|
||||
|
||||
def test_router_de_serialization(self):
|
||||
routes = [
|
||||
{"condition": "{{streams|length < 2}}", "output": "{{query}}", "output_type": str, "output_name": "query"},
|
||||
{
|
||||
"condition": "{{streams|length >= 2}}",
|
||||
"output": "{{streams}}",
|
||||
"output_type": list[int],
|
||||
"output_name": "streams",
|
||||
},
|
||||
]
|
||||
router = ConditionalRouter(routes)
|
||||
router_dict = router.to_dict()
|
||||
|
||||
# assert that the router dict is correct, with all keys and values being strings
|
||||
for route in router_dict["init_parameters"]["routes"]:
|
||||
for key in route.keys():
|
||||
assert isinstance(key, str)
|
||||
assert isinstance(route[key], str)
|
||||
|
||||
new_router = ConditionalRouter.from_dict(router_dict)
|
||||
assert router.routes == new_router.routes
|
||||
|
||||
# now use both routers with the same input
|
||||
kwargs = {"streams": [1, 2, 3], "query": "Haystack"}
|
||||
result1 = router.run(**kwargs)
|
||||
result2 = new_router.run(**kwargs)
|
||||
|
||||
# check that the result is the same and correct
|
||||
assert result1 == result2 and result1 == {"streams": [1, 2, 3]}
|
||||
|
||||
def test_router_de_serialization_with_none_argument(self):
|
||||
new_router = ConditionalRouter.from_dict(
|
||||
{
|
||||
"type": "haystack.components.routers.conditional_router.ConditionalRouter",
|
||||
"init_parameters": {
|
||||
"routes": [
|
||||
{
|
||||
"condition": "{{streams|length < 2}}",
|
||||
"output": "{{query}}",
|
||||
"output_type": "str",
|
||||
"output_name": "query",
|
||||
},
|
||||
{
|
||||
"condition": "{{streams|length >= 2}}",
|
||||
"output": "{{streams}}",
|
||||
"output_type": "list[int]",
|
||||
"output_name": "streams",
|
||||
},
|
||||
],
|
||||
"custom_filters": None,
|
||||
"unsafe": False,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
# now use both routers with the same input
|
||||
kwargs = {"streams": [1, 2, 3], "query": "Haystack"}
|
||||
result2 = new_router.run(**kwargs)
|
||||
assert result2 == {"streams": [1, 2, 3]}
|
||||
|
||||
def test_router_serialization_idempotence(self):
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{streams|length < 2}}",
|
||||
"output": "{{message}}",
|
||||
"output_type": ChatMessage,
|
||||
"output_name": "message",
|
||||
},
|
||||
{
|
||||
"condition": "{{streams|length >= 2}}",
|
||||
"output": "{{streams}}",
|
||||
"output_type": list[int],
|
||||
"output_name": "streams",
|
||||
},
|
||||
]
|
||||
router = ConditionalRouter(routes)
|
||||
# invoke to_dict twice and check that the result is the same
|
||||
router_dict_first_invocation = copy.deepcopy(router.to_dict())
|
||||
router_dict_second_invocation = router.to_dict()
|
||||
assert router_dict_first_invocation == router_dict_second_invocation
|
||||
|
||||
def test_custom_filter(self):
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{phone_num|get_area_code == 123}}",
|
||||
"output": "Phone number has a 123 area code",
|
||||
"output_name": "good_phone_num",
|
||||
"output_type": str,
|
||||
},
|
||||
{
|
||||
"condition": "{{phone_num|get_area_code != 123}}",
|
||||
"output": "Phone number does not have 123 area code",
|
||||
"output_name": "bad_phone_num",
|
||||
"output_type": str,
|
||||
},
|
||||
]
|
||||
|
||||
router = ConditionalRouter(routes, custom_filters={"get_area_code": custom_filter_to_sede})
|
||||
kwargs = {"phone_num": "123-456-7890"}
|
||||
result = router.run(**kwargs)
|
||||
assert result == {"good_phone_num": "Phone number has a 123 area code"}
|
||||
kwargs = {"phone_num": "321-456-7890"}
|
||||
result = router.run(**kwargs)
|
||||
assert result == {"bad_phone_num": "Phone number does not have 123 area code"}
|
||||
|
||||
def test_sede_with_custom_filter(self):
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{ test|custom_filter_to_sede == 123 }}",
|
||||
"output": "123",
|
||||
"output_name": "test",
|
||||
"output_type": int,
|
||||
}
|
||||
]
|
||||
custom_filters = {"custom_filter_to_sede": custom_filter_to_sede}
|
||||
router = ConditionalRouter(routes, custom_filters=custom_filters)
|
||||
kwargs = {"test": "123-456-789"}
|
||||
result = router.run(**kwargs)
|
||||
assert result == {"test": 123}
|
||||
serialized_router = router.to_dict()
|
||||
deserialized_router = ConditionalRouter.from_dict(serialized_router)
|
||||
assert deserialized_router.custom_filters == router.custom_filters
|
||||
assert deserialized_router.custom_filters["custom_filter_to_sede"]("123-456-789") == 123
|
||||
assert result == deserialized_router.run(**kwargs)
|
||||
|
||||
def test_unsafe(self):
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{streams|length < 2}}",
|
||||
"output": "{{message}}",
|
||||
"output_type": ChatMessage,
|
||||
"output_name": "message",
|
||||
},
|
||||
{
|
||||
"condition": "{{streams|length >= 2}}",
|
||||
"output": "{{streams}}",
|
||||
"output_type": list[int],
|
||||
"output_name": "streams",
|
||||
},
|
||||
]
|
||||
router = ConditionalRouter(routes, unsafe=True)
|
||||
streams = [1]
|
||||
message = ChatMessage.from_user("This is a message")
|
||||
res = router.run(streams=streams, message=message)
|
||||
assert res == {"message": message}
|
||||
|
||||
def test_validate_output_type_without_unsafe(self):
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{streams|length < 2}}",
|
||||
"output": "{{message}}",
|
||||
"output_type": ChatMessage,
|
||||
"output_name": "message",
|
||||
},
|
||||
{
|
||||
"condition": "{{streams|length >= 2}}",
|
||||
"output": "{{streams}}",
|
||||
"output_type": list[int],
|
||||
"output_name": "streams",
|
||||
},
|
||||
]
|
||||
router = ConditionalRouter(routes, validate_output_type=True)
|
||||
streams = [1]
|
||||
message = ChatMessage.from_user("This is a message")
|
||||
with pytest.raises(ValueError, match="Route 'message' type doesn't match expected type"):
|
||||
router.run(streams=streams, message=message)
|
||||
|
||||
def test_validate_output_type_with_unsafe(self):
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{streams|length < 2}}",
|
||||
"output": "{{message}}",
|
||||
"output_type": ChatMessage,
|
||||
"output_name": "message",
|
||||
},
|
||||
{
|
||||
"condition": "{{streams|length >= 2}}",
|
||||
"output": "{{streams}}",
|
||||
"output_type": list[int],
|
||||
"output_name": "streams",
|
||||
},
|
||||
]
|
||||
router = ConditionalRouter(routes, unsafe=True, validate_output_type=True)
|
||||
streams = [1]
|
||||
message = ChatMessage.from_user("This is a message")
|
||||
res = router.run(streams=streams, message=message)
|
||||
assert isinstance(res["message"], ChatMessage)
|
||||
|
||||
streams = ["1", "2", "3", "4"]
|
||||
with pytest.raises(ValueError, match="Route 'streams' type doesn't match expected type"):
|
||||
router.run(streams=streams, message=message)
|
||||
|
||||
def test_validate_output_type_with_pep604(self):
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{True}}",
|
||||
"output": "{{value}}",
|
||||
"output_type": list[str] | dict[str, int] | None,
|
||||
"output_name": "result",
|
||||
}
|
||||
]
|
||||
router = ConditionalRouter(routes, validate_output_type=True)
|
||||
|
||||
result = router.run(value=["a", "b"])
|
||||
assert result == {"result": ["a", "b"]}
|
||||
|
||||
result = router.run(value={"key": 1})
|
||||
assert result == {"result": {"key": 1}}
|
||||
|
||||
result = router.run(value=None)
|
||||
assert result == {"result": None}
|
||||
|
||||
with pytest.raises(ValueError, match="Route 'result' type doesn't match expected type"):
|
||||
router.run(value=42)
|
||||
|
||||
def test_str_not_matching_list_str(self):
|
||||
"""
|
||||
Test that a plain str value does not incorrectly validate as list[str].
|
||||
str is a Sequence, but when the expected type is list[str], a bare string
|
||||
should be rejected.
|
||||
"""
|
||||
routes = [{"condition": "{{True}}", "output": "{{value}}", "output_type": list[str], "output_name": "result"}]
|
||||
router = ConditionalRouter(routes, validate_output_type=True)
|
||||
|
||||
# A list of strings should pass
|
||||
result = router.run(value=["a", "b"])
|
||||
assert result == {"result": ["a", "b"]}
|
||||
|
||||
# A plain string should NOT pass as list[str]
|
||||
with pytest.raises(ValueError, match="Route 'result' type doesn't match expected type"):
|
||||
router.run(value="hello")
|
||||
|
||||
def test_router_with_optional_parameters(self):
|
||||
"""
|
||||
Test that the router works with optional parameters, particularly testing the default/fallback route
|
||||
when an expected parameter is not provided.
|
||||
"""
|
||||
routes = [
|
||||
{"condition": '{{path == "rag"}}', "output": "{{question}}", "output_name": "normal", "output_type": str},
|
||||
{
|
||||
"condition": '{{path == "followup_short"}}',
|
||||
"output": "{{question}}",
|
||||
"output_name": "followup_short",
|
||||
"output_type": str,
|
||||
},
|
||||
{
|
||||
"condition": '{{path == "followup_elaborate"}}',
|
||||
"output": "{{question}}",
|
||||
"output_name": "followup_elaborate",
|
||||
"output_type": str,
|
||||
},
|
||||
{"condition": "{{ True }}", "output": "{{ question }}", "output_name": "fallback", "output_type": str},
|
||||
]
|
||||
|
||||
router = ConditionalRouter(routes, optional_variables=["path"])
|
||||
|
||||
# Test direct component usage
|
||||
result = router.run(question="What?")
|
||||
assert result == {"fallback": "What?"}, "Default route should be taken when 'path' is not provided"
|
||||
|
||||
# Test with path parameter
|
||||
result = router.run(question="What?", path="rag")
|
||||
assert result == {"normal": "What?"}, "Specific route should be taken when 'path' is provided"
|
||||
|
||||
pipe = Pipeline()
|
||||
pipe.add_component("router", router)
|
||||
|
||||
# Test pipeline without path parameter
|
||||
result = pipe.run(data={"router": {"question": "What?"}})
|
||||
assert result["router"] == {"fallback": "What?"}, (
|
||||
"Default route should work in pipeline when 'path' is not provided"
|
||||
)
|
||||
|
||||
# Test pipeline with path parameter
|
||||
result = pipe.run(data={"router": {"question": "What?", "path": "followup_short"}})
|
||||
assert result["router"] == {"followup_short": "What?"}, "Specific route should work in pipeline"
|
||||
|
||||
def test_router_with_multiple_optional_parameters(self):
|
||||
"""
|
||||
Test ConditionalRouter with a mix of mandatory and optional parameters,
|
||||
exploring various combinations of provided/missing optional variables.
|
||||
"""
|
||||
routes = [
|
||||
{
|
||||
"condition": '{{mode == "chat" and language == "en" and source == "doc"}}',
|
||||
"output": "{{question}}",
|
||||
"output_name": "en_doc_chat",
|
||||
"output_type": str,
|
||||
},
|
||||
{
|
||||
"condition": '{{mode == "qa" and source == "web"}}',
|
||||
"output": "{{question}}",
|
||||
"output_name": "web_qa",
|
||||
"output_type": str,
|
||||
},
|
||||
{
|
||||
"condition": '{{mode == "qa" and source == "doc"}}',
|
||||
"output": "{{question}}",
|
||||
"output_name": "doc_qa",
|
||||
"output_type": str,
|
||||
},
|
||||
{
|
||||
"condition": '{{mode == "chat" and language == "en"}}',
|
||||
"output": "{{question}}",
|
||||
"output_name": "en_chat",
|
||||
"output_type": str,
|
||||
},
|
||||
{
|
||||
"condition": '{{mode == "chat"}}', # fallback for chat without language
|
||||
"output": "{{question}}",
|
||||
"output_name": "default_chat",
|
||||
"output_type": str,
|
||||
},
|
||||
{
|
||||
"condition": "{{ True }}", # global fallback
|
||||
"output": "{{question}}",
|
||||
"output_name": "fallback",
|
||||
"output_type": str,
|
||||
},
|
||||
]
|
||||
|
||||
# There are four variables in the routes:
|
||||
# - mandatory: mode, question (always must be provided) or we'll route to fallback
|
||||
# - optional: source, language
|
||||
router = ConditionalRouter(routes, optional_variables=["source", "language"])
|
||||
|
||||
# Test with mandatory parameter only
|
||||
result = router.run(question="What?", mode="chat")
|
||||
assert result == {"default_chat": "What?"}, "Should use chat fallback when language not provided"
|
||||
|
||||
# Test with all parameters provided
|
||||
result = router.run(question="What?", mode="chat", language="en", source="doc")
|
||||
assert result == {"en_doc_chat": "What?"}, "Should use specific route when all params provided"
|
||||
|
||||
# Test with different mandatory value and one optional
|
||||
result = router.run(question="What?", mode="qa", source="web")
|
||||
assert result == {"web_qa": "What?"}, "Should route qa with source correctly"
|
||||
|
||||
# Test with mandatory the routes to fallback
|
||||
result = router.run(question="What?", mode="qa")
|
||||
assert result == {"fallback": "What?"}, "Should use global fallback for qa without source"
|
||||
|
||||
# Test in pipeline
|
||||
pipe = Pipeline()
|
||||
pipe.add_component("router", router)
|
||||
|
||||
# Test pipeline with mandatory only
|
||||
result = pipe.run(data={"router": {"question": "What?", "mode": "chat"}})
|
||||
assert result["router"] == {"default_chat": "What?"}, "Pipeline should handle missing optionals"
|
||||
|
||||
# Test pipeline with mandatory and one optional
|
||||
result = pipe.run(data={"router": {"question": "What?", "mode": "qa", "source": "doc"}})
|
||||
assert result["router"] == {"doc_qa": "What?"}, "Pipeline should handle all parameters"
|
||||
|
||||
# Test pipeline with mandatory and both optionals
|
||||
result = pipe.run(data={"router": {"question": "What?", "mode": "chat", "language": "en", "source": "doc"}})
|
||||
assert result["router"] == {"en_doc_chat": "What?"}, "Pipeline should handle all parameters"
|
||||
|
||||
def test_warns_on_unused_optional_variables(self, caplog):
|
||||
"""
|
||||
Test that a warning is raised when optional_variables contains variables
|
||||
that are not used in any route conditions or outputs.
|
||||
"""
|
||||
routes = [
|
||||
{"condition": '{{mode == "chat"}}', "output": "{{question}}", "output_name": "chat", "output_type": str},
|
||||
{"condition": "{{ True }}", "output": "{{question}}", "output_name": "fallback", "output_type": str},
|
||||
]
|
||||
|
||||
# Initialize with unused optional variables and capture warning
|
||||
router = ConditionalRouter(routes=routes, optional_variables=["unused_var1", "unused_var2"])
|
||||
assert "optional variables" in caplog.records[0].message
|
||||
|
||||
# Verify router still works normally
|
||||
result = router.run(question="What?", mode="chat")
|
||||
assert result == {"chat": "What?"}
|
||||
|
||||
def test_router_to_dict_does_not_mutate_routes(self):
|
||||
routes = [
|
||||
{"condition": "{{streams|length < 2}}", "output": "{{query}}", "output_type": str, "output_name": "query"}
|
||||
]
|
||||
|
||||
router = ConditionalRouter(routes)
|
||||
|
||||
# Store the original output_type before serializing
|
||||
original_output_type = router.routes[0]["output_type"]
|
||||
assert original_output_type is str
|
||||
|
||||
router_dict = router.to_dict()
|
||||
|
||||
# Verify that the original routes are not mutated
|
||||
assert router.routes[0]["output_type"] is str
|
||||
assert router.routes[0]["output_type"] is original_output_type
|
||||
|
||||
# Verify that the serialized output_type is a string
|
||||
assert isinstance(router_dict["init_parameters"]["routes"][0]["output_type"], str), (
|
||||
"Serialized output_type should be a string"
|
||||
)
|
||||
|
||||
# Verify that the router still works correctly after to_dict()
|
||||
result = router.run(streams=[1], query="test")
|
||||
assert result == {"query": "test"}, "Router should still work correctly after to_dict()"
|
||||
|
||||
# Double check on another ConditionalRouter instance
|
||||
new_router = ConditionalRouter.from_dict(router_dict)
|
||||
assert new_router.routes == router.routes
|
||||
assert new_router.routes[0]["output_type"] is str
|
||||
assert new_router.routes[0]["output_type"] is original_output_type
|
||||
|
||||
def test_multiple_outputs_per_route(self):
|
||||
"""Test that router handles multiple outputs per route correctly"""
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{streams|length >= 2}}",
|
||||
"output": ["{{streams}}", "{{query}}"],
|
||||
"output_type": [list[int], str],
|
||||
"output_name": ["streams", "query"],
|
||||
},
|
||||
{
|
||||
"condition": "{{streams|length < 2}}",
|
||||
"output": ["{{streams}}", "{{custom_error_message}}"],
|
||||
"output_type": [list[int], str],
|
||||
"output_name": ["streams", "custom_error_message"],
|
||||
},
|
||||
]
|
||||
router = ConditionalRouter(routes)
|
||||
|
||||
# Test with sufficient input streams
|
||||
result = router.run(streams=[1, 2, 3], query="test_1", custom_error_message="Not enough streams")
|
||||
assert result == {"streams": [1, 2, 3], "query": "test_1"}
|
||||
|
||||
# Test with insufficient input streams
|
||||
result = router.run(streams=[1], query="test_2", custom_error_message="Not enough streams")
|
||||
assert result == {"streams": [1], "custom_error_message": "Not enough streams"}
|
||||
|
||||
def test_multiple_outputs_validation(self):
|
||||
"""Test validation of routes with multiple outputs"""
|
||||
# Test mismatched lengths
|
||||
with pytest.raises(ValueError, match="must have same length"):
|
||||
ConditionalRouter(
|
||||
[
|
||||
{
|
||||
"condition": "{{streams|length >= 2}}",
|
||||
"output": ["{{streams}}", "{{query}}"],
|
||||
"output_type": [list[int]],
|
||||
"output_name": ["streams"],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
def test_sede_multiple_outputs(self):
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{phone_num|get_area_code == 123}}",
|
||||
"output": ["{{phone_num}}", "{{phone_num|get_area_code}}"],
|
||||
"output_name": ["phone_num", "area_code"],
|
||||
"output_type": [str, int],
|
||||
},
|
||||
{
|
||||
"condition": "{{phone_num|get_area_code != 123}}",
|
||||
"output": ["{{phone_num}}", "{{phone_num|get_area_code}}"],
|
||||
"output_name": ["phone_num", "area_code"],
|
||||
"output_type": [str, int],
|
||||
},
|
||||
]
|
||||
|
||||
router = ConditionalRouter(routes, custom_filters={"get_area_code": custom_filter_to_sede})
|
||||
reloaded_router = ConditionalRouter.from_dict(router.to_dict())
|
||||
assert reloaded_router.custom_filters == router.custom_filters
|
||||
assert reloaded_router.routes == router.routes
|
||||
|
||||
def test_extract_variables_correct_with_assignment(self):
|
||||
condition = """{%- if control == 'something' -%}
|
||||
{% set streams = 1 %}
|
||||
{%- else -%}
|
||||
{% set streams = 2 %}
|
||||
{%- endif -%}
|
||||
{{streams == 1}}
|
||||
"""
|
||||
templates = [condition, "{{query}}"]
|
||||
extracted_variables = ConditionalRouter._extract_variables(env=NativeEnvironment(), templates=templates)
|
||||
assert extracted_variables == {"control", "query"}
|
||||
|
||||
def test_conditional_router_passthrough_serialization_roundtrip(self):
|
||||
"""Test that output_passthrough survives to_dict/from_dict."""
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{flag}}",
|
||||
"output": "value",
|
||||
"output_name": "matched",
|
||||
"output_type": str,
|
||||
"output_passthrough": True,
|
||||
},
|
||||
{
|
||||
"condition": "{{not flag}}",
|
||||
"output": "value",
|
||||
"output_name": "unmatched",
|
||||
"output_type": str,
|
||||
"output_passthrough": True,
|
||||
},
|
||||
]
|
||||
|
||||
router = ConditionalRouter(routes)
|
||||
reloaded = ConditionalRouter.from_dict(router.to_dict())
|
||||
|
||||
assert reloaded.routes == router.routes
|
||||
assert reloaded.routes[0].get("output_passthrough") is True
|
||||
assert reloaded.routes[1].get("output_passthrough") is True
|
||||
|
||||
assert reloaded.run(flag=True, value="hello") == {"matched": "hello"}
|
||||
assert reloaded.run(flag=False, value="hello") == {"unmatched": "hello"}
|
||||
|
||||
def test_conditional_router_passthrough_with_custom_type(self):
|
||||
"""Test passthrough routing for custom types without Jinja2."""
|
||||
from dataclasses import dataclass
|
||||
|
||||
@dataclass
|
||||
class CustomDocument:
|
||||
content: str
|
||||
metadata: dict
|
||||
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{is_important}}",
|
||||
"output": "document",
|
||||
"output_name": "important",
|
||||
"output_type": CustomDocument,
|
||||
"output_passthrough": True,
|
||||
},
|
||||
{
|
||||
"condition": "{{not is_important}}",
|
||||
"output": "document",
|
||||
"output_name": "regular",
|
||||
"output_type": CustomDocument,
|
||||
"output_passthrough": True,
|
||||
},
|
||||
]
|
||||
|
||||
router = ConditionalRouter(routes)
|
||||
doc = CustomDocument(content="Important", metadata={"priority": "high"})
|
||||
|
||||
result = router.run(is_important=True, document=doc)
|
||||
assert "important" in result
|
||||
assert result["important"] == doc
|
||||
assert result["important"].content == "Important"
|
||||
|
||||
result = router.run(is_important=False, document=doc)
|
||||
assert "regular" in result
|
||||
assert result["regular"] == doc
|
||||
|
||||
def test_conditional_router_passthrough_missing_variable(self):
|
||||
"""Test that passthrough routing raises ValueError when the named variable is not provided."""
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{True}}",
|
||||
"output": "missing_var",
|
||||
"output_name": "out",
|
||||
"output_type": str,
|
||||
"output_passthrough": True,
|
||||
}
|
||||
]
|
||||
|
||||
router = ConditionalRouter(routes)
|
||||
|
||||
with pytest.raises(ValueError, match="Variable 'missing_var' not found in inputs"):
|
||||
router.run(other_var="value")
|
||||
|
||||
def test_conditional_router_passthrough_mixed(self):
|
||||
"""Test mixing passthrough and Jinja2 routes in the same router."""
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{mode == 'direct'}}",
|
||||
"output": "data",
|
||||
"output_name": "direct_route",
|
||||
"output_type": list,
|
||||
"output_passthrough": True,
|
||||
},
|
||||
{
|
||||
"condition": "{{mode == 'transform'}}",
|
||||
"output": "{{data | reverse | list}}",
|
||||
"output_name": "transformed_route",
|
||||
"output_type": list,
|
||||
},
|
||||
]
|
||||
|
||||
router = ConditionalRouter(routes)
|
||||
test_list = [1, 2, 3]
|
||||
|
||||
result = router.run(mode="direct", data=test_list)
|
||||
assert result["direct_route"] == test_list
|
||||
|
||||
result = router.run(mode="transform", data=test_list)
|
||||
assert result["transformed_route"] == [3, 2, 1]
|
||||
|
||||
def test_conditional_router_passthrough_multi_output(self):
|
||||
"""Test output_passthrough with a list of output variable names."""
|
||||
from dataclasses import dataclass
|
||||
|
||||
@dataclass
|
||||
class Payload:
|
||||
body: str
|
||||
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{flag}}",
|
||||
"output": ["label", "payload"],
|
||||
"output_name": ["out_label", "out_payload"],
|
||||
"output_type": [str, Payload],
|
||||
"output_passthrough": True,
|
||||
}
|
||||
]
|
||||
|
||||
router = ConditionalRouter(routes)
|
||||
p = Payload(body="test")
|
||||
result = router.run(flag=True, label="hello", payload=p)
|
||||
assert result == {"out_label": "hello", "out_payload": p}
|
||||
assert isinstance(result["out_payload"], Payload)
|
||||
|
||||
def test_conditional_router_passthrough_validate_output_type_mismatch(self):
|
||||
"""Test that validate_output_type catches a type mismatch on a passthrough route."""
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{True}}",
|
||||
"output": "value",
|
||||
"output_name": "out",
|
||||
"output_type": int,
|
||||
"output_passthrough": True,
|
||||
}
|
||||
]
|
||||
|
||||
router = ConditionalRouter(routes, validate_output_type=True)
|
||||
|
||||
with pytest.raises(ValueError, match="type doesn't match"):
|
||||
router.run(value="not_an_int")
|
||||
|
||||
def test_conditional_router_passthrough_optional_variable_routes_none(self):
|
||||
"""Test that a passthrough variable in optional_variables routes None when the pipeline omits it.
|
||||
|
||||
optional_variables registers the input with default=None. Inside a pipeline, missing optional
|
||||
inputs are filled with their default before run() is called. We simulate that here by passing
|
||||
maybe_value=None explicitly.
|
||||
"""
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{True}}",
|
||||
"output": "maybe_value",
|
||||
"output_name": "out",
|
||||
"output_type": str,
|
||||
"output_passthrough": True,
|
||||
}
|
||||
]
|
||||
|
||||
router = ConditionalRouter(routes, optional_variables=["maybe_value"])
|
||||
# Simulate pipeline behaviour: optional input not connected → filled with default None
|
||||
result = router.run(maybe_value=None)
|
||||
assert result == {"out": None}
|
||||
|
||||
def test_conditional_router_passthrough_skips_output_template_validation(self):
|
||||
"""Test that an invalid Jinja2 string in output is accepted when output_passthrough is True."""
|
||||
routes = [
|
||||
{
|
||||
"condition": "{{True}}",
|
||||
"output": "{{unclosed", # would be rejected as a Jinja2 template
|
||||
"output_name": "out",
|
||||
"output_type": str,
|
||||
"output_passthrough": True,
|
||||
}
|
||||
]
|
||||
|
||||
# Construction must not raise even though the output string is not valid Jinja2
|
||||
router = ConditionalRouter(routes)
|
||||
result = router.run(**{"{{unclosed": "value"})
|
||||
assert result == {"out": "value"}
|
||||
@@ -0,0 +1,62 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
from haystack.components.routers import DocumentLengthRouter
|
||||
from haystack.core.serialization import component_from_dict, component_to_dict
|
||||
from haystack.dataclasses import Document
|
||||
|
||||
|
||||
class TestDocumentLengthRouter:
|
||||
def test_init(self):
|
||||
router = DocumentLengthRouter(threshold=20)
|
||||
assert router.threshold == 20
|
||||
|
||||
def test_run(self):
|
||||
docs = [Document(content="Short"), Document(content="Long document " * 20)]
|
||||
router = DocumentLengthRouter(threshold=10)
|
||||
result = router.run(documents=docs)
|
||||
|
||||
assert len(result["short_documents"]) == 1
|
||||
assert len(result["long_documents"]) == 1
|
||||
assert result["short_documents"][0] == docs[0]
|
||||
assert result["long_documents"][0] == docs[1]
|
||||
|
||||
def test_run_with_null_content(self):
|
||||
docs = [Document(content=None), Document(content="Long document " * 20)]
|
||||
router = DocumentLengthRouter(threshold=10)
|
||||
result = router.run(documents=docs)
|
||||
|
||||
assert len(result["short_documents"]) == 1
|
||||
assert len(result["long_documents"]) == 1
|
||||
assert result["short_documents"][0] == docs[0]
|
||||
assert result["long_documents"][0] == docs[1]
|
||||
|
||||
def test_run_with_negative_threshold(self):
|
||||
docs = [Document(content=None), Document(content="Short"), Document(content="Long document " * 20)]
|
||||
router = DocumentLengthRouter(threshold=-1)
|
||||
result = router.run(documents=docs)
|
||||
|
||||
assert len(result["short_documents"]) == 1
|
||||
assert len(result["long_documents"]) == 2
|
||||
assert result["short_documents"][0] == docs[0]
|
||||
assert result["long_documents"][0] == docs[1]
|
||||
assert result["long_documents"][1] == docs[2]
|
||||
|
||||
def test_to_dict(self):
|
||||
router = DocumentLengthRouter(threshold=10)
|
||||
expected_dict = {
|
||||
"type": "haystack.components.routers.document_length_router.DocumentLengthRouter",
|
||||
"init_parameters": {"threshold": 10},
|
||||
}
|
||||
assert component_to_dict(router, "router") == expected_dict
|
||||
|
||||
def test_from_dict(self):
|
||||
router_dict = {
|
||||
"type": "haystack.components.routers.document_length_router.DocumentLengthRouter",
|
||||
"init_parameters": {"threshold": 10},
|
||||
}
|
||||
loaded_router = component_from_dict(DocumentLengthRouter, router_dict, name="router")
|
||||
|
||||
assert isinstance(loaded_router, DocumentLengthRouter)
|
||||
assert loaded_router.threshold == 10
|
||||
@@ -0,0 +1,301 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import pytest
|
||||
|
||||
from haystack import Pipeline
|
||||
from haystack.components.routers import DocumentTypeRouter
|
||||
from haystack.core.pipeline.base import component_from_dict, component_to_dict
|
||||
from haystack.dataclasses import Document
|
||||
|
||||
|
||||
class TestDocumentTypeRouter:
|
||||
def test_init(self):
|
||||
router = DocumentTypeRouter(
|
||||
mime_type_meta_field="mime_type",
|
||||
file_path_meta_field="file_path",
|
||||
mime_types=["text/plain", "audio/x-wav", "image/jpeg"],
|
||||
additional_mimetypes={"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx"},
|
||||
)
|
||||
assert router.mime_types == ["text/plain", "audio/x-wav", "image/jpeg"]
|
||||
assert router.mime_type_meta_field == "mime_type"
|
||||
assert router.file_path_meta_field == "file_path"
|
||||
assert router.additional_mimetypes == {
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx"
|
||||
}
|
||||
|
||||
def test_init_fail_wo_mime_types(self):
|
||||
with pytest.raises(ValueError, match="The list of mime types cannot be empty"):
|
||||
DocumentTypeRouter(mime_type_meta_field="mime_type", mime_types=[])
|
||||
|
||||
def test_init_fail_wo_meta_fields(self):
|
||||
with pytest.raises(
|
||||
ValueError, match="At least one of 'mime_type_meta_field' or 'file_path_meta_field' must be provided"
|
||||
):
|
||||
DocumentTypeRouter(mime_types=["text/plain"])
|
||||
|
||||
def test_init_with_invalid_regex(self):
|
||||
with pytest.raises(ValueError, match="Invalid regex pattern"):
|
||||
DocumentTypeRouter(mime_type_meta_field="mime_type", mime_types=["[Invalid-Regex"])
|
||||
|
||||
def test_to_dict(self):
|
||||
router = DocumentTypeRouter(
|
||||
mime_type_meta_field="mime_type",
|
||||
file_path_meta_field="file_path",
|
||||
mime_types=["text/plain", "audio/x-wav", "image/jpeg"],
|
||||
additional_mimetypes={"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx"},
|
||||
)
|
||||
expected_dict = {
|
||||
"type": "haystack.components.routers.document_type_router.DocumentTypeRouter",
|
||||
"init_parameters": {
|
||||
"mime_type_meta_field": "mime_type",
|
||||
"file_path_meta_field": "file_path",
|
||||
"mime_types": ["text/plain", "audio/x-wav", "image/jpeg"],
|
||||
"additional_mimetypes": {
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx"
|
||||
},
|
||||
},
|
||||
}
|
||||
assert component_to_dict(router, "router") == expected_dict
|
||||
|
||||
def test_from_dict(self):
|
||||
router_dict = {
|
||||
"type": "haystack.components.routers.document_type_router.DocumentTypeRouter",
|
||||
"init_parameters": {
|
||||
"mime_type_meta_field": "mime_type",
|
||||
"file_path_meta_field": "file_path",
|
||||
"mime_types": ["text/plain", "audio/x-wav", "image/jpeg"],
|
||||
"additional_mimetypes": {
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx"
|
||||
},
|
||||
},
|
||||
}
|
||||
loaded_router = component_from_dict(DocumentTypeRouter, router_dict, name="router")
|
||||
|
||||
expected_router = DocumentTypeRouter(
|
||||
mime_type_meta_field="mime_type",
|
||||
file_path_meta_field="file_path",
|
||||
mime_types=["text/plain", "audio/x-wav", "image/jpeg"],
|
||||
additional_mimetypes={"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx"},
|
||||
)
|
||||
|
||||
assert loaded_router.mime_types == expected_router.mime_types
|
||||
assert loaded_router.mime_type_meta_field == expected_router.mime_type_meta_field
|
||||
assert loaded_router.file_path_meta_field == expected_router.file_path_meta_field
|
||||
assert loaded_router.additional_mimetypes == expected_router.additional_mimetypes
|
||||
|
||||
def test_run_with_mime_type_meta_field(self):
|
||||
docs = [
|
||||
Document(content="Example text", meta={"mime_type": "text/plain"}),
|
||||
Document(content="Another document", meta={"mime_type": "application/pdf"}),
|
||||
Document(content="Audio content", meta={"mime_type": "audio/x-wav"}),
|
||||
Document(content="Unknown type", meta={"mime_type": "application/unknown"}),
|
||||
]
|
||||
|
||||
router = DocumentTypeRouter(
|
||||
mime_type_meta_field="mime_type", mime_types=["text/plain", "application/pdf", "audio/x-wav"]
|
||||
)
|
||||
result = router.run(documents=docs)
|
||||
|
||||
assert len(result["text/plain"]) == 1
|
||||
assert len(result["application/pdf"]) == 1
|
||||
assert len(result["audio/x-wav"]) == 1
|
||||
assert len(result["unclassified"]) == 1
|
||||
assert result["text/plain"][0].content == "Example text"
|
||||
assert result["application/pdf"][0].content == "Another document"
|
||||
assert result["audio/x-wav"][0].content == "Audio content"
|
||||
assert result["unclassified"][0].content == "Unknown type"
|
||||
|
||||
def test_run_with_file_path_meta_field(self):
|
||||
docs = [
|
||||
Document(content="Example text", meta={"file_path": "example.txt"}),
|
||||
Document(content="PDF document", meta={"file_path": "document.pdf"}),
|
||||
Document(content="Markdown content", meta={"file_path": "readme.md"}),
|
||||
Document(content="Unknown extension", meta={"file_path": "file.xyz"}),
|
||||
]
|
||||
|
||||
router = DocumentTypeRouter(
|
||||
file_path_meta_field="file_path", mime_types=["text/plain", "application/pdf", "text/markdown"]
|
||||
)
|
||||
result = router.run(documents=docs)
|
||||
|
||||
assert len(result["text/plain"]) == 1
|
||||
assert len(result["application/pdf"]) == 1
|
||||
assert len(result["text/markdown"]) == 1
|
||||
assert len(result["unclassified"]) == 1
|
||||
|
||||
def test_run_with_both_meta_fields(self):
|
||||
docs = [
|
||||
Document(
|
||||
content="Text with explicit mime type", meta={"mime_type": "application/pdf", "file_path": "file.txt"}
|
||||
),
|
||||
Document(content="Text inferred from path", meta={"file_path": "file.txt"}),
|
||||
]
|
||||
|
||||
router = DocumentTypeRouter(
|
||||
mime_type_meta_field="mime_type",
|
||||
file_path_meta_field="file_path",
|
||||
mime_types=["text/plain", "application/pdf"],
|
||||
)
|
||||
result = router.run(documents=docs)
|
||||
|
||||
# First doc should be classified as PDF (explicit mime type)
|
||||
assert len(result["application/pdf"]) == 1
|
||||
assert result["application/pdf"][0].content == "Text with explicit mime type"
|
||||
|
||||
# Second doc should be classified as text/plain (inferred from .txt)
|
||||
assert len(result["text/plain"]) == 1
|
||||
assert result["text/plain"][0].content == "Text inferred from path"
|
||||
|
||||
def test_run_with_missing_metadata(self):
|
||||
docs = [
|
||||
Document(content="No metadata"),
|
||||
Document(content="Empty meta", meta={}),
|
||||
Document(content="Wrong meta field", meta={"other_field": "value"}),
|
||||
]
|
||||
|
||||
router = DocumentTypeRouter(mime_type_meta_field="mime_type", mime_types=["text/plain"])
|
||||
result = router.run(documents=docs)
|
||||
|
||||
assert len(result["unclassified"]) == 3
|
||||
assert "text/plain" not in result
|
||||
|
||||
def test_run_with_regex_patterns(self):
|
||||
docs = [
|
||||
Document(content="Plain text", meta={"mime_type": "text/plain"}),
|
||||
Document(content="HTML text", meta={"mime_type": "text/html"}),
|
||||
Document(content="Markdown text", meta={"mime_type": "text/markdown"}),
|
||||
Document(content="JPEG image", meta={"mime_type": "image/jpeg"}),
|
||||
Document(content="PNG image", meta={"mime_type": "image/png"}),
|
||||
Document(content="PDF document", meta={"mime_type": "application/pdf"}),
|
||||
]
|
||||
|
||||
router = DocumentTypeRouter(mime_type_meta_field="mime_type", mime_types=[r"text/.*", r"image/.*"])
|
||||
result = router.run(documents=docs)
|
||||
|
||||
assert len(result[r"text/.*"]) == 3
|
||||
assert len(result[r"image/.*"]) == 2
|
||||
assert len(result["unclassified"]) == 1
|
||||
|
||||
def test_run_with_exact_matching(self):
|
||||
docs = [
|
||||
Document(content="Plain text", meta={"mime_type": "text/plain"}),
|
||||
Document(content="Markdown text", meta={"mime_type": "text/markdown"}),
|
||||
Document(content="PDF document", meta={"mime_type": "application/pdf"}),
|
||||
]
|
||||
|
||||
router = DocumentTypeRouter(mime_type_meta_field="mime_type", mime_types=["text/plain", "application/pdf"])
|
||||
result = router.run(documents=docs)
|
||||
|
||||
assert len(result["text/plain"]) == 1
|
||||
assert len(result["application/pdf"]) == 1
|
||||
assert len(result["unclassified"]) == 1
|
||||
assert result["unclassified"][0].content == "Markdown text"
|
||||
|
||||
def test_run_with_empty_documents_list(self):
|
||||
router = DocumentTypeRouter(mime_type_meta_field="mime_type", mime_types=["text/plain", "application/pdf"])
|
||||
result = router.run(documents=[])
|
||||
|
||||
assert len(result) == 0
|
||||
|
||||
def test_run_with_custom_mime_types(self):
|
||||
docs = [
|
||||
Document(content="Word document", meta={"file_path": "document.docx"}),
|
||||
Document(content="Markdown file", meta={"file_path": "readme.md"}),
|
||||
Document(content="Outlook message", meta={"file_path": "email.msg"}),
|
||||
]
|
||||
|
||||
router = DocumentTypeRouter(
|
||||
file_path_meta_field="file_path",
|
||||
mime_types=[
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
"text/markdown",
|
||||
"application/vnd.ms-outlook",
|
||||
],
|
||||
additional_mimetypes={"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx"},
|
||||
)
|
||||
result = router.run(documents=docs)
|
||||
|
||||
assert len(result["application/vnd.openxmlformats-officedocument.wordprocessingml.document"]) == 1
|
||||
assert len(result["text/markdown"]) == 1
|
||||
assert len(result["application/vnd.ms-outlook"]) == 1
|
||||
|
||||
def test_serde_in_pipeline(self):
|
||||
document_type_router = DocumentTypeRouter(
|
||||
mime_type_meta_field="mime_type", mime_types=["text/plain", "application/pdf"]
|
||||
)
|
||||
|
||||
pipeline = Pipeline()
|
||||
pipeline.add_component(instance=document_type_router, name="document_type_router")
|
||||
|
||||
pipeline_dict = pipeline.to_dict()
|
||||
|
||||
expected_components = {
|
||||
"document_type_router": {
|
||||
"type": "haystack.components.routers.document_type_router.DocumentTypeRouter",
|
||||
"init_parameters": {
|
||||
"mime_type_meta_field": "mime_type",
|
||||
"file_path_meta_field": None,
|
||||
"mime_types": ["text/plain", "application/pdf"],
|
||||
"additional_mimetypes": None,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
assert pipeline_dict["components"] == expected_components
|
||||
|
||||
pipeline_yaml = pipeline.dumps()
|
||||
new_pipeline = Pipeline.loads(pipeline_yaml)
|
||||
assert new_pipeline == pipeline
|
||||
|
||||
def test_run_preserves_document_metadata(self):
|
||||
docs = [
|
||||
Document(
|
||||
content="Test content",
|
||||
meta={
|
||||
"mime_type": "text/plain",
|
||||
"author": "John Doe",
|
||||
"created_at": "2023-01-01",
|
||||
"custom_field": "custom_value",
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
router = DocumentTypeRouter(mime_type_meta_field="mime_type", mime_types=["text/plain"])
|
||||
result = router.run(documents=docs)
|
||||
|
||||
classified_doc = result["text/plain"][0]
|
||||
assert classified_doc.meta["author"] == "John Doe"
|
||||
assert classified_doc.meta["created_at"] == "2023-01-01"
|
||||
assert classified_doc.meta["custom_field"] == "custom_value"
|
||||
assert classified_doc.meta["mime_type"] == "text/plain"
|
||||
|
||||
def test_run_with_multiple_matching_patterns(self):
|
||||
docs = [Document(content="Plain text file", meta={"mime_type": "text/plain"})]
|
||||
|
||||
# Both patterns should match text/plain, but only the first should be used
|
||||
router = DocumentTypeRouter(mime_type_meta_field="mime_type", mime_types=[r"text/.*", r"text/plain"])
|
||||
result = router.run(documents=docs)
|
||||
|
||||
assert len(result[r"text/.*"]) == 1
|
||||
assert r"text/plain" not in result or len(result[r"text/plain"]) == 0
|
||||
|
||||
def test_run_integration_example(self):
|
||||
docs = [
|
||||
Document(content="Example text", meta={"file_path": "example.txt"}),
|
||||
Document(content="Another document", meta={"mime_type": "application/pdf"}),
|
||||
Document(content="Unknown type"),
|
||||
]
|
||||
|
||||
router = DocumentTypeRouter(
|
||||
mime_type_meta_field="mime_type",
|
||||
file_path_meta_field="file_path",
|
||||
mime_types=["text/plain", "application/pdf"],
|
||||
)
|
||||
|
||||
result = router.run(documents=docs)
|
||||
|
||||
assert len(result["text/plain"]) == 1
|
||||
assert len(result["application/pdf"]) == 1
|
||||
assert len(result["unclassified"]) == 1
|
||||
@@ -0,0 +1,513 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import io
|
||||
import mimetypes
|
||||
import sys
|
||||
from pathlib import PosixPath
|
||||
from unittest.mock import mock_open, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from haystack import Pipeline, component
|
||||
from haystack.components.converters import PyPDFToDocument, TextFileToDocument
|
||||
from haystack.components.routers.file_type_router import FileTypeRouter
|
||||
from haystack.dataclasses import ByteStream
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
sys.platform in ["win32", "cygwin"],
|
||||
reason="Can't run on Windows Github CI, need access to registry to get mime types",
|
||||
)
|
||||
class TestFileTypeRouter:
|
||||
def test_init(self):
|
||||
"""
|
||||
Test that the component initializes correctly.
|
||||
"""
|
||||
router = FileTypeRouter(mime_types=["text/plain", "audio/x-wav", "image/jpeg"])
|
||||
assert router.mime_types == ["text/plain", "audio/x-wav", "image/jpeg"]
|
||||
assert router._additional_mimetypes is None
|
||||
|
||||
router = FileTypeRouter(
|
||||
mime_types=["text/plain"],
|
||||
additional_mimetypes={"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx"},
|
||||
)
|
||||
assert router.mime_types == ["text/plain"]
|
||||
assert router._additional_mimetypes == {
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx"
|
||||
}
|
||||
|
||||
def test_init_fail_wo_mime_types(self):
|
||||
"""
|
||||
Test that the component raises an error if no mime types are provided.
|
||||
"""
|
||||
with pytest.raises(ValueError):
|
||||
FileTypeRouter(mime_types=[])
|
||||
|
||||
def test_to_dict(self):
|
||||
router = FileTypeRouter(
|
||||
mime_types=["text/plain", "audio/x-wav", "image/jpeg"],
|
||||
additional_mimetypes={"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx"},
|
||||
)
|
||||
expected_dict = {
|
||||
"type": "haystack.components.routers.file_type_router.FileTypeRouter",
|
||||
"init_parameters": {
|
||||
"mime_types": ["text/plain", "audio/x-wav", "image/jpeg"],
|
||||
"additional_mimetypes": {
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx"
|
||||
},
|
||||
"raise_on_failure": False,
|
||||
},
|
||||
}
|
||||
assert router.to_dict() == expected_dict
|
||||
|
||||
def test_from_dict(self):
|
||||
router_dict = {
|
||||
"type": "haystack.components.routers.file_type_router.FileTypeRouter",
|
||||
"init_parameters": {
|
||||
"mime_types": ["text/plain", "audio/x-wav", "image/jpeg"],
|
||||
"additional_mimetypes": {
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx"
|
||||
},
|
||||
},
|
||||
}
|
||||
loaded_router = FileTypeRouter.from_dict(router_dict)
|
||||
|
||||
expected_router = FileTypeRouter(
|
||||
mime_types=["text/plain", "audio/x-wav", "image/jpeg"],
|
||||
additional_mimetypes={"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx"},
|
||||
)
|
||||
|
||||
assert loaded_router.mime_types == expected_router.mime_types
|
||||
assert loaded_router._additional_mimetypes == expected_router._additional_mimetypes
|
||||
|
||||
def test_run(self, test_files_path):
|
||||
"""
|
||||
Test if the component runs correctly in the simplest happy path.
|
||||
"""
|
||||
file_paths = [
|
||||
test_files_path / "txt" / "doc_1.txt",
|
||||
test_files_path / "txt" / "doc_2.txt",
|
||||
test_files_path / "audio" / "the context for this answer is here.wav",
|
||||
test_files_path / "images" / "apple.jpg",
|
||||
test_files_path / "msg" / "sample.msg",
|
||||
]
|
||||
|
||||
router = FileTypeRouter(mime_types=[r"text/plain", r"audio/x-wav", r"image/jpeg", "application/vnd.ms-outlook"])
|
||||
output = router.run(sources=file_paths)
|
||||
assert output
|
||||
assert len(output[r"text/plain"]) == 2
|
||||
assert len(output[r"audio/x-wav"]) == 1
|
||||
assert len(output[r"image/jpeg"]) == 1
|
||||
assert len(output["application/vnd.ms-outlook"]) == 1
|
||||
assert not output.get("unclassified")
|
||||
|
||||
def test_run_with_single_meta(self, test_files_path):
|
||||
"""
|
||||
Test if the component runs correctly when a single metadata dictionary is provided.
|
||||
"""
|
||||
file_paths = [
|
||||
test_files_path / "txt" / "doc_1.txt",
|
||||
test_files_path / "txt" / "doc_2.txt",
|
||||
test_files_path / "audio" / "the context for this answer is here.wav",
|
||||
]
|
||||
|
||||
meta = {"meta_field": "meta_value"}
|
||||
|
||||
router = FileTypeRouter(mime_types=[r"text/plain", r"audio/x-wav"])
|
||||
output = router.run(sources=file_paths, meta=meta)
|
||||
assert output
|
||||
|
||||
assert len(output[r"text/plain"]) == 2
|
||||
assert len(output[r"audio/x-wav"]) == 1
|
||||
assert not output.get("unclassified")
|
||||
|
||||
for elements in output.values():
|
||||
for el in elements:
|
||||
assert isinstance(el, ByteStream)
|
||||
assert el.meta["meta_field"] == "meta_value"
|
||||
|
||||
def test_run_with_meta_list(self, test_files_path):
|
||||
"""
|
||||
Test if the component runs correctly when a list of metadata dictionaries is provided.
|
||||
"""
|
||||
file_paths = [
|
||||
test_files_path / "txt" / "doc_1.txt",
|
||||
test_files_path / "images" / "apple.jpg",
|
||||
test_files_path / "audio" / "the context for this answer is here.wav",
|
||||
]
|
||||
|
||||
meta = [{"key1": "value1"}, {"key2": "value2"}, {"key3": "value3"}]
|
||||
|
||||
router = FileTypeRouter(mime_types=[r"text/plain", r"audio/x-wav", r"image/jpeg"])
|
||||
output = router.run(sources=file_paths, meta=meta)
|
||||
assert output
|
||||
|
||||
assert len(output[r"text/plain"]) == 1
|
||||
assert len(output[r"audio/x-wav"]) == 1
|
||||
assert len(output[r"image/jpeg"]) == 1
|
||||
assert not output.get("unclassified")
|
||||
|
||||
for i, elements in enumerate(output.values()):
|
||||
for el in elements:
|
||||
assert isinstance(el, ByteStream)
|
||||
expected_meta_key, expected_meta_value = list(meta[i].items())[0]
|
||||
assert el.meta[expected_meta_key] == expected_meta_value
|
||||
|
||||
def test_run_with_meta_and_bytestreams(self):
|
||||
"""
|
||||
Test if the component runs correctly with ByteStream inputs and meta.
|
||||
The original meta is preserved and the new meta is added.
|
||||
"""
|
||||
|
||||
bs = ByteStream.from_string("Haystack!", mime_type="text/plain", meta={"foo": "bar"})
|
||||
meta = {"another_key": "another_value"}
|
||||
router = FileTypeRouter(mime_types=[r"text/plain"])
|
||||
output = router.run(sources=[bs], meta=meta)
|
||||
|
||||
assert output
|
||||
assert len(output[r"text/plain"]) == 1
|
||||
assert not output.get("unclassified")
|
||||
|
||||
assert isinstance(output[r"text/plain"][0], ByteStream)
|
||||
assert output[r"text/plain"][0].meta["foo"] == "bar"
|
||||
assert output[r"text/plain"][0].meta["another_key"] == "another_value"
|
||||
|
||||
def test_run_fails_if_meta_length_does_not_match_sources(self, test_files_path):
|
||||
"""
|
||||
Test that the component raises an error if the length of the metadata list does not match the number of sources.
|
||||
"""
|
||||
file_paths = [test_files_path / "txt" / "doc_1.txt"]
|
||||
meta = [{"key1": "value1"}, {"key2": "value2"}, {"key3": "value3"}]
|
||||
router = FileTypeRouter(mime_types=[r"text/plain"])
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
router.run(sources=file_paths, meta=meta)
|
||||
|
||||
def test_run_with_bytestreams(self, test_files_path):
|
||||
"""
|
||||
Test if the component runs correctly with ByteStream inputs.
|
||||
"""
|
||||
file_paths = [
|
||||
test_files_path / "txt" / "doc_1.txt",
|
||||
test_files_path / "txt" / "doc_2.txt",
|
||||
test_files_path / "audio" / "the context for this answer is here.wav",
|
||||
test_files_path / "images" / "apple.jpg",
|
||||
]
|
||||
mime_types = [r"text/plain", r"text/plain", r"audio/x-wav", r"image/jpeg"]
|
||||
# Convert file paths to ByteStream objects and set metadata
|
||||
byte_streams = []
|
||||
for path, mime_type in zip(file_paths, mime_types, strict=True):
|
||||
byte_streams.append(ByteStream(path.read_bytes(), mime_type=mime_type))
|
||||
|
||||
# add unclassified ByteStream
|
||||
bs = ByteStream(b"unclassified content")
|
||||
bs.meta["content_type"] = "unknown_type"
|
||||
byte_streams.append(bs)
|
||||
|
||||
router = FileTypeRouter(mime_types=[r"text/plain", r"audio/x-wav", r"image/jpeg"])
|
||||
output = router.run(sources=byte_streams)
|
||||
assert output
|
||||
assert len(output[r"text/plain"]) == 2
|
||||
assert len(output[r"audio/x-wav"]) == 1
|
||||
assert len(output[r"image/jpeg"]) == 1
|
||||
assert len(output.get("unclassified")) == 1
|
||||
|
||||
def test_run_with_bytestreams_and_file_paths(self, test_files_path):
|
||||
"""
|
||||
Test if the component raises an error for unsupported data source types.
|
||||
"""
|
||||
file_paths = [
|
||||
test_files_path / "txt" / "doc_1.txt",
|
||||
test_files_path / "audio" / "the context for this answer is here.wav",
|
||||
test_files_path / "txt" / "doc_2.txt",
|
||||
test_files_path / "images" / "apple.jpg",
|
||||
test_files_path / "markdown" / "sample.md",
|
||||
]
|
||||
mime_types = [r"text/plain", r"audio/x-wav", r"text/plain", r"image/jpeg", r"text/markdown"]
|
||||
byte_stream_sources = []
|
||||
for path, mime_type in zip(file_paths, mime_types, strict=True):
|
||||
byte_stream_sources.append(ByteStream(path.read_bytes(), mime_type=mime_type))
|
||||
|
||||
mixed_sources = file_paths[:2] + byte_stream_sources[2:]
|
||||
|
||||
router = FileTypeRouter(mime_types=[r"text/plain", r"audio/x-wav", r"image/jpeg", r"text/markdown"])
|
||||
output = router.run(sources=mixed_sources)
|
||||
assert len(output[r"text/plain"]) == 2
|
||||
assert len(output[r"audio/x-wav"]) == 1
|
||||
assert len(output[r"image/jpeg"]) == 1
|
||||
assert len(output[r"text/markdown"]) == 1
|
||||
|
||||
def test_no_files(self):
|
||||
"""
|
||||
Test that the component runs correctly when no files are provided.
|
||||
"""
|
||||
router = FileTypeRouter(mime_types=[r"text/plain", r"audio/x-wav", r"image/jpeg"])
|
||||
output = router.run(sources=[])
|
||||
assert not output
|
||||
|
||||
def test_unlisted_extensions(self, test_files_path):
|
||||
"""
|
||||
Test that the component correctly handles files with non specified mime types.
|
||||
"""
|
||||
file_paths = [
|
||||
test_files_path / "txt" / "doc_1.txt",
|
||||
test_files_path / "audio" / "ignored.mp3",
|
||||
test_files_path / "audio" / "this is the content of the document.wav",
|
||||
]
|
||||
router = FileTypeRouter(mime_types=[r"text/plain"], raise_on_failure=False)
|
||||
output = router.run(sources=file_paths)
|
||||
assert len(output[r"text/plain"]) == 1
|
||||
assert "mp3" not in output
|
||||
assert len(output.get("unclassified")) == 1
|
||||
assert output.get("failed")[0].name == "ignored.mp3"
|
||||
|
||||
def test_no_extension(self, test_files_path):
|
||||
"""
|
||||
Test that the component ignores files with no extension.
|
||||
"""
|
||||
file_paths = [
|
||||
test_files_path / "txt" / "doc_1.txt",
|
||||
test_files_path / "txt" / "doc_2.txt",
|
||||
test_files_path / "txt" / "doc_4",
|
||||
]
|
||||
router = FileTypeRouter(mime_types=[r"text/plain"])
|
||||
output = router.run(sources=file_paths)
|
||||
assert len(output[r"text/plain"]) == 2
|
||||
assert len(output.get("unclassified")) == 1
|
||||
|
||||
def test_unsupported_source_type(self):
|
||||
"""
|
||||
Test if the component raises an error for unsupported data source types.
|
||||
"""
|
||||
router = FileTypeRouter(mime_types=[r"text/plain", r"audio/x-wav", r"image/jpeg"])
|
||||
with pytest.raises(TypeError, match="Unsupported data source type:"):
|
||||
router.run(sources=[{"unsupported": "type"}])
|
||||
|
||||
def test_invalid_regex_pattern(self):
|
||||
"""
|
||||
Test that the component raises a ValueError for invalid regex patterns.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="Invalid MIME type or regex pattern"):
|
||||
FileTypeRouter(mime_types=["[Invalid-Regex"])
|
||||
|
||||
def test_regex_mime_type_matching(self, test_files_path):
|
||||
"""
|
||||
Test if the component correctly matches mime types using regex.
|
||||
"""
|
||||
router = FileTypeRouter(mime_types=[r"text\/.*", r"audio\/.*", r"image\/.*"])
|
||||
file_paths = [
|
||||
test_files_path / "txt" / "doc_1.txt",
|
||||
test_files_path / "audio" / "the context for this answer is here.wav",
|
||||
test_files_path / "images" / "apple.jpg",
|
||||
]
|
||||
output = router.run(sources=file_paths)
|
||||
assert len(output[r"text\/.*"]) == 1, "Failed to match text file with regex"
|
||||
assert len(output[r"audio\/.*"]) == 1, "Failed to match audio file with regex"
|
||||
assert len(output[r"image\/.*"]) == 1, "Failed to match image file with regex"
|
||||
|
||||
@patch("pathlib.Path.open", new_callable=mock_open, read_data=b"Mock file content.")
|
||||
def test_exact_mime_type_matching(self, mock_file):
|
||||
"""
|
||||
Test if the component correctly matches mime types exactly, without regex patterns.
|
||||
"""
|
||||
txt_stream = ByteStream(io.BytesIO(b"Text file content").read(), mime_type="text/plain")
|
||||
jpg_stream = ByteStream(io.BytesIO(b"JPEG file content").read(), mime_type="image/jpeg")
|
||||
mp3_stream = ByteStream(io.BytesIO(b"MP3 file content").read(), mime_type="audio/mpeg")
|
||||
|
||||
byte_streams = [txt_stream, jpg_stream, mp3_stream]
|
||||
router = FileTypeRouter(mime_types=["text/plain", "image/jpeg"])
|
||||
output = router.run(sources=byte_streams)
|
||||
|
||||
assert len(output["text/plain"]) == 1, "Failed to match 'text/plain' MIME type exactly"
|
||||
assert txt_stream in output["text/plain"], "'doc_1.txt' ByteStream not correctly classified as 'text/plain'"
|
||||
|
||||
assert len(output["image/jpeg"]) == 1, "Failed to match 'image/jpeg' MIME type exactly"
|
||||
assert jpg_stream in output["image/jpeg"], "'apple.jpg' ByteStream not correctly classified as 'image/jpeg'"
|
||||
|
||||
assert len(output.get("unclassified")) == 1, "Failed to handle unclassified file types"
|
||||
assert mp3_stream in output["unclassified"], "'sound.mp3' ByteStream should be unclassified but is not"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"literal_mime",
|
||||
[
|
||||
"image/svg+xml",
|
||||
"application/ld+json",
|
||||
"application/atom+xml",
|
||||
"application/vnd.api+json",
|
||||
"application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
],
|
||||
)
|
||||
def test_literal_mime_with_regex_metacharacters_matches_self(self, literal_mime):
|
||||
"""MIME types with regex metacharacters (`+`, `.`) match themselves via the equality short-circuit."""
|
||||
router = FileTypeRouter(mime_types=[literal_mime])
|
||||
bs = ByteStream(b"x", mime_type=literal_mime)
|
||||
output = router.run(sources=[bs])
|
||||
|
||||
assert output[literal_mime] == [bs]
|
||||
assert "unclassified" not in output
|
||||
|
||||
def test_to_dict_from_dict_preserves_literal_and_regex_mix(self):
|
||||
"""A literal + regex mix survives serde and still routes correctly."""
|
||||
router = FileTypeRouter(mime_types=["image/svg+xml", r"audio/.*"])
|
||||
|
||||
loaded = FileTypeRouter.from_dict(router.to_dict())
|
||||
assert loaded.mime_types == ["image/svg+xml", r"audio/.*"]
|
||||
|
||||
svg = ByteStream(b"<svg/>", mime_type="image/svg+xml")
|
||||
wav = ByteStream(b"x", mime_type="audio/x-wav")
|
||||
output = loaded.run(sources=[svg, wav])
|
||||
|
||||
assert output["image/svg+xml"] == [svg]
|
||||
assert output[r"audio/.*"] == [wav]
|
||||
assert "unclassified" not in output
|
||||
|
||||
def test_pipeline_output_socket_name_matches_literal_mime_with_plus(self):
|
||||
"""A downstream component wired to `router.image/svg+xml` actually receives the SVG stream."""
|
||||
|
||||
@component
|
||||
class _Sink:
|
||||
@component.output_types(received=list)
|
||||
def run(self, value: list) -> dict:
|
||||
return {"received": value}
|
||||
|
||||
pipe = Pipeline()
|
||||
pipe.add_component(instance=FileTypeRouter(mime_types=["image/svg+xml"]), name="router")
|
||||
pipe.add_component(instance=_Sink(), name="sink")
|
||||
pipe.connect("router.image/svg+xml", "sink.value")
|
||||
|
||||
svg = ByteStream(b"<svg/>", mime_type="image/svg+xml")
|
||||
output = pipe.run(data={"router": {"sources": [svg]}})
|
||||
|
||||
assert output["sink"]["received"] == [svg]
|
||||
|
||||
def test_additional_mimetypes_with_literal_plus(self, tmp_path):
|
||||
"""Custom `additional_mimetypes` registration still works for `+`-containing literal MIMEs."""
|
||||
custom_mime = "application/x-custom+xml"
|
||||
custom_extension = ".cxml"
|
||||
test_file = tmp_path / f"test{custom_extension}"
|
||||
test_file.touch()
|
||||
|
||||
router = FileTypeRouter(mime_types=[custom_mime], additional_mimetypes={custom_mime: custom_extension})
|
||||
output = router.run(sources=[test_file])
|
||||
|
||||
assert custom_mime in output
|
||||
assert test_file in output[custom_mime]
|
||||
assert "unclassified" not in output
|
||||
|
||||
def test_serde_in_pipeline(self):
|
||||
"""
|
||||
Test if a pipeline containing the component can be serialized and deserialized without errors.
|
||||
"""
|
||||
|
||||
file_type_router = FileTypeRouter(mime_types=["text/plain", "application/pdf"])
|
||||
|
||||
pipeline = Pipeline()
|
||||
pipeline.add_component(instance=file_type_router, name="file_type_router")
|
||||
|
||||
pipeline_dict = pipeline.to_dict()
|
||||
|
||||
assert pipeline_dict == {
|
||||
"metadata": {},
|
||||
"max_runs_per_component": 100,
|
||||
"connection_type_validation": True,
|
||||
"components": {
|
||||
"file_type_router": {
|
||||
"type": "haystack.components.routers.file_type_router.FileTypeRouter",
|
||||
"init_parameters": {
|
||||
"mime_types": ["text/plain", "application/pdf"],
|
||||
"additional_mimetypes": None,
|
||||
"raise_on_failure": False,
|
||||
},
|
||||
}
|
||||
},
|
||||
"connections": [],
|
||||
}
|
||||
|
||||
pipeline_yaml = pipeline.dumps()
|
||||
|
||||
new_pipeline = Pipeline.loads(pipeline_yaml)
|
||||
assert new_pipeline == pipeline
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_pipeline_with_converters(self, test_files_path):
|
||||
"""
|
||||
Test if the component runs correctly in a pipeline with converters and passes metadata correctly.
|
||||
"""
|
||||
file_type_router = FileTypeRouter(
|
||||
mime_types=["text/plain", "application/pdf"],
|
||||
additional_mimetypes={"application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx"},
|
||||
)
|
||||
|
||||
pipe = Pipeline()
|
||||
pipe.add_component(instance=file_type_router, name="file_type_router")
|
||||
pipe.add_component(instance=TextFileToDocument(), name="text_file_converter")
|
||||
pipe.add_component(instance=PyPDFToDocument(), name="pypdf_converter")
|
||||
pipe.connect("file_type_router.text/plain", "text_file_converter.sources")
|
||||
pipe.connect("file_type_router.application/pdf", "pypdf_converter.sources")
|
||||
|
||||
file_paths = [test_files_path / "txt" / "doc_1.txt", test_files_path / "pdf" / "sample_pdf_1.pdf"]
|
||||
|
||||
meta = [{"meta_field_1": "meta_value_1"}, {"meta_field_2": "meta_value_2"}]
|
||||
|
||||
output = pipe.run(data={"file_type_router": {"sources": file_paths, "meta": meta}})
|
||||
|
||||
assert output["text_file_converter"]["documents"][0].meta["meta_field_1"] == "meta_value_1"
|
||||
assert output["pypdf_converter"]["documents"][0].meta["meta_field_2"] == "meta_value_2"
|
||||
|
||||
def test_additional_mimetypes_integration(self, tmp_path):
|
||||
"""
|
||||
Test if the component runs correctly in a pipeline with additional mimetypes correctly.
|
||||
"""
|
||||
custom_mime_type = "application/x-spam"
|
||||
custom_extension = ".spam"
|
||||
test_file = tmp_path / f"test.{custom_extension}"
|
||||
test_file.touch()
|
||||
|
||||
# confirm that mimetypes module doesn't know about this extension by default
|
||||
assert custom_mime_type not in mimetypes.types_map.values()
|
||||
|
||||
# make haystack aware of the custom mime type
|
||||
router = FileTypeRouter(
|
||||
mime_types=[custom_mime_type], additional_mimetypes={custom_mime_type: custom_extension}
|
||||
)
|
||||
mappings = router.run(sources=[test_file])
|
||||
|
||||
# assert the file was classified under the custom mime type
|
||||
assert custom_mime_type in mappings
|
||||
assert test_file in mappings[custom_mime_type]
|
||||
|
||||
def test_non_existent_file(self):
|
||||
"""
|
||||
Test conditional FileNotFoundError behavior in FileTypeRouter.
|
||||
"""
|
||||
router = FileTypeRouter(mime_types=[r"text/plain"], raise_on_failure=True)
|
||||
|
||||
# no metadata
|
||||
with pytest.raises(FileNotFoundError):
|
||||
router.run(sources=["non_existent.txt"])
|
||||
|
||||
# with metadata
|
||||
with pytest.raises(FileNotFoundError):
|
||||
router.run(sources=["non_existent.txt"], meta={"spam": "eggs"})
|
||||
|
||||
def test_logging_for_non_existent_file(self, caplog: pytest.LogCaptureFixture):
|
||||
"""
|
||||
Test that a logging warning is triggered when a non-existent file is encountered
|
||||
and raise_on_failure is False.
|
||||
"""
|
||||
import logging
|
||||
|
||||
router = FileTypeRouter(mime_types=[r"text/plain"], raise_on_failure=False)
|
||||
|
||||
# Capture log messages to verify they are triggered
|
||||
with caplog.at_level(logging.WARNING):
|
||||
result = router.run(sources=["non_existent_file.txt"])
|
||||
assert "File not found:" in caplog.text
|
||||
assert "Skipping it." in caplog.text
|
||||
|
||||
# Verify the file is added to the "failed" category
|
||||
assert "failed" in result
|
||||
assert PosixPath("non_existent_file.txt") in result["failed"]
|
||||
assert "text/plain" not in result
|
||||
@@ -0,0 +1,347 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import os
|
||||
import re
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from haystack.components.generators.chat.openai import OpenAIChatGenerator
|
||||
from haystack.components.routers.llm_messages_router import LLMMessagesRouter
|
||||
from haystack.core.serialization import default_from_dict, default_to_dict
|
||||
from haystack.dataclasses import ChatMessage
|
||||
|
||||
|
||||
class MockChatGenerator:
|
||||
def __init__(self, return_text: str = "safe"):
|
||||
self.return_text = return_text
|
||||
|
||||
def run(self, messages: list[ChatMessage]) -> dict[str, Any]:
|
||||
return {"replies": [ChatMessage.from_assistant(self.return_text)]}
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return default_to_dict(self, return_text=self.return_text)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "MockChatGenerator":
|
||||
return default_from_dict(cls, data)
|
||||
|
||||
|
||||
class TestLLMMessagesRouter:
|
||||
def test_init(self):
|
||||
system_prompt = "Classify the messages as safe or unsafe."
|
||||
chat_generator = MockChatGenerator()
|
||||
|
||||
router = LLMMessagesRouter(
|
||||
chat_generator=chat_generator,
|
||||
system_prompt=system_prompt,
|
||||
output_names=["safe", "unsafe"],
|
||||
output_patterns=["safe", "unsafe"],
|
||||
)
|
||||
|
||||
assert router._chat_generator is chat_generator
|
||||
assert router._system_prompt == system_prompt
|
||||
assert router._output_names == ["safe", "unsafe"]
|
||||
assert router._output_patterns == ["safe", "unsafe"]
|
||||
assert router._compiled_patterns == [re.compile(pattern) for pattern in ["safe", "unsafe"]]
|
||||
|
||||
def test_init_errors(self):
|
||||
chat_generator = MockChatGenerator()
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
LLMMessagesRouter(chat_generator=chat_generator, output_names=[], output_patterns=["pattern1", "pattern2"])
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
LLMMessagesRouter(chat_generator=chat_generator, output_names=["name1", "name2"], output_patterns=[])
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
LLMMessagesRouter(
|
||||
chat_generator=chat_generator, output_names=["name1", "name2"], output_patterns=["pattern1"]
|
||||
)
|
||||
|
||||
def test_run_input_errors(self):
|
||||
router = LLMMessagesRouter(
|
||||
chat_generator=MockChatGenerator(), output_names=["safe", "unsafe"], output_patterns=["safe", "unsafe"]
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
router.run([])
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
router.run([ChatMessage.from_system("You are a helpful assistant.")])
|
||||
|
||||
def test_run_no_warm_up_with_unwarmable_chat_generator(self):
|
||||
router = LLMMessagesRouter(
|
||||
chat_generator=MockChatGenerator(), output_names=["safe", "unsafe"], output_patterns=["safe", "unsafe"]
|
||||
)
|
||||
|
||||
router.run([ChatMessage.from_user("Hello")])
|
||||
|
||||
def test_run_no_warm_up_with_warmable_chat_generator(self):
|
||||
def mock_run(messages):
|
||||
return {"replies": [ChatMessage.from_assistant("safe")]}
|
||||
|
||||
chat_generator = Mock()
|
||||
chat_generator.run = mock_run
|
||||
router = LLMMessagesRouter(
|
||||
chat_generator=chat_generator, output_names=["safe", "unsafe"], output_patterns=["safe", "unsafe"]
|
||||
)
|
||||
router.run([ChatMessage.from_user("Hello")])
|
||||
assert chat_generator.warm_up.call_count == 1
|
||||
|
||||
def test_run(self):
|
||||
router = LLMMessagesRouter(
|
||||
chat_generator=MockChatGenerator(return_text="safe"),
|
||||
output_names=["safe", "unsafe"],
|
||||
output_patterns=["safe", "unsafe"],
|
||||
)
|
||||
|
||||
messages = [ChatMessage.from_user("Hello")]
|
||||
result = router.run(messages)
|
||||
|
||||
assert result["chat_generator_text"] == "safe"
|
||||
assert result["safe"] == messages
|
||||
assert "unsafe" not in result
|
||||
assert "unmatched" not in result
|
||||
|
||||
def test_run_with_system_prompt(self):
|
||||
chat_generator = Mock()
|
||||
chat_generator.run.return_value = {"replies": [ChatMessage.from_assistant("safe")]}
|
||||
|
||||
system_prompt = "Classify the messages as safe or unsafe."
|
||||
|
||||
router = LLMMessagesRouter(
|
||||
chat_generator=chat_generator,
|
||||
output_names=["safe", "unsafe"],
|
||||
output_patterns=["safe", "unsafe"],
|
||||
system_prompt=system_prompt,
|
||||
)
|
||||
|
||||
messages = [ChatMessage.from_user("Hello")]
|
||||
router.run(messages)
|
||||
|
||||
chat_generator.run.assert_called_once_with(messages=[ChatMessage.from_system(system_prompt)] + messages)
|
||||
|
||||
def test_run_unmatched_output(self):
|
||||
router = LLMMessagesRouter(
|
||||
chat_generator=MockChatGenerator(return_text="irrelevant"),
|
||||
output_names=["safe", "unsafe"],
|
||||
output_patterns=["safe", "unsafe"],
|
||||
)
|
||||
|
||||
messages = [ChatMessage.from_user("Hello")]
|
||||
result = router.run(messages)
|
||||
|
||||
assert result["chat_generator_text"] == "irrelevant"
|
||||
assert result["unmatched"] == messages
|
||||
assert "safe" not in result
|
||||
assert "unsafe" not in result
|
||||
|
||||
def test_to_dict(self):
|
||||
chat_generator = MockChatGenerator(return_text="safe")
|
||||
|
||||
router = LLMMessagesRouter(
|
||||
chat_generator=chat_generator, output_names=["safe", "unsafe"], output_patterns=["safe", "unsafe"]
|
||||
)
|
||||
|
||||
result = router.to_dict()
|
||||
|
||||
assert result["type"] == "haystack.components.routers.llm_messages_router.LLMMessagesRouter"
|
||||
assert result["init_parameters"]["chat_generator"] == chat_generator.to_dict()
|
||||
assert result["init_parameters"]["output_names"] == ["safe", "unsafe"]
|
||||
assert result["init_parameters"]["output_patterns"] == ["safe", "unsafe"]
|
||||
assert result["init_parameters"]["system_prompt"] is None
|
||||
|
||||
def test_from_dict(self):
|
||||
chat_generator = MockChatGenerator(return_text="safe")
|
||||
|
||||
data = {
|
||||
"type": "haystack.components.routers.llm_messages_router.LLMMessagesRouter",
|
||||
"init_parameters": {
|
||||
"chat_generator": chat_generator.to_dict(),
|
||||
"output_names": ["safe", "unsafe"],
|
||||
"output_patterns": ["safe", "unsafe"],
|
||||
"system_prompt": None,
|
||||
},
|
||||
}
|
||||
|
||||
router = LLMMessagesRouter.from_dict(data)
|
||||
|
||||
assert router._chat_generator.to_dict() == chat_generator.to_dict()
|
||||
assert router._output_names == ["safe", "unsafe"]
|
||||
assert router._output_patterns == ["safe", "unsafe"]
|
||||
assert router._system_prompt is None
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.skipif(
|
||||
not os.environ.get("OPENAI_API_KEY", None),
|
||||
reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.",
|
||||
)
|
||||
def test_live_run(self):
|
||||
system_prompt = "Classify the messages into safe or unsafe. Respond with the label only, no other text."
|
||||
router = LLMMessagesRouter(
|
||||
chat_generator=OpenAIChatGenerator(model="gpt-4.1-nano"),
|
||||
system_prompt=system_prompt,
|
||||
output_names=["safe", "unsafe"],
|
||||
output_patterns=[r"(?i)safe", r"(?i)unsafe"],
|
||||
)
|
||||
|
||||
messages = [ChatMessage.from_user("Hello")]
|
||||
result = router.run(messages)
|
||||
print(result)
|
||||
|
||||
assert result["safe"] == messages
|
||||
assert result["chat_generator_text"].lower() == "safe"
|
||||
assert "unsafe" not in result
|
||||
assert "unmatched" not in result
|
||||
|
||||
|
||||
class TestLLMMessagesRouterAsync:
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_matched_output(self):
|
||||
chat_generator = Mock(spec=OpenAIChatGenerator)
|
||||
chat_generator.run_async = AsyncMock(return_value={"replies": [ChatMessage.from_assistant("safe")]})
|
||||
router = LLMMessagesRouter(
|
||||
chat_generator=chat_generator, output_names=["safe", "unsafe"], output_patterns=["safe", "unsafe"]
|
||||
)
|
||||
|
||||
messages = [ChatMessage.from_user("Hello")]
|
||||
result = await router.run_async(messages)
|
||||
|
||||
assert result["chat_generator_text"] == "safe"
|
||||
assert result["safe"] == messages
|
||||
assert "unsafe" not in result
|
||||
assert "unmatched" not in result
|
||||
chat_generator.run_async.assert_awaited_once()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_unmatched_output(self):
|
||||
chat_generator = Mock(spec=OpenAIChatGenerator)
|
||||
chat_generator.run_async = AsyncMock(return_value={"replies": [ChatMessage.from_assistant("irrelevant")]})
|
||||
router = LLMMessagesRouter(
|
||||
chat_generator=chat_generator, output_names=["safe", "unsafe"], output_patterns=["safe", "unsafe"]
|
||||
)
|
||||
|
||||
messages = [ChatMessage.from_user("Hello")]
|
||||
result = await router.run_async(messages)
|
||||
|
||||
assert result["chat_generator_text"] == "irrelevant"
|
||||
assert result["unmatched"] == messages
|
||||
assert "safe" not in result
|
||||
assert "unsafe" not in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_fallback_to_sync_run(self):
|
||||
# MockChatGenerator defines only a synchronous `run`, so the utility falls back to it.
|
||||
chat_generator = MockChatGenerator(return_text="safe")
|
||||
assert not hasattr(chat_generator, "run_async")
|
||||
router = LLMMessagesRouter(
|
||||
chat_generator=chat_generator, output_names=["safe", "unsafe"], output_patterns=["safe", "unsafe"]
|
||||
)
|
||||
|
||||
messages = [ChatMessage.from_user("Hello")]
|
||||
result = await router.run_async(messages)
|
||||
|
||||
assert result["chat_generator_text"] == "safe"
|
||||
assert result["safe"] == messages
|
||||
assert "unsafe" not in result
|
||||
assert "unmatched" not in result
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_empty_messages_raises(self):
|
||||
chat_generator = Mock(spec=OpenAIChatGenerator)
|
||||
chat_generator.run_async = AsyncMock(return_value={"replies": [ChatMessage.from_assistant("safe")]})
|
||||
router = LLMMessagesRouter(
|
||||
chat_generator=chat_generator, output_names=["safe", "unsafe"], output_patterns=["safe", "unsafe"]
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
await router.run_async([])
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_unsupported_role_raises(self):
|
||||
chat_generator = Mock(spec=OpenAIChatGenerator)
|
||||
chat_generator.run_async = AsyncMock(return_value={"replies": [ChatMessage.from_assistant("safe")]})
|
||||
router = LLMMessagesRouter(
|
||||
chat_generator=chat_generator, output_names=["safe", "unsafe"], output_patterns=["safe", "unsafe"]
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
await router.run_async([ChatMessage.from_system("You are a helpful assistant.")])
|
||||
|
||||
@pytest.mark.integration
|
||||
@pytest.mark.skipif(
|
||||
not os.environ.get("OPENAI_API_KEY", None),
|
||||
reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.",
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_run_async(self):
|
||||
system_prompt = "Classify the messages into safe or unsafe. Respond with the label only, no other text."
|
||||
router = LLMMessagesRouter(
|
||||
chat_generator=OpenAIChatGenerator(model="gpt-4.1-nano"),
|
||||
system_prompt=system_prompt,
|
||||
output_names=["safe", "unsafe"],
|
||||
output_patterns=[r"(?i)safe", r"(?i)unsafe"],
|
||||
)
|
||||
|
||||
messages = [ChatMessage.from_user("Hello")]
|
||||
result = await router.run_async(messages)
|
||||
|
||||
assert result["safe"] == messages
|
||||
assert result["chat_generator_text"].lower() == "safe"
|
||||
assert "unsafe" not in result
|
||||
assert "unmatched" not in result
|
||||
|
||||
|
||||
class TestComponentLifecycle:
|
||||
def _make_router(self, chat_generator):
|
||||
return LLMMessagesRouter(
|
||||
chat_generator=chat_generator, output_names=["safe", "unsafe"], output_patterns=["safe", "unsafe"]
|
||||
)
|
||||
|
||||
def test_warm_up_delegates_to_chat_generator(self):
|
||||
chat_generator = Mock()
|
||||
router = self._make_router(chat_generator)
|
||||
router.warm_up()
|
||||
chat_generator.warm_up.assert_called_once()
|
||||
|
||||
async def test_warm_up_async_delegates_to_chat_generator(self):
|
||||
chat_generator = Mock()
|
||||
chat_generator.warm_up_async = AsyncMock()
|
||||
router = self._make_router(chat_generator)
|
||||
await router.warm_up_async()
|
||||
chat_generator.warm_up_async.assert_awaited_once()
|
||||
|
||||
async def test_warm_up_async_falls_back_to_sync_warm_up(self):
|
||||
chat_generator = Mock(spec=["run", "warm_up"])
|
||||
router = self._make_router(chat_generator)
|
||||
await router.warm_up_async()
|
||||
chat_generator.warm_up.assert_called_once()
|
||||
|
||||
def test_close_delegates_to_chat_generator(self):
|
||||
chat_generator = Mock()
|
||||
router = self._make_router(chat_generator)
|
||||
router.close()
|
||||
chat_generator.close.assert_called_once()
|
||||
|
||||
async def test_close_async_delegates_to_chat_generator(self):
|
||||
chat_generator = Mock()
|
||||
chat_generator.close_async = AsyncMock()
|
||||
router = self._make_router(chat_generator)
|
||||
await router.close_async()
|
||||
chat_generator.close_async.assert_awaited_once()
|
||||
|
||||
async def test_close_async_falls_back_to_sync_close(self):
|
||||
chat_generator = Mock(spec=["run", "close"])
|
||||
router = self._make_router(chat_generator)
|
||||
await router.close_async()
|
||||
chat_generator.close.assert_called_once()
|
||||
|
||||
def test_lifecycle_is_safe_when_chat_generator_lacks_methods(self):
|
||||
chat_generator = Mock(spec=["run"])
|
||||
router = self._make_router(chat_generator)
|
||||
router.warm_up()
|
||||
router.close()
|
||||
@@ -0,0 +1,207 @@
|
||||
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
||||
#
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
import pytest
|
||||
|
||||
from haystack import Pipeline
|
||||
from haystack.components.routers.metadata_router import MetadataRouter
|
||||
from haystack.components.writers import DocumentWriter
|
||||
from haystack.dataclasses import ByteStream, Document
|
||||
|
||||
|
||||
class TestMetadataRouter:
|
||||
def test_run(self):
|
||||
rules = {
|
||||
"edge_1": {
|
||||
"operator": "AND",
|
||||
"conditions": [
|
||||
{"field": "meta.created_at", "operator": ">=", "value": "2023-01-01"},
|
||||
{"field": "meta.created_at", "operator": "<", "value": "2023-04-01"},
|
||||
],
|
||||
},
|
||||
"edge_2": {
|
||||
"operator": "AND",
|
||||
"conditions": [
|
||||
{"field": "meta.created_at", "operator": ">=", "value": "2023-04-01"},
|
||||
{"field": "meta.created_at", "operator": "<", "value": "2023-07-01"},
|
||||
],
|
||||
},
|
||||
}
|
||||
router = MetadataRouter(rules=rules)
|
||||
documents = [
|
||||
Document(meta={"created_at": "2023-02-01"}),
|
||||
Document(meta={"created_at": "2023-05-01"}),
|
||||
Document(meta={"created_at": "2023-08-01"}),
|
||||
]
|
||||
output = router.run(documents=documents)
|
||||
assert output["edge_1"][0].meta["created_at"] == "2023-02-01"
|
||||
assert output["edge_2"][0].meta["created_at"] == "2023-05-01"
|
||||
assert output["unmatched"][0].meta["created_at"] == "2023-08-01"
|
||||
|
||||
def test_run_with_byte_stream(self):
|
||||
byt1 = ByteStream.from_string(text="What is this", meta={"language": "en"})
|
||||
byt2 = ByteStream.from_string(text="Berlin ist die Haupststadt von Deutschland.", meta={"language": "de"})
|
||||
docs = [byt1, byt2]
|
||||
router = MetadataRouter(
|
||||
rules={"en": {"field": "meta.language", "operator": "==", "value": "en"}}, output_type=list[ByteStream]
|
||||
)
|
||||
output = router.run(documents=docs)
|
||||
assert output["en"][0].data == byt1.data
|
||||
assert output["unmatched"][0].data == byt2.data
|
||||
|
||||
def test_run_with_mixed_documents_and_byte_streams(self):
|
||||
byt1 = ByteStream.from_string(text="What is this", meta={"language": "en"})
|
||||
byt2 = ByteStream.from_string(text="Berlin ist die Haupststadt von Deutschland.", meta={"language": "de"})
|
||||
doc1 = Document(content="What is this", meta={"language": "en"})
|
||||
doc2 = Document(content="Berlin ist die Haupststadt von Deutschland.", meta={"language": "de"})
|
||||
docs = [byt1, byt2, doc1, doc2]
|
||||
router = MetadataRouter(
|
||||
rules={"en": {"field": "meta.language", "operator": "==", "value": "en"}},
|
||||
output_type=list[Document | ByteStream],
|
||||
)
|
||||
output = router.run(documents=docs)
|
||||
assert output["en"][0].data == byt1.data
|
||||
assert output["en"][1].content == "What is this"
|
||||
assert output["unmatched"][0].data == byt2.data
|
||||
assert output["unmatched"][1].content == "Berlin ist die Haupststadt von Deutschland."
|
||||
|
||||
def test_run_wrong_filter(self):
|
||||
rules = {
|
||||
"edge_1": {"field": "meta.created_at", "operator": ">=", "value": "2023-01-01"},
|
||||
"wrong_filter": {"wrong_value": "meta.created_at == 2023-04-01"},
|
||||
}
|
||||
with pytest.raises(ValueError):
|
||||
MetadataRouter(rules=rules)
|
||||
|
||||
def test_run_datetime_with_timezone(self):
|
||||
rules = {
|
||||
"edge_1": {
|
||||
"operator": "AND",
|
||||
"conditions": [{"field": "meta.created_at", "operator": ">=", "value": "2025-02-01"}],
|
||||
}
|
||||
}
|
||||
router = MetadataRouter(rules=rules)
|
||||
documents = [
|
||||
Document(meta={"created_at": "2025-02-03T12:45:46.435816Z"}),
|
||||
Document(meta={"created_at": "2025-02-01T12:45:46.435816Z"}),
|
||||
Document(meta={"created_at": "2025-01-03T12:45:46.435816Z"}),
|
||||
]
|
||||
output = router.run(documents=documents)
|
||||
assert len(output["edge_1"]) == 2
|
||||
assert output["edge_1"][0].meta["created_at"] == "2025-02-03T12:45:46.435816Z"
|
||||
assert output["edge_1"][1].meta["created_at"] == "2025-02-01T12:45:46.435816Z"
|
||||
assert output["unmatched"][0].meta["created_at"] == "2025-01-03T12:45:46.435816Z"
|
||||
|
||||
def test_to_dict(self):
|
||||
rules = {
|
||||
"edge_1": {
|
||||
"operator": "AND",
|
||||
"conditions": [{"field": "meta.created_at", "operator": ">=", "value": "2025-02-01"}],
|
||||
}
|
||||
}
|
||||
router = MetadataRouter(rules=rules)
|
||||
expected_dict = {
|
||||
"type": "haystack.components.routers.metadata_router.MetadataRouter",
|
||||
"init_parameters": {"rules": rules, "output_type": "list[haystack.dataclasses.document.Document]"},
|
||||
}
|
||||
assert router.to_dict() == expected_dict
|
||||
|
||||
def test_to_dict_with_parameters(self):
|
||||
rules = {
|
||||
"edge_1": {
|
||||
"operator": "AND",
|
||||
"conditions": [{"field": "meta.created_at", "operator": ">=", "value": "2025-02-01"}],
|
||||
}
|
||||
}
|
||||
router = MetadataRouter(rules=rules, output_type=list[ByteStream | Document])
|
||||
expected_dict = {
|
||||
"type": "haystack.components.routers.metadata_router.MetadataRouter",
|
||||
"init_parameters": {
|
||||
"rules": rules,
|
||||
"output_type": "list[haystack.dataclasses.byte_stream.ByteStream "
|
||||
"| haystack.dataclasses.document.Document]",
|
||||
},
|
||||
}
|
||||
assert router.to_dict() == expected_dict
|
||||
|
||||
def test_from_dict(self):
|
||||
router_dict = {
|
||||
"type": "haystack.components.routers.metadata_router.MetadataRouter",
|
||||
"init_parameters": {
|
||||
"rules": {
|
||||
"edge_1": {
|
||||
"operator": "AND",
|
||||
"conditions": [{"field": "meta.created_at", "operator": ">=", "value": "2025-02-01"}],
|
||||
}
|
||||
},
|
||||
"output_type": "list[haystack.dataclasses.document.Document]",
|
||||
},
|
||||
}
|
||||
router = MetadataRouter.from_dict(router_dict)
|
||||
assert router.rules == {
|
||||
"edge_1": {
|
||||
"operator": "AND",
|
||||
"conditions": [{"field": "meta.created_at", "operator": ">=", "value": "2025-02-01"}],
|
||||
}
|
||||
}
|
||||
assert router.output_type == list[Document]
|
||||
|
||||
def test_from_dict_with_parameters(self):
|
||||
router_dict = {
|
||||
"type": "haystack.components.routers.metadata_router.MetadataRouter",
|
||||
"init_parameters": {
|
||||
"rules": {
|
||||
"edge_1": {
|
||||
"operator": "AND",
|
||||
"conditions": [{"field": "meta.created_at", "operator": ">=", "value": "2025-02-01"}],
|
||||
}
|
||||
},
|
||||
"output_type": "list[typing.Union[haystack.dataclasses.document.Document, "
|
||||
"haystack.dataclasses.byte_stream.ByteStream]]",
|
||||
},
|
||||
}
|
||||
router = MetadataRouter.from_dict(router_dict)
|
||||
assert router.rules == {
|
||||
"edge_1": {
|
||||
"operator": "AND",
|
||||
"conditions": [{"field": "meta.created_at", "operator": ">=", "value": "2025-02-01"}],
|
||||
}
|
||||
}
|
||||
assert router.output_type == list[ByteStream | Document]
|
||||
|
||||
def test_from_dict_no_output_type(self):
|
||||
router_dict = {
|
||||
"type": "haystack.components.routers.metadata_router.MetadataRouter",
|
||||
"init_parameters": {
|
||||
"rules": {
|
||||
"edge_1": {
|
||||
"operator": "AND",
|
||||
"conditions": [{"field": "meta.created_at", "operator": ">=", "value": "2025-02-01"}],
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
router = MetadataRouter.from_dict(router_dict)
|
||||
assert router.rules == {
|
||||
"edge_1": {
|
||||
"operator": "AND",
|
||||
"conditions": [{"field": "meta.created_at", "operator": ">=", "value": "2025-02-01"}],
|
||||
}
|
||||
}
|
||||
assert router.output_type == list[Document]
|
||||
|
||||
def test_metadata_router_in_pipeline(self, in_memory_doc_store):
|
||||
p = Pipeline()
|
||||
docs = [
|
||||
Document(content="Hello, welcome to the world of Haystack!", meta={"language": "en"}),
|
||||
Document(content="Hallo, willkommen in der Welt von Haystack!", meta={"language": "de"}),
|
||||
]
|
||||
p.add_component(
|
||||
instance=MetadataRouter(rules={"en": {"field": "meta.language", "operator": "==", "value": "en"}}),
|
||||
name="router",
|
||||
)
|
||||
p.add_component(instance=DocumentWriter(document_store=in_memory_doc_store), name="writer")
|
||||
p.connect("router.en", "writer.documents")
|
||||
p.run({"router": {"documents": docs}})
|
||||
assert in_memory_doc_store.filter_documents() == [docs[0]]
|
||||
Reference in New Issue
Block a user