chore: import upstream snapshot with attribution

test/components/converters/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0

test/components/converters/image/__init__.py

@@ -0,0 +1,3 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0

test/components/converters/image/test_document_to_image_content.py

@@ -0,0 +1,143 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from unittest.mock import patch
+
+import pytest
+from PIL import Image
+
+from haystack import Document
+from haystack.components.converters.image.document_to_image import DocumentToImageContent
+from haystack.core.serialization import component_from_dict, component_to_dict
+from haystack.dataclasses import ByteStream
+
+
+class TestDocumentToImageContent:
+    def test_to_dict(self) -> None:
+        converter = DocumentToImageContent()
+        assert component_to_dict(converter, "converter") == {
+            "init_parameters": {"file_path_meta_field": "file_path", "root_path": "", "detail": None, "size": None},
+            "type": "haystack.components.converters.image.document_to_image.DocumentToImageContent",
+        }
+
+    def test_to_dict_not_defaults(self) -> None:
+        converter = DocumentToImageContent(
+            file_path_meta_field="image_path", root_path="/data", detail="high", size=(800, 600)
+        )
+        assert component_to_dict(converter, "converter") == {
+            "init_parameters": {
+                "file_path_meta_field": "image_path",
+                "root_path": "/data",
+                "detail": "high",
+                "size": (800, 600),
+            },
+            "type": "haystack.components.converters.image.document_to_image.DocumentToImageContent",
+        }
+
+    def test_from_dict(self) -> None:
+        data = {
+            "init_parameters": {
+                "file_path_meta_field": "image_path",
+                "root_path": "/test",
+                "detail": "auto",
+                "size": (512, 512),
+            },
+            "type": "haystack.components.converters.image.document_to_image.DocumentToImageContent",
+        }
+        converter = component_from_dict(DocumentToImageContent, data, "name")
+        assert component_to_dict(converter, "converter") == data
+
+    def test_run_with_empty_documents_list(self) -> None:
+        converter = DocumentToImageContent()
+        results = converter.run(documents=[])
+        assert results == {"image_contents": []}
+
+    def test_run_with_missing_file_path_metadata(self) -> None:
+        converter = DocumentToImageContent()
+        # Document without file_path in metadata
+        doc_no_path = Document(content="test", meta={})
+        # Document with file_path but file doesn't exist
+        doc_no_file = Document(content="test", meta={"file_path": "nonexistent.jpg"})
+        with pytest.raises(ValueError, match="is missing the 'file_path' key"):
+            _ = converter.run(documents=[doc_no_path, doc_no_file])
+
+    def test_run_with_non_image_documents(self) -> None:
+        converter = DocumentToImageContent()
+        docx_doc = Document(content="test", meta={"file_path": "test/test_files/docx/sample_docx.docx"})
+        with pytest.raises(ValueError, match="has an unsupported MIME type"):
+            _ = converter.run(documents=[docx_doc])
+
+    def test_run_with_invalid_file_path(self, caplog) -> None:
+        converter = DocumentToImageContent()
+        pdf_doc = Document(content="test", meta={"file_path": "wrong_name.jpg"})
+        with pytest.raises(ValueError, match="has an invalid file path 'wrong_name.jpg'"):
+            _ = converter.run(documents=[pdf_doc])
+
+    def test_run_with_pdf_missing_page_number(self, caplog) -> None:
+        converter = DocumentToImageContent()
+        pdf_doc = Document(content="test", meta={"file_path": "test/test_files/pdf/sample_pdf_1.pdf"})
+        with pytest.raises(ValueError, match="is missing the 'page_number' key"):
+            _ = converter.run(documents=[pdf_doc])
+
+    def test_run_with_image_documents(self) -> None:
+        converter = DocumentToImageContent(root_path="test/test_files/images")
+        image_doc = Document(content="test", meta={"file_path": "apple.jpg"})
+        results = converter.run(documents=[image_doc])
+        assert len(results["image_contents"]) == 1
+        assert results["image_contents"][0].meta == {"file_path": "apple.jpg"}
+
+    def test_run_with_pdf_documents(self) -> None:
+        converter = DocumentToImageContent()
+        pdf_doc = Document(content="test", meta={"file_path": "test/test_files/pdf/sample_pdf_1.pdf", "page_number": 1})
+        results = converter.run(documents=[pdf_doc])
+        assert len(results["image_contents"]) == 1
+        assert results["image_contents"][0].meta == {
+            "file_path": "test/test_files/pdf/sample_pdf_1.pdf",
+            "page_number": 1,
+        }
+
+    def test_run_with_mixed_document_types(self) -> None:
+        converter = DocumentToImageContent(root_path="test/test_files")
+        documents = [
+            Document(content="", meta={"file_path": "images/apple.jpg"}),
+            Document(content="", meta={"file_path": "pdf/sample_pdf_1.pdf", "page_number": 1}),
+            Document(content="text", meta={"file_path": "docx/sample_docx.docx"}),
+        ]
+        with pytest.raises(ValueError, match="has an unsupported MIME type"):
+            _ = converter.run(documents=documents)
+
+    @patch("haystack.components.converters.image.document_to_image._extract_image_sources_info")
+    @patch("haystack.components.converters.image.document_to_image._batch_convert_pdf_pages_to_images")
+    @patch("PIL.Image.open")
+    @patch("haystack.components.converters.image.document_to_image.ByteStream")
+    def test_run_none_images(
+        self,
+        mocked_byte_stream,
+        mocked_pil_open,
+        mocked_batch_convert_pdf_pages_to_images,
+        mocked_extract_image_sources_info,
+        caplog,
+    ):
+        converter = DocumentToImageContent()
+
+        mocked_extract_image_sources_info.return_value = [
+            {"path": "doc1.pdf", "mime_type": "application/pdf", "page_number": 999},  # Page 999 doesn't exist
+            {"path": "image1.jpg", "mime_type": "image/jpeg"},
+        ]
+        mocked_batch_convert_pdf_pages_to_images.return_value = {}  # Empty dict because page was skipped
+        mocked_pil_open.return_value = Image.new("RGB", (100, 100))
+        mocked_byte_stream.from_file_path.return_value = ByteStream(b"")
+
+        documents = [
+            Document(content="PDF 1", meta={"file_path": "doc1.pdf", "page_number": 999}),
+            Document(content="Image 1", meta={"file_path": "image1.jpg"}),
+        ]
+
+        image_contents = converter.run(documents=documents)["image_contents"]
+
+        assert caplog.records[-1].levelname == "WARNING"
+        assert "Conversion failed for some documents." in caplog.records[-1].message
+
+        assert image_contents[0] is None
+        assert image_contents[1] is not None

test/components/converters/image/test_file_to_document.py

@@ -0,0 +1,82 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from pathlib import Path
+
+import pytest
+
+from haystack.components.converters.image.file_to_document import ImageFileToDocument
+from haystack.core.serialization import component_from_dict, component_to_dict
+from haystack.dataclasses import ByteStream
+
+
+class TestImageFileToDocument:
+    def test_to_dict(self) -> None:
+        converter = ImageFileToDocument()
+        assert component_to_dict(converter, "converter") == {
+            "init_parameters": {"store_full_path": False},
+            "type": "haystack.components.converters.image.file_to_document.ImageFileToDocument",
+        }
+
+    def test_to_dict_not_defaults(self) -> None:
+        converter = ImageFileToDocument(store_full_path=True)
+        assert component_to_dict(converter, "converter") == {
+            "init_parameters": {"store_full_path": True},
+            "type": "haystack.components.converters.image.file_to_document.ImageFileToDocument",
+        }
+
+    def test_from_dict(self) -> None:
+        data = {
+            "init_parameters": {"store_full_path": False},
+            "type": "haystack.components.converters.image.file_to_document.ImageFileToDocument",
+        }
+        converter = component_from_dict(ImageFileToDocument, data, "name")
+        assert component_to_dict(converter, "converter") == data
+
+    @pytest.mark.parametrize(
+        ("image_path", "mime_type"),
+        [
+            ("./test/test_files/images/haystack-logo.png", "image/png"),
+            ("./test/test_files/images/apple.jpg", "image/jpeg"),
+        ],
+    )
+    def test_run_with_valid_sources(self, image_path: str, mime_type: str) -> None:
+        converter = ImageFileToDocument(store_full_path=True)
+        results = converter.run(sources=[image_path], meta={"source": "test_source"})
+
+        assert len(results["documents"]) == 1
+        assert results["documents"][0].content is None
+        assert results["documents"][0].meta == {"source": "test_source", "file_path": image_path}
+
+    def test_run_with_no_sources(self) -> None:
+        converter = ImageFileToDocument()
+        results = converter.run(sources=[])
+        assert len(results["documents"]) == 0
+        assert results == {"documents": []}
+
+    def test_run_with_invalid_source_type(self, caplog) -> None:
+        converter = ImageFileToDocument()
+        converter.run(sources=[123])  # Invalid source type
+        assert "Could not read" in caplog.text
+
+    def test_run_with_non_existent_file(self, caplog) -> None:
+        converter = ImageFileToDocument()
+        converter.run(sources=["./non_existent_file.png"])
+        assert "Could not read" in caplog.text
+        assert "No such file or directory:" in caplog.text
+
+    @pytest.mark.parametrize(
+        ("image_path", "mime_type"),
+        [
+            ("./test/test_files/images/haystack-logo.png", "image/png"),
+            ("./test/test_files/images/apple.jpg", "image/jpeg"),
+        ],
+    )
+    def test_run_with_bytestream_sources(self, image_path: str, mime_type: str) -> None:
+        byte_stream = ByteStream.from_file_path(Path(image_path), mime_type=mime_type, meta={"file_path": image_path})
+        converter = ImageFileToDocument(store_full_path=True)
+        results = converter.run(sources=[byte_stream])
+        assert len(results["documents"]) == 1
+        assert results["documents"][0].content is None
+        assert results["documents"][0].meta == {"file_path": image_path}

test/components/converters/image/test_file_to_image.py

@@ -0,0 +1,116 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from pathlib import Path
+
+import pytest
+
+from haystack.components.converters.image.file_to_image import ImageFileToImageContent
+from haystack.components.converters.image.image_utils import _encode_image_to_base64
+from haystack.core.serialization import component_from_dict, component_to_dict
+from haystack.dataclasses import ByteStream
+
+
+class TestImageFileToImageContent:
+    def test_to_dict(self) -> None:
+        converter = ImageFileToImageContent()
+        assert component_to_dict(converter, "converter") == {
+            "init_parameters": {"detail": None, "size": None},
+            "type": "haystack.components.converters.image.file_to_image.ImageFileToImageContent",
+        }
+
+    def test_to_dict_not_defaults(self) -> None:
+        converter = ImageFileToImageContent(detail="low", size=(128, 128))
+        assert component_to_dict(converter, "converter") == {
+            "init_parameters": {"detail": "low", "size": (128, 128)},
+            "type": "haystack.components.converters.image.file_to_image.ImageFileToImageContent",
+        }
+
+    def test_from_dict(self) -> None:
+        data = {
+            "init_parameters": {"detail": "auto", "size": None},
+            "type": "haystack.components.converters.image.file_to_image.ImageFileToImageContent",
+        }
+        converter = component_from_dict(ImageFileToImageContent, data, "name")
+        assert component_to_dict(converter, "converter") == data
+
+    @pytest.mark.parametrize(
+        ("image_path", "mime_type"),
+        [
+            ("./test/test_files/images/haystack-logo.png", "image/png"),
+            ("./test/test_files/images/apple.jpg", "image/jpeg"),
+        ],
+    )
+    def test_run_with_valid_sources(self, image_path: str, mime_type: str) -> None:
+        converter = ImageFileToImageContent()
+        results = converter.run(sources=[image_path], size=(128, 128))
+
+        byte_stream = ByteStream.from_file_path(
+            Path(image_path), mime_type=mime_type, meta={"file_name": image_path.rsplit("/", maxsplit=1)[-1]}
+        )
+        assert len(results["image_contents"]) == 1
+        assert results["image_contents"][0].base64_image is not None
+        assert (
+            results["image_contents"][0].base64_image
+            == _encode_image_to_base64(bytestream=byte_stream, size=(128, 128))[1]
+        )
+        assert results["image_contents"][0].mime_type == mime_type
+        assert results["image_contents"][0].detail is None
+        assert results["image_contents"][0].meta["file_path"] == str(Path(image_path))
+
+    def test_run_with_no_sources(self) -> None:
+        converter = ImageFileToImageContent()
+        results = converter.run(sources=[])
+        assert len(results["image_contents"]) == 0
+        assert results == {"image_contents": []}
+
+    def test_run_with_invalid_source_type(self, caplog) -> None:
+        converter = ImageFileToImageContent()
+        converter.run(sources=[123])  # Invalid source type
+        assert "Could not read" in caplog.text
+
+    def test_run_with_non_existent_file(self, caplog) -> None:
+        converter = ImageFileToImageContent()
+        converter.run(sources=["./non_existent_file.png"])
+        assert "Could not read" in caplog.text
+        assert "No such file or directory:" in caplog.text
+
+    @pytest.mark.parametrize(
+        ("image_path", "mime_type"),
+        [
+            ("./test/test_files/images/haystack-logo.png", "image/png"),
+            ("./test/test_files/images/apple.jpg", "image/jpeg"),
+        ],
+    )
+    def test_run_with_bytestream_sources(self, image_path: str, mime_type: str) -> None:
+        byte_stream = ByteStream.from_file_path(Path(image_path), mime_type=mime_type, meta={"file_path": image_path})
+
+        # Initialize the converter
+        converter = ImageFileToImageContent(size=(128, 128))
+
+        # Run the converter with the ByteStream
+        results = converter.run(sources=[byte_stream])
+
+        # Assertions
+        assert len(results["image_contents"]) == 1
+        assert results["image_contents"][0].base64_image is not None
+        assert (
+            results["image_contents"][0].base64_image
+            == _encode_image_to_base64(bytestream=byte_stream, size=(128, 128))[1]
+        )
+        assert results["image_contents"][0].mime_type == mime_type
+        assert results["image_contents"][0].detail is None
+        assert results["image_contents"][0].meta["file_path"] == image_path
+
+    def test_run_with_empty_bytestream(self) -> None:
+        # Create an empty ByteStream object
+        byte_stream = ByteStream(data=b"", meta={"file_path": "empty_file.png"})
+
+        # Initialize the converter
+        converter = ImageFileToImageContent()
+
+        # Run the converter with the empty ByteStream
+        results = converter.run(sources=[byte_stream])
+
+        assert results["image_contents"] == []

test/components/converters/image/test_image_utils.py

@@ -0,0 +1,227 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import glob
+import logging
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+import pytest
+from PIL import Image
+from pytest import LogCaptureFixture
+
+from haystack.components.converters.image.image_utils import (
+    _batch_convert_pdf_pages_to_images,
+    _convert_pdf_to_images,
+    _encode_image_to_base64,
+    _encode_pil_image_to_base64,
+    _extract_image_sources_info,
+    _PDFPageInfo,
+)
+from haystack.components.converters.utils import get_bytestream_from_source
+from haystack.dataclasses import ByteStream, Document
+
+
+class TestToBase64Jpeg:
+    def test_to_base64_jpeg(self) -> None:
+        image_array = np.array(
+            [
+                [[34.215402, 132.78745697, 71.04739979], [24.23156181, 35.26147199, 124.95610316]],
+                [[155.47443501, 196.98050276, 154.74734292], [253.24590033, 84.62392497, 157.34396641]],
+            ]
+        )
+        image = Image.fromarray(image_array.astype("uint8"))
+        b64_str = _encode_pil_image_to_base64(image=image, mime_type="image/jpeg")
+        assert (
+            b64_str
+            == "/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwcKDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCAACAAIDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwCe50jTTdTE6daffb/livr9KKKK9aHwo9ql8EfRH//Z"  # noqa: E501
+        )
+
+
+class TestConvertPdfToImages:
+    def test_convert_pdf_to_images(self) -> None:
+        bytestream = get_bytestream_from_source(Path("test/test_files/pdf/sample_pdf_1.pdf"))
+        output = _convert_pdf_to_images(bytestream=bytestream, page_range=[1])
+        assert output is not None
+
+    def test_convert_pdf_to_images_with_size(self) -> None:
+        bytestream = get_bytestream_from_source(Path("test/test_files/pdf/sample_pdf_1.pdf"))
+        pages_images = _convert_pdf_to_images(bytestream=bytestream, page_range=[1], size=(100, 100))
+
+        assert len(pages_images) == 1
+        assert pages_images[0][0] == 1
+        assert pages_images[0][1].width <= 100
+        assert pages_images[0][1].height <= 100
+
+    def test_convert_pdf_to_images_invalid_page(self, caplog: LogCaptureFixture) -> None:
+        bytestream = get_bytestream_from_source(Path("test/test_files/pdf/sample_pdf_1.pdf"))
+        out = _convert_pdf_to_images(bytestream=bytestream, page_range=[5])
+        assert out == []
+        assert "Page 5 is out of range for the PDF file. Skipping it." in caplog.text
+
+    def test_convert_pdf_to_images_error_reading_file(self, caplog: LogCaptureFixture) -> None:
+        bytestream = ByteStream(data=b"", mime_type="application/pdf")
+        out = _convert_pdf_to_images(bytestream=bytestream, page_range=[1])
+        assert out == []
+        assert "Could not read PDF file" in caplog.text
+
+    def test_convert_pdf_to_images_empty_file(self, caplog: LogCaptureFixture) -> None:
+        bytestream = get_bytestream_from_source(Path("test/test_files/pdf/sample_pdf_1.pdf"))
+
+        with patch("haystack.components.converters.image.image_utils.PdfDocument") as mock_pdf_document:
+            mock_pdf_document.__len__.return_value = 0
+            out = _convert_pdf_to_images(bytestream=bytestream, page_range=[1])
+
+        assert out == []
+        assert "PDF file is empty" in caplog.text
+
+    def test_convert_pdf_to_images_scale_if_large_pdf(self, caplog: LogCaptureFixture) -> None:
+        bytestream = get_bytestream_from_source(Path("test/test_files/pdf/sample_pdf_1.pdf"))
+
+        caplog.set_level(logging.INFO)
+
+        mock_pdf_document = MagicMock()
+        mock_pdf_document.__len__.return_value = 1
+        mock_page = MagicMock()
+        mock_page.get_mediabox.return_value = (0, 0, 1e6, 1e6)
+        mock_pdf_document.__getitem__.return_value = mock_page
+
+        with patch("haystack.components.converters.image.image_utils.PdfDocument", return_value=mock_pdf_document):
+            _convert_pdf_to_images(bytestream=bytestream, page_range=[1])
+
+        assert "Large PDF detected" in caplog.text
+
+
+class TestEncodeImageToBase64:
+    def test_encode_image_to_base64(self) -> None:
+        bytestream = get_bytestream_from_source(Path("test/test_files/images/haystack-logo.png"))
+        base64_str = _encode_image_to_base64(bytestream=bytestream)
+        assert base64_str is not None
+
+    def test_encode_image_to_base64_downsize(self) -> None:
+        bytestream = get_bytestream_from_source(Path("test/test_files/images/haystack-logo.png"))
+        base64_str = _encode_image_to_base64(bytestream=bytestream, size=(128, 128))
+        assert base64_str is not None
+
+
+class TestExtractImageSourcesInfo:
+    def test_extract_image_source_info(self, test_files_path):
+        image_paths = glob.glob(str(test_files_path / "images" / "*.*")) + glob.glob(
+            str(test_files_path / "pdf" / "*.pdf")
+        )
+
+        documents = []
+        for i, path in enumerate(image_paths):
+            document = Document(content=f"document number {i}", meta={"file_path": path})
+            if path.endswith(".pdf"):
+                document.meta["page_number"] = 1
+            documents.append(document)
+
+        images_source_info = _extract_image_sources_info(
+            documents=documents, file_path_meta_field="file_path", root_path=""
+        )
+        assert len(images_source_info) == len(documents)
+
+        for image_source_info in images_source_info:
+            assert str(image_source_info["path"]) in image_paths
+            assert image_source_info["mime_type"] in ["image/jpeg", "image/png", "application/pdf"]
+            if image_source_info["mime_type"] == "application/pdf":
+                assert image_source_info.get("page_number") == 1
+            else:
+                assert "page_number" not in image_source_info
+
+    def test_extract_image_source_info_errors(self, test_files_path):
+        document = Document(content="test")
+        with pytest.raises(ValueError, match="missing the 'file_path' key"):
+            _extract_image_sources_info(documents=[document], file_path_meta_field="file_path", root_path="")
+
+        document = Document(content="test", meta={"file_path": "invalid_path"})
+        with pytest.raises(ValueError, match="has an invalid file path"):
+            _extract_image_sources_info(documents=[document], file_path_meta_field="file_path", root_path="")
+
+        document = Document(content="test", meta={"file_path": str(test_files_path / "docx" / "sample_docx.docx")})
+        with pytest.raises(ValueError, match="has an unsupported MIME type"):
+            _extract_image_sources_info(documents=[document], file_path_meta_field="file_path", root_path="")
+
+        document = Document(content="test", meta={"file_path": str(test_files_path / "pdf" / "sample_pdf_1.pdf")})
+        with pytest.raises(ValueError, match="missing the 'page_number' key"):
+            _extract_image_sources_info(documents=[document], file_path_meta_field="file_path", root_path="")
+
+    def test_extract_image_source_info_rejects_path_traversal(self, test_files_path):
+        # Attacker-controlled document metadata attempts to escape the configured root.
+        document = Document(content="test", meta={"file_path": "../../../../../../etc/passwd"})
+        with pytest.raises(ValueError, match="escapes the configured root"):
+            _extract_image_sources_info(
+                documents=[document], file_path_meta_field="file_path", root_path=str(test_files_path / "images")
+            )
+
+    def test_extract_image_source_info_rejects_absolute_outside_root(self, test_files_path):
+        # Absolute path that lies outside the configured root must be rejected before any IO.
+        document = Document(content="test", meta={"file_path": "/etc/passwd"})
+        with pytest.raises(ValueError, match="escapes the configured root"):
+            _extract_image_sources_info(
+                documents=[document], file_path_meta_field="file_path", root_path=str(test_files_path / "images")
+            )
+
+    def test_extract_image_source_info_accepts_path_inside_root(self, test_files_path):
+        # When the resolved path is inside the configured root, processing must succeed.
+        document = Document(content="test", meta={"file_path": "haystack-logo.png"})
+        images_source_info = _extract_image_sources_info(
+            documents=[document], file_path_meta_field="file_path", root_path=str(test_files_path / "images")
+        )
+        assert len(images_source_info) == 1
+        assert images_source_info[0]["mime_type"] == "image/png"
+
+
+class TestBatchConvertPdfPagesToImages:
+    @patch("haystack.components.converters.image.image_utils._convert_pdf_to_images")
+    def test_batch_convert_pdf_pages_to_images(self, mocked_convert_pdf_to_images, test_files_path):
+        mocked_convert_pdf_to_images.return_value = [
+            (1, Image.new("RGB", (100, 100))),
+            (2, Image.new("RGB", (100, 100))),
+        ]
+
+        pdf_path = test_files_path / "pdf" / "sample_pdf_1.pdf"
+        pdf_doc_1: _PDFPageInfo = {"doc_idx": 0, "path": pdf_path, "page_number": 1}
+        pdf_doc_2: _PDFPageInfo = {"doc_idx": 1, "path": pdf_path, "page_number": 2}
+        pdf_documents = [pdf_doc_1, pdf_doc_2]
+
+        result = _batch_convert_pdf_pages_to_images(pdf_page_infos=pdf_documents, return_base64=False)
+
+        pdf_bytestream = ByteStream.from_file_path(pdf_path)
+
+        mocked_convert_pdf_to_images.assert_called_once_with(
+            bytestream=pdf_bytestream, page_range=[1, 2], size=None, return_base64=False
+        )
+
+        assert len(result) == len(pdf_documents)
+        assert 0 in result and 1 in result
+        assert isinstance(result[0], Image.Image)
+        assert isinstance(result[1], Image.Image)
+
+    @patch("haystack.components.converters.image.image_utils._convert_pdf_to_images")
+    def test_batch_convert_pdf_pages_to_images_base64(self, mocked_convert_pdf_to_images, test_files_path):
+        mocked_convert_pdf_to_images.return_value = [(1, "base64_image_1"), (2, "base64_image_2")]
+
+        pdf_path = test_files_path / "pdf" / "sample_pdf_1.pdf"
+        pdf_doc_1: _PDFPageInfo = {"doc_idx": 0, "path": pdf_path, "page_number": 1}
+        pdf_doc_2: _PDFPageInfo = {"doc_idx": 1, "path": pdf_path, "page_number": 2}
+        pdf_documents = [pdf_doc_1, pdf_doc_2]
+
+        result = _batch_convert_pdf_pages_to_images(pdf_page_infos=pdf_documents, return_base64=True)
+
+        pdf_bytestream = ByteStream.from_file_path(pdf_path)
+
+        mocked_convert_pdf_to_images.assert_called_once_with(
+            bytestream=pdf_bytestream, page_range=[1, 2], size=None, return_base64=True
+        )
+
+        assert result == {0: "base64_image_1", 1: "base64_image_2"}
+
+    def test_batch_convert_pdf_pages_to_images_no_pages_info(self):
+        result = _batch_convert_pdf_pages_to_images(pdf_page_infos=[])
+
+        assert isinstance(result, dict)
+        assert len(result) == 0

test/components/converters/image/test_pdf_to_image.py

@@ -0,0 +1,107 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from pathlib import Path
+
+from haystack.components.converters.image.image_utils import _convert_pdf_to_images
+from haystack.components.converters.image.pdf_to_image import PDFToImageContent
+from haystack.core.serialization import component_from_dict, component_to_dict
+from haystack.dataclasses import ByteStream
+
+
+class TestPDFToImageContent:
+    def test_to_dict(self) -> None:
+        converter = PDFToImageContent()
+        assert component_to_dict(converter, "converter") == {
+            "init_parameters": {"detail": None, "size": None, "page_range": None},
+            "type": "haystack.components.converters.image.pdf_to_image.PDFToImageContent",
+        }
+
+    def test_to_dict_not_defaults(self) -> None:
+        converter = PDFToImageContent(detail="low", size=(128, 128), page_range=[1])
+        assert component_to_dict(converter, "converter") == {
+            "init_parameters": {"detail": "low", "size": (128, 128), "page_range": [1]},
+            "type": "haystack.components.converters.image.pdf_to_image.PDFToImageContent",
+        }
+
+    def test_from_dict(self) -> None:
+        data = {
+            "init_parameters": {"detail": "auto", "size": None, "page_range": [1]},
+            "type": "haystack.components.converters.image.pdf_to_image.PDFToImageContent",
+        }
+        converter = component_from_dict(PDFToImageContent, data, "name")
+        assert component_to_dict(converter, "converter") == data
+
+    def test_run_with_valid_source(self) -> None:
+        file_path = "./test/test_files/pdf/sample_pdf_1.pdf"
+        mime_type = "application/pdf"
+        converter = PDFToImageContent()
+        results = converter.run(sources=[file_path])
+
+        byte_stream = ByteStream.from_file_path(Path(file_path), mime_type=mime_type, meta={"file_path": file_path})
+        assert len(results["image_contents"]) == 4
+        assert results["image_contents"][0].base64_image is not None
+        assert (
+            results["image_contents"][0].base64_image
+            == _convert_pdf_to_images(bytestream=byte_stream, size=None, page_range=[1], return_base64=True)[0][1]
+        )
+        assert results["image_contents"][0].mime_type == "image/jpeg"
+        assert results["image_contents"][0].detail is None
+        assert results["image_contents"][0].meta["file_path"] == str(Path(file_path))
+        assert results["image_contents"][0].meta["page_number"] == 1
+        assert results["image_contents"][1].meta["page_number"] == 2
+        assert results["image_contents"][2].meta["page_number"] == 3
+        assert results["image_contents"][3].meta["page_number"] == 4
+
+    def test_run_with_no_sources(self) -> None:
+        converter = PDFToImageContent()
+        results = converter.run(sources=[])
+        assert len(results["image_contents"]) == 0
+        assert results == {"image_contents": []}
+
+    def test_run_with_invalid_source_type(self, caplog) -> None:
+        converter = PDFToImageContent()
+        converter.run(sources=[123])  # Invalid source type
+        assert "Could not read" in caplog.text
+
+    def test_run_with_non_existent_file(self, caplog) -> None:
+        converter = PDFToImageContent()
+        converter.run(sources=["./non_existent_file.png"])
+        assert "Could not read" in caplog.text
+        assert "No such file or directory:" in caplog.text
+
+    def test_run_with_bytestream_sources(self) -> None:
+        file_path = "./test/test_files/pdf/sample_pdf_1.pdf"
+        mime_type = "application/pdf"
+        byte_stream = ByteStream.from_file_path(Path(file_path), mime_type=mime_type, meta={"file_path": file_path})
+
+        # Initialize the converter
+        converter = PDFToImageContent()
+
+        # Run the converter with the ByteStream
+        results = converter.run(sources=[byte_stream])
+
+        # Assertions
+        assert len(results["image_contents"]) == 4
+        assert results["image_contents"][0].base64_image is not None
+        assert (
+            results["image_contents"][0].base64_image
+            == _convert_pdf_to_images(bytestream=byte_stream, size=None, page_range=[1], return_base64=True)[0][1]
+        )
+        assert results["image_contents"][0].mime_type == "image/jpeg"
+        assert results["image_contents"][0].detail is None
+        assert results["image_contents"][0].meta["file_path"] == file_path
+        assert results["image_contents"][0].meta["page_number"] == 1
+
+    def test_run_with_empty_bytestream(self) -> None:
+        # Create an empty ByteStream object
+        byte_stream = ByteStream(data=b"", meta={"file_path": "empty_file.pdf"})
+
+        # Initialize the converter
+        converter = PDFToImageContent()
+
+        # Run the converter with the empty ByteStream
+        results = converter.run(sources=[byte_stream])
+
+        assert results["image_contents"] == []

test/components/converters/test_csv_to_document.py

@@ -0,0 +1,244 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import os
+
+import pytest
+
+from haystack.components.converters.csv import CSVToDocument
+from haystack.dataclasses import ByteStream
+
+
+@pytest.fixture
+def csv_converter():
+    return CSVToDocument()
+
+
+class TestCSVToDocument:
+    def test_init(self, csv_converter):
+        assert isinstance(csv_converter, CSVToDocument)
+
+    def test_run(self, test_files_path):
+        """
+        Test if the component runs correctly.
+        """
+        bytestream = ByteStream.from_file_path(test_files_path / "csv" / "sample_1.csv")
+        bytestream.meta["file_path"] = str(test_files_path / "csv" / "sample_1.csv")
+        bytestream.meta["key"] = "value"
+        files = [bytestream, test_files_path / "csv" / "sample_2.csv", test_files_path / "csv" / "sample_3.csv"]
+        converter = CSVToDocument()
+        output = converter.run(sources=files)
+        docs = output["documents"]
+        assert len(docs) == 3
+        assert docs[0].content == "Name,Age\r\nJohn Doe,27\r\nJane Smith,37\r\nMike Johnson,47\r\n"
+        assert isinstance(docs[0].content, str)
+        assert docs[0].meta == {"file_path": os.path.basename(bytestream.meta["file_path"]), "key": "value"}
+        assert docs[1].meta["file_path"] == os.path.basename(files[1])
+        assert docs[2].meta["file_path"] == os.path.basename(files[2])
+
+    def test_run_with_store_full_path_false(self, test_files_path):
+        """
+        Test if the component runs correctly with store_full_path=False
+        """
+        bytestream = ByteStream.from_file_path(test_files_path / "csv" / "sample_1.csv")
+        bytestream.meta["file_path"] = str(test_files_path / "csv" / "sample_1.csv")
+        bytestream.meta["key"] = "value"
+        files = [bytestream, test_files_path / "csv" / "sample_2.csv", test_files_path / "csv" / "sample_3.csv"]
+        converter = CSVToDocument(store_full_path=False)
+        output = converter.run(sources=files)
+        docs = output["documents"]
+        assert len(docs) == 3
+        assert docs[0].content == "Name,Age\r\nJohn Doe,27\r\nJane Smith,37\r\nMike Johnson,47\r\n"
+        assert isinstance(docs[0].content, str)
+        assert docs[0].meta["file_path"] == "sample_1.csv"
+        assert docs[0].meta["key"] == "value"
+        assert docs[1].meta["file_path"] == "sample_2.csv"
+        assert docs[2].meta["file_path"] == "sample_3.csv"
+
+    def test_run_error_handling(self, test_files_path, caplog):
+        """
+        Test if the component correctly handles errors.
+        """
+        paths = [
+            test_files_path / "csv" / "sample_2.csv",
+            "non_existing_file.csv",
+            test_files_path / "csv" / "sample_3.csv",
+        ]
+        converter = CSVToDocument()
+        with caplog.at_level(logging.WARNING):
+            output = converter.run(sources=paths)
+            assert "non_existing_file.csv" in caplog.text
+        docs = output["documents"]
+        assert len(docs) == 2
+        assert docs[0].meta["file_path"] == os.path.basename(paths[0])
+
+    def test_encoding_override(self, test_files_path, caplog):
+        """
+        Test if the encoding metadata field is used properly
+        """
+        bytestream = ByteStream.from_file_path(test_files_path / "csv" / "sample_1.csv")
+        bytestream.meta["key"] = "value"
+
+        converter = CSVToDocument(encoding="utf-16-le")
+        _ = converter.run(sources=[bytestream])
+        with caplog.at_level(logging.ERROR):
+            _ = converter.run(sources=[bytestream])
+            assert "codec can't decode" in caplog.text
+
+        converter = CSVToDocument(encoding="utf-8")
+        output = converter.run(sources=[bytestream])
+        assert "Name,Age\r\n" in output["documents"][0].content
+
+    def test_run_with_meta(self):
+        bytestream = ByteStream(
+            data=b"Name,Age,City\r\nAlice,30,New York\r\nBob,25,Los Angeles\r\nCharlie,35,Chicago\r\n",
+            meta={"name": "test_name", "language": "en"},
+        )
+        converter = CSVToDocument()
+        output = converter.run(sources=[bytestream], meta=[{"language": "it"}])
+        document = output["documents"][0]
+        assert document.meta == {"name": "test_name", "language": "it"}
+
+    # --- NEW TESTS for strict row mode ---
+
+    def test_row_mode_requires_content_column_param(self, tmp_path):
+        # Missing content_column must raise in row mode
+        f = tmp_path / "t.csv"
+        f.write_text("a,b\r\n1,2\r\n", encoding="utf-8")
+        conv = CSVToDocument(conversion_mode="row")
+        with pytest.raises(ValueError):
+            _ = conv.run(sources=[f])  # content_column missing
+
+    def test_row_mode_missing_header_raises(self, tmp_path):
+        # content_column must exist in header
+        f = tmp_path / "t.csv"
+        f.write_text("a,b\r\n1,2\r\n", encoding="utf-8")
+        conv = CSVToDocument(conversion_mode="row")
+        with pytest.raises(ValueError):
+            _ = conv.run(sources=[f], content_column="missing")
+
+    def test_row_mode_with_content_column(self, tmp_path):
+        csv_text = "text,author,stars\r\nNice app,Ada,5\r\nBuggy,Bob,2\r\n"
+        f = tmp_path / "fb.csv"
+        f.write_text(csv_text, encoding="utf-8")
+
+        bytestream = ByteStream.from_file_path(f)
+        bytestream.meta["file_path"] = str(f)
+
+        converter = CSVToDocument(conversion_mode="row")
+        output = converter.run(sources=[bytestream], content_column="text")
+        docs = output["documents"]
+
+        assert len(docs) == 2
+        assert [d.content for d in docs] == ["Nice app", "Buggy"]
+        assert docs[0].meta["author"] == "Ada"
+        assert docs[0].meta["stars"] == "5"
+        assert docs[0].meta["row_number"] == 0
+        assert os.path.basename(f) == docs[0].meta["file_path"]
+
+    def test_row_mode_meta_collision_prefixed(self, tmp_path):
+        # ByteStream meta has file_path and encoding; CSV also has those columns.
+        csv_text = "file_path,encoding,comment\r\nrowpath.csv,latin1,ok\r\n"
+        f = tmp_path / "collide.csv"
+        f.write_text(csv_text, encoding="utf-8")
+        bs = ByteStream.from_file_path(f)
+        bs.meta["file_path"] = str(f)
+        bs.meta["encoding"] = "utf-8"
+
+        conv = CSVToDocument(conversion_mode="row")
+        out = conv.run(sources=[bs], content_column="comment")
+        d = out["documents"][0]
+        # Original meta preserved
+        assert d.meta["file_path"] == os.path.basename(str(f))
+        assert d.meta["encoding"] == "utf-8"
+        # CSV columns stored with csv_ prefix (no clobber)
+        assert d.meta["csv_file_path"] == "rowpath.csv"
+        assert d.meta["csv_encoding"] == "latin1"
+        # content column isn't duplicated in meta
+        assert "comment" not in d.meta
+        assert d.meta["row_number"] == 0
+        assert d.content == "ok"
+
+    def test_row_mode_meta_collision_multiple_suffixes(self, tmp_path):
+        """
+        If meta already has csv_file_path and csv_file_path_1, we should write the next as csv_file_path_2.
+        """
+        csv_text = "file_path,comment\r\nrow.csv,ok\r\n"
+        f = tmp_path / "multi.csv"
+        f.write_text(csv_text, encoding="utf-8")
+
+        bs = ByteStream.from_file_path(f)
+        bs.meta["file_path"] = str(f)
+
+        # Pre-seed meta so we force two collisions.
+        extra_meta = {"csv_file_path": "existing0", "csv_file_path_1": "existing1"}
+
+        conv = CSVToDocument(conversion_mode="row")
+        out = conv.run(sources=[bs], meta=[extra_meta], content_column="comment")
+        d = out["documents"][0]
+
+        assert d.meta["csv_file_path"] == "existing0"
+        assert d.meta["csv_file_path_1"] == "existing1"
+        assert d.meta["csv_file_path_2"] == "row.csv"
+        assert d.content == "ok"
+
+    def test_init_validates_delimiter_and_quotechar(self):
+        with pytest.raises(ValueError):
+            CSVToDocument(delimiter=";;")
+        with pytest.raises(ValueError):
+            CSVToDocument(quotechar='""')
+
+    def test_row_mode_large_file_warns(self, caplog, monkeypatch):
+        # Make the threshold tiny so the warning always triggers.
+        import haystack.components.converters.csv as csv_mod
+
+        monkeypatch.setattr(csv_mod, "_ROW_MODE_SIZE_WARN_BYTES", 1, raising=False)
+
+        bs = ByteStream(data=b"text,author\nhi,Ada\n", meta={"file_path": "big.csv"})
+        conv = CSVToDocument(conversion_mode="row")
+        with caplog.at_level(logging.WARNING, logger="haystack.components.converters.csv"):
+            _ = conv.run(sources=[bs], content_column="text")
+        assert "parsing a large CSV" in caplog.text
+
+    def test_row_mode_reader_failure_raises_runtimeerror(self, monkeypatch, tmp_path):
+        # Simulate DictReader failing -> we should raise RuntimeError (no fallback).
+        import haystack.components.converters.csv as csv_mod
+
+        f = tmp_path / "bad.csv"
+        f.write_text("a,b\n1,2\n", encoding="utf-8")
+        conv = CSVToDocument(conversion_mode="row")
+
+        class Boom(Exception):
+            pass
+
+        def broken_reader(*_args, **_kwargs):  # noqa: D401
+            raise Boom("broken")
+
+        monkeypatch.setattr(csv_mod.csv, "DictReader", broken_reader, raising=True)
+        with pytest.raises(RuntimeError):
+            _ = conv.run(sources=[f], content_column="a")
+
+    def test_row_mode_ragged_row_does_not_crash(self):
+        # A data row with more fields than the header (e.g. an unquoted comma inside a value).
+        # Previously the surplus value landed under the None key, which broke Document id
+        # generation (TypeError sorting None against str keys) and aborted the whole batch.
+        valid = ByteStream(data=b"text,author\r\nfine,Ada\r\n", meta={"file_path": "valid.csv"})
+        ragged = ByteStream(data=b"text,note\r\nhello,city,state\r\n", meta={"file_path": "ragged.csv"})
+
+        conv = CSVToDocument(conversion_mode="row")
+        out = conv.run(sources=[valid, ragged], content_column="text")
+        docs = out["documents"]
+
+        # Both sources yielded a Document; the earlier valid source is not lost.
+        assert len(docs) == 2
+        assert docs[0].content == "fine"
+        assert docs[0].meta["author"] == "Ada"
+
+        ragged_doc = docs[1]
+        assert ragged_doc.content == "hello"
+        assert ragged_doc.meta["note"] == "city"
+        # Surplus value is preserved under an explicit (non-None) string meta key.
+        assert None not in ragged_doc.meta
+        assert "state" in ragged_doc.meta["extra_columns"]

test/components/converters/test_docx_file_to_document.py

@@ -0,0 +1,455 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import csv
+import json
+import logging
+import os
+from io import StringIO
+
+import pytest
+
+from haystack import Document, Pipeline
+from haystack.components.converters.docx import DOCXLinkFormat, DOCXMetadata, DOCXTableFormat, DOCXToDocument
+from haystack.dataclasses import ByteStream
+
+
+@pytest.fixture
+def docx_converter():
+    return DOCXToDocument()
+
+
+class TestDOCXToDocument:
+    def test_init(self, docx_converter):
+        assert isinstance(docx_converter, DOCXToDocument)
+
+    def test_init_with_string(self):
+        converter = DOCXToDocument(table_format="markdown")
+        assert isinstance(converter, DOCXToDocument)
+        assert converter.table_format == DOCXTableFormat.MARKDOWN
+
+    def test_init_with_invalid_string(self):
+        with pytest.raises(ValueError, match="Unknown table format 'invalid_format'"):
+            DOCXToDocument(table_format="invalid_format")
+
+    def test_to_dict(self):
+        converter = DOCXToDocument()
+        data = converter.to_dict()
+        assert data == {
+            "type": "haystack.components.converters.docx.DOCXToDocument",
+            "init_parameters": {"store_full_path": False, "table_format": "csv", "link_format": "none"},
+        }
+
+    def test_to_dict_custom_parameters(self):
+        converter = DOCXToDocument(table_format="markdown", link_format="markdown")
+        data = converter.to_dict()
+        assert data == {
+            "type": "haystack.components.converters.docx.DOCXToDocument",
+            "init_parameters": {"store_full_path": False, "table_format": "markdown", "link_format": "markdown"},
+        }
+
+        converter = DOCXToDocument(table_format="csv", link_format="plain")
+        data = converter.to_dict()
+        assert data == {
+            "type": "haystack.components.converters.docx.DOCXToDocument",
+            "init_parameters": {"store_full_path": False, "table_format": "csv", "link_format": "plain"},
+        }
+
+        converter = DOCXToDocument(table_format=DOCXTableFormat.MARKDOWN, link_format=DOCXLinkFormat.MARKDOWN)
+        data = converter.to_dict()
+        assert data == {
+            "type": "haystack.components.converters.docx.DOCXToDocument",
+            "init_parameters": {"store_full_path": False, "table_format": "markdown", "link_format": "markdown"},
+        }
+
+        converter = DOCXToDocument(table_format=DOCXTableFormat.CSV, link_format=DOCXLinkFormat.PLAIN)
+        data = converter.to_dict()
+        assert data == {
+            "type": "haystack.components.converters.docx.DOCXToDocument",
+            "init_parameters": {"store_full_path": False, "table_format": "csv", "link_format": "plain"},
+        }
+
+    def test_from_dict(self):
+        data = {
+            "type": "haystack.components.converters.docx.DOCXToDocument",
+            "init_parameters": {"table_format": "csv"},
+        }
+        converter = DOCXToDocument.from_dict(data)
+        assert converter.table_format == DOCXTableFormat.CSV
+
+    def test_from_dict_custom_parameters(self):
+        data = {
+            "type": "haystack.components.converters.docx.DOCXToDocument",
+            "init_parameters": {"table_format": "markdown", "link_format": "markdown"},
+        }
+        converter = DOCXToDocument.from_dict(data)
+        assert converter.table_format == DOCXTableFormat.MARKDOWN
+        assert converter.link_format == DOCXLinkFormat.MARKDOWN
+
+    def test_from_dict_invalid_table_format(self):
+        data = {
+            "type": "haystack.components.converters.docx.DOCXToDocument",
+            "init_parameters": {"table_format": "invalid_format"},
+        }
+        with pytest.raises(ValueError, match="Unknown table format 'invalid_format'"):
+            DOCXToDocument.from_dict(data)
+
+    def test_from_dict_empty_init_parameters(self):
+        data = {"type": "haystack.components.converters.docx.DOCXToDocument", "init_parameters": {}}
+        converter = DOCXToDocument.from_dict(data)
+        assert converter.table_format == DOCXTableFormat.CSV
+
+    def test_pipeline_serde(self):
+        pipeline = Pipeline()
+        converter = DOCXToDocument(table_format=DOCXTableFormat.MARKDOWN)
+        pipeline.add_component("converter", converter)
+
+        pipeline_str = pipeline.dumps()
+        assert "haystack.components.converters.docx.DOCXToDocument" in pipeline_str
+        assert "table_format" in pipeline_str
+        assert "markdown" in pipeline_str
+
+        new_pipeline = Pipeline.loads(pipeline_str)
+        new_converter = new_pipeline.get_component("converter")
+        assert isinstance(new_converter, DOCXToDocument)
+        assert new_converter.table_format == DOCXTableFormat.MARKDOWN
+
+    def test_run(self, test_files_path, docx_converter):
+        """
+        Test if the component runs correctly
+        """
+        paths = [test_files_path / "docx" / "sample_docx_1.docx"]
+        output = docx_converter.run(sources=paths)
+        docs = output["documents"]
+        assert len(docs) == 1
+        assert "History" in docs[0].content
+        assert docs[0].meta.keys() == {"file_path", "docx"}
+        assert docs[0].meta == {
+            "file_path": os.path.basename(paths[0]),
+            "docx": {
+                "author": "Microsoft Office User",
+                "category": "",
+                "comments": "",
+                "content_status": "",
+                "created": "2024-06-09T21:17:00+00:00",
+                "identifier": "",
+                "keywords": "",
+                "language": "",
+                "last_modified_by": "Carlos Fernández Lorán",
+                "last_printed": None,
+                "modified": "2024-06-09T21:27:00+00:00",
+                "revision": 2,
+                "subject": "",
+                "title": "",
+                "version": "",
+            },
+        }
+
+    def test_run_with_table(self, test_files_path):
+        """
+        Test if the component runs correctly
+        """
+        docx_converter = DOCXToDocument(table_format=DOCXTableFormat.MARKDOWN)
+        paths = [test_files_path / "docx" / "sample_docx.docx"]
+        output = docx_converter.run(sources=paths)
+        docs = output["documents"]
+        assert len(docs) == 1
+        assert "Donald Trump" in docs[0].content  ## :-)
+        assert docs[0].meta.keys() == {"file_path", "docx"}
+        assert docs[0].meta == {
+            "file_path": os.path.basename(paths[0]),
+            "docx": {
+                "author": "Saha, Anirban",
+                "category": "",
+                "comments": "",
+                "content_status": "",
+                "created": "2020-07-14T08:14:00+00:00",
+                "identifier": "",
+                "keywords": "",
+                "language": "",
+                "last_modified_by": "Saha, Anirban",
+                "last_printed": None,
+                "modified": "2020-07-14T08:16:00+00:00",
+                "revision": 1,
+                "subject": "",
+                "title": "",
+                "version": "",
+            },
+        }
+        # let's now detect that the table markdown is correctly added and that order of elements is correct
+        content_parts = docs[0].content.split("\n\n")
+        table_index = next(i for i, part in enumerate(content_parts) if "| This | Is     | Just a |" in part)
+        # check that natural order of the document is preserved
+        assert any("Donald Trump" in part for part in content_parts[:table_index]), "Text before table not found"
+        assert any("Now we are in Page 2" in part for part in content_parts[table_index + 1 :]), (
+            "Text after table not found"
+        )
+
+    def test_run_with_store_full_path_false(self, test_files_path):
+        """
+        Test if the component runs correctly with store_full_path=False
+        """
+        docx_converter = DOCXToDocument(store_full_path=False)
+        paths = [test_files_path / "docx" / "sample_docx_1.docx"]
+        output = docx_converter.run(sources=paths)
+        docs = output["documents"]
+        assert len(docs) == 1
+        assert "History" in docs[0].content
+        assert docs[0].meta.keys() == {"file_path", "docx"}
+        assert docs[0].meta == {
+            "file_path": "sample_docx_1.docx",
+            "docx": {
+                "author": "Microsoft Office User",
+                "category": "",
+                "comments": "",
+                "content_status": "",
+                "created": "2024-06-09T21:17:00+00:00",
+                "identifier": "",
+                "keywords": "",
+                "language": "",
+                "last_modified_by": "Carlos Fernández Lorán",
+                "last_printed": None,
+                "modified": "2024-06-09T21:27:00+00:00",
+                "revision": 2,
+                "subject": "",
+                "title": "",
+                "version": "",
+            },
+        }
+
+    @pytest.mark.parametrize("table_format", ["markdown", "csv"])
+    def test_table_between_two_paragraphs(self, test_files_path, table_format):
+        docx_converter = DOCXToDocument(table_format=table_format)
+        paths = [test_files_path / "docx" / "sample_docx_3.docx"]
+        output = docx_converter.run(sources=paths)
+
+        content = output["documents"][0].content
+
+        paragraphs_one = content.find("Table: AI Use Cases in Different Industries")
+        paragraphs_two = content.find("Paragraph 2:")
+        table = content[
+            paragraphs_one + len("Table: AI Use Cases in Different Industries") + 1 : paragraphs_two
+        ].strip()
+
+        if table_format == "markdown":
+            split = list(filter(None, table.split("\n")))
+            expected_table_header = "| Industry   | AI Use Case                    | Impact                    |"
+            expected_last_row = "| Finance    | Fraud detection and prevention | Reduced financial losses  |"
+
+            assert split[0] == expected_table_header
+            assert split[-1] == expected_last_row
+        if table_format == "csv":  # CSV format
+            csv_reader = csv.reader(StringIO(table))
+            rows = list(csv_reader)
+            assert len(rows) == 3  # Header + 2 data rows
+            assert rows[0] == ["Industry", "AI Use Case", "Impact"]
+            assert rows[-1] == ["Finance", "Fraud detection and prevention", "Reduced financial losses"]
+
+    @pytest.mark.parametrize("table_format", ["markdown", "csv"])
+    def test_table_content_correct_parsing(self, test_files_path, table_format):
+        docx_converter = DOCXToDocument(table_format=table_format)
+        paths = [test_files_path / "docx" / "sample_docx_3.docx"]
+        output = docx_converter.run(sources=paths)
+        content = output["documents"][0].content
+
+        paragraphs_one = content.find("Table: AI Use Cases in Different Industries")
+        paragraphs_two = content.find("Paragraph 2:")
+        table = content[
+            paragraphs_one + len("Table: AI Use Cases in Different Industries") + 1 : paragraphs_two
+        ].strip()
+
+        if table_format == "markdown":
+            split = list(filter(None, table.split("\n")))
+            assert len(split) == 4
+
+            expected_table_header = "| Industry   | AI Use Case                    | Impact                    |"
+            expected_table_top_border = "| ---------- | ------------------------------ | ------------------------- |"
+            expected_table_row_one = "| Healthcare | Predictive diagnostics         | Improved patient outcomes |"
+            expected_table_row_two = "| Finance    | Fraud detection and prevention | Reduced financial losses  |"
+
+            assert split[0] == expected_table_header
+            assert split[1] == expected_table_top_border
+            assert split[2] == expected_table_row_one
+            assert split[3] == expected_table_row_two
+        if table_format == "csv":  # CSV format
+            csv_reader = csv.reader(StringIO(table))
+            rows = list(csv_reader)
+            assert len(rows) == 3  # Header + 2 data rows
+
+            expected_header = ["Industry", "AI Use Case", "Impact"]
+            expected_row_one = ["Healthcare", "Predictive diagnostics", "Improved patient outcomes"]
+            expected_row_two = ["Finance", "Fraud detection and prevention", "Reduced financial losses"]
+
+            assert rows[0] == expected_header
+            assert rows[1] == expected_row_one
+            assert rows[2] == expected_row_two
+
+    def test_run_with_additional_meta(self, test_files_path, docx_converter):
+        paths = [test_files_path / "docx" / "sample_docx_1.docx"]
+        output = docx_converter.run(sources=paths, meta={"language": "it", "author": "test_author"})
+        doc = output["documents"][0]
+        assert doc.meta == {
+            "file_path": os.path.basename(paths[0]),
+            "docx": {
+                "author": "Microsoft Office User",
+                "category": "",
+                "comments": "",
+                "content_status": "",
+                "created": "2024-06-09T21:17:00+00:00",
+                "identifier": "",
+                "keywords": "",
+                "language": "",
+                "last_modified_by": "Carlos Fernández Lorán",
+                "last_printed": None,
+                "modified": "2024-06-09T21:27:00+00:00",
+                "revision": 2,
+                "subject": "",
+                "title": "",
+                "version": "",
+            },
+            "language": "it",
+            "author": "test_author",
+        }
+
+    def test_run_error_wrong_file_type(self, caplog, test_files_path, docx_converter):
+        sources = [str(test_files_path / "txt" / "doc_1.txt")]
+        with caplog.at_level(logging.WARNING):
+            results = docx_converter.run(sources=sources)
+            assert "doc_1.txt and convert it" in caplog.text
+            assert results["documents"] == []
+
+    def test_run_error_non_existent_file(self, docx_converter, caplog):
+        """
+        Test if the component correctly handles errors.
+        """
+        paths = ["non_existing_file.docx"]
+        with caplog.at_level(logging.WARNING):
+            docx_converter.run(sources=paths)
+            assert "Could not read non_existing_file.docx" in caplog.text
+
+    def test_run_page_breaks(self, test_files_path, docx_converter):
+        """
+        Test if the component correctly parses page breaks.
+        """
+        paths = [test_files_path / "docx" / "sample_docx_2_page_breaks.docx"]
+        output = docx_converter.run(sources=paths)
+        docs = output["documents"]
+        assert len(docs) == 1
+        assert docs[0].content.count("\f") == 4
+
+    def test_mixed_sources_run(self, test_files_path, docx_converter):
+        """
+        Test if the component runs correctly when mixed sources are provided.
+        """
+        paths = [test_files_path / "docx" / "sample_docx_1.docx"]
+        with open(test_files_path / "docx" / "sample_docx_1.docx", "rb") as f:
+            paths.append(ByteStream(f.read()))
+
+        output = docx_converter.run(sources=paths)
+        docs = output["documents"]
+        assert len(docs) == 2
+        assert "History and standardization" in docs[0].content
+        assert "History and standardization" in docs[1].content
+
+    def test_document_with_docx_metadata_to_dict(self):
+        docx_metadata = DOCXMetadata(
+            author="Microsoft Office User",
+            category="category",
+            comments="comments",
+            content_status="",
+            created="2024-06-09T21:17:00+00:00",
+            identifier="",
+            keywords="",
+            language="",
+            last_modified_by="Carlos Fernández Lorán",
+            last_printed=None,
+            modified="2024-06-09T21:27:00+00:00",
+            revision=2,
+            subject="",
+            title="",
+            version="",
+        )
+        doc = Document(content="content", meta={"test": 1, "docx": docx_metadata}, id="1")
+        assert doc.to_dict(flatten=False) == {
+            "blob": None,
+            "content": "content",
+            "id": "1",
+            "score": None,
+            "embedding": None,
+            "sparse_embedding": None,
+            "meta": {
+                "test": 1,
+                "docx": {
+                    "author": "Microsoft Office User",
+                    "category": "category",
+                    "comments": "comments",
+                    "content_status": "",
+                    "created": "2024-06-09T21:17:00+00:00",
+                    "identifier": "",
+                    "keywords": "",
+                    "language": "",
+                    "last_modified_by": "Carlos Fernández Lorán",
+                    "last_printed": None,
+                    "modified": "2024-06-09T21:27:00+00:00",
+                    "revision": 2,
+                    "subject": "",
+                    "title": "",
+                    "version": "",
+                },
+            },
+        }
+
+        # check it is JSON serializable
+        json_str = json.dumps(doc.to_dict(flatten=False))
+        assert json.loads(json_str) == doc.to_dict(flatten=False)
+
+    def test_link_format_initialization(self):
+        converter = DOCXToDocument(link_format="markdown")
+        assert converter.link_format == DOCXLinkFormat.MARKDOWN
+
+        converter = DOCXToDocument(link_format=DOCXLinkFormat.PLAIN)
+        assert converter.link_format == DOCXLinkFormat.PLAIN
+
+    def test_link_format_invalid(self):
+        with pytest.raises(ValueError, match="Unknown link format 'invalid_format'"):
+            DOCXToDocument(link_format="invalid_format")
+
+    @pytest.mark.parametrize("link_format", ["markdown", "plain"])
+    def test_link_extraction(self, test_files_path, link_format):
+        docx_converter = DOCXToDocument(link_format=link_format)
+        paths = [test_files_path / "docx" / "sample_docx_with_single_link.docx"]
+        output = docx_converter.run(sources=paths)
+        content = output["documents"][0].content
+
+        if link_format == "markdown":
+            assert "[PDF](https://en.wikipedia.org/wiki/PDF)" in content
+        else:  # plain format
+            assert "PDF (https://en.wikipedia.org/wiki/PDF)" in content
+
+    @pytest.mark.parametrize("link_format", ["markdown", "plain"])
+    def test_link_extraction_page_break(self, test_files_path, link_format):
+        docx_converter = DOCXToDocument(link_format=link_format)
+        paths = [test_files_path / "docx" / "sample_docx_with_links.docx"]
+        output = docx_converter.run(sources=paths)
+        content = output["documents"][0].content
+
+        if link_format == "markdown":
+            assert "[PDF](https://en.wikipedia.org/wiki/PDF)" in content
+            assert "[of](https://en.wikipedia.org/wiki/OF)" in content
+            assert "[charge](https://en.wikipedia.org/wiki/Charge)" in content
+            assert "[disambiguation link](https://en.wikipedia.org/wiki/PDF_(disambiguation))" in content
+        else:  # plain format
+            assert "PDF (https://en.wikipedia.org/wiki/PDF)" in content
+            assert "of (https://en.wikipedia.org/wiki/OF)" in content
+            assert "charge (https://en.wikipedia.org/wiki/Charge)" in content
+            assert "disambiguation link (https://en.wikipedia.org/wiki/PDF_(disambiguation))" in content
+
+    def test_no_link_extraction(self, test_files_path):
+        docx_converter = DOCXToDocument()
+        paths = [test_files_path / "docx" / "sample_docx_with_single_link.docx"]
+        output = docx_converter.run(sources=paths)
+        content = output["documents"][0].content
+
+        assert "[PDF](https://en.wikipedia.org/wiki/PDF)" not in content
+        assert "PDF (https://en.wikipedia.org/wiki/PDF)" not in content

test/components/converters/test_file_to_file_content.py

@@ -0,0 +1,138 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import base64
+from pathlib import Path
+
+import pytest
+
+from haystack.components.converters.file_to_file_content import FileToFileContent
+from haystack.core.serialization import component_from_dict, component_to_dict
+from haystack.dataclasses import ByteStream
+
+
+class TestFileToFileContent:
+    def test_to_dict(self) -> None:
+        converter = FileToFileContent()
+        assert component_to_dict(converter, "converter") == {
+            "init_parameters": {},
+            "type": "haystack.components.converters.file_to_file_content.FileToFileContent",
+        }
+
+    def test_from_dict(self) -> None:
+        data = {"init_parameters": {}, "type": "haystack.components.converters.file_to_file_content.FileToFileContent"}
+        converter = component_from_dict(FileToFileContent, data, "name")
+        assert component_to_dict(converter, "converter") == data
+
+    @pytest.mark.parametrize(
+        ("file_path", "mime_type"),
+        [
+            ("./test/test_files/pdf/sample_pdf_1.pdf", "application/pdf"),
+            ("./test/test_files/txt/doc_1.txt", "text/plain"),
+        ],
+    )
+    def test_run_with_valid_sources(self, file_path: str, mime_type: str) -> None:
+        converter = FileToFileContent()
+        results = converter.run(sources=[file_path])
+
+        assert len(results["file_contents"]) == 1
+        file_content = results["file_contents"][0]
+
+        assert file_content.base64_data is not None
+        assert file_content.mime_type == mime_type
+        assert file_content.filename == Path(file_path).name
+        assert file_content.extra == {}
+
+        with open(file_path, "rb") as f:
+            expected_base64 = base64.b64encode(f.read()).decode("utf-8")
+        assert file_content.base64_data == expected_base64
+
+    @pytest.mark.parametrize(
+        ("file_path", "mime_type"),
+        [
+            ("./test/test_files/pdf/sample_pdf_1.pdf", "application/pdf"),
+            ("./test/test_files/audio/answer.wav", "audio/x-wav"),
+        ],
+    )
+    def test_run_with_bytestream_sources(self, file_path: str, mime_type: str) -> None:
+        byte_stream = ByteStream.from_file_path(Path(file_path))
+
+        converter = FileToFileContent()
+        results = converter.run(sources=[byte_stream])
+
+        assert len(results["file_contents"]) == 1
+        file_content = results["file_contents"][0]
+
+        assert file_content.base64_data is not None
+        assert file_content.mime_type == mime_type
+        assert file_content.filename is None
+        assert file_content.extra == {}
+
+        expected_base64 = base64.b64encode(byte_stream.data).decode("utf-8")
+        assert file_content.base64_data == expected_base64
+
+    def test_run_with_no_sources(self) -> None:
+        converter = FileToFileContent()
+        results = converter.run(sources=[])
+        assert results == {"file_contents": []}
+
+    def test_run_with_invalid_source_type(self, caplog) -> None:
+        converter = FileToFileContent()
+        converter.run(sources=[123])
+        assert "Could not read" in caplog.text
+
+    def test_run_with_non_existent_file(self, caplog) -> None:
+        converter = FileToFileContent()
+        converter.run(sources=["./non_existent_file.pdf"])
+        assert "Could not read" in caplog.text
+
+    def test_run_with_empty_bytestream(self) -> None:
+        byte_stream = ByteStream(data=b"")
+
+        converter = FileToFileContent()
+        results = converter.run(sources=[byte_stream])
+
+        assert results["file_contents"] == []
+
+    def test_run_with_extra_dict(self) -> None:
+        converter = FileToFileContent()
+        extra = {"key": "value"}
+        results = converter.run(sources=["./test/test_files/txt/doc_1.txt"], extra=extra)
+
+        assert len(results["file_contents"]) == 1
+        assert results["file_contents"][0].extra == extra
+
+    def test_run_with_extra_list(self) -> None:
+        converter = FileToFileContent()
+        sources = ["./test/test_files/txt/doc_1.txt", "./test/test_files/txt/doc_2.txt"]
+        extra = [{"key": "value1"}, {"key": "value2"}]
+        results = converter.run(sources=sources, extra=extra)
+
+        assert len(results["file_contents"]) == 2
+        assert results["file_contents"][0].extra == {"key": "value1"}
+        assert results["file_contents"][1].extra == {"key": "value2"}
+
+    def test_run_with_extra_dict_does_not_share_reference(self) -> None:
+        # A single ``extra`` dict is applied to every source; each FileContent must get its own copy
+        # so that mutating one file's ``extra`` downstream does not leak into the others.
+        converter = FileToFileContent()
+        sources = ["./test/test_files/txt/doc_1.txt", "./test/test_files/txt/doc_2.txt"]
+        results = converter.run(sources=sources, extra={"tenant": "acme"})
+
+        file_contents = results["file_contents"]
+        assert len(file_contents) == 2
+        assert file_contents[0].extra is not file_contents[1].extra
+
+        file_contents[0].extra["page"] = 1
+        assert "page" not in file_contents[1].extra
+
+    def test_run_skips_empty_files_among_valid(self, caplog) -> None:
+        byte_stream_empty = ByteStream(data=b"")
+        valid_source = "./test/test_files/txt/doc_1.txt"
+
+        converter = FileToFileContent()
+        results = converter.run(sources=[byte_stream_empty, valid_source])
+
+        assert len(results["file_contents"]) == 1
+        assert "empty" in caplog.text

test/components/converters/test_html_to_document.py

@@ -0,0 +1,254 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from haystack.components.converters import HTMLToDocument
+from haystack.dataclasses import ByteStream
+
+
+class TestHTMLToDocument:
+    def test_run(self, test_files_path):
+        """
+        Test if the component runs correctly.
+        """
+        sources = [test_files_path / "html" / "what_is_haystack.html"]
+        converter = HTMLToDocument()
+        results = converter.run(sources=sources, meta={"test": "TEST"})
+        docs = results["documents"]
+        assert len(docs) == 1
+        assert "Haystack" in docs[0].content
+        assert docs[0].meta["test"] == "TEST"
+
+    def test_run_doc_metadata(self, test_files_path):
+        """
+        Test if the component runs correctly when metadata is supplied by the user.
+        """
+        converter = HTMLToDocument()
+        sources = [test_files_path / "html" / "what_is_haystack.html"]
+        metadata = [{"file_name": "what_is_haystack.html"}]
+        results = converter.run(sources=sources, meta=metadata)
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        assert "Haystack" in docs[0].content
+        assert docs[0].meta["file_name"] == "what_is_haystack.html"
+
+    def test_run_with_store_full_path(self, test_files_path):
+        """
+        Test if the component runs correctly when metadata is supplied by the user.
+        """
+        converter = HTMLToDocument(store_full_path=True)
+        sources = [test_files_path / "html" / "what_is_haystack.html"]
+
+        results = converter.run(sources=sources)  # store_full_path is True by default
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        assert "Haystack" in docs[0].content
+        assert docs[0].meta["file_path"] == str(sources[0])
+
+        converter_2 = HTMLToDocument(store_full_path=False)
+        results = converter_2.run(sources=sources)
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        assert "Haystack" in docs[0].content
+        assert docs[0].meta["file_path"] == "what_is_haystack.html"
+
+    def test_incorrect_meta(self, test_files_path):
+        """
+        Test if the component raises an error when incorrect metadata is supplied by the user.
+        """
+        converter = HTMLToDocument()
+        sources = [test_files_path / "html" / "what_is_haystack.html"]
+        metadata = [{"file_name": "what_is_haystack.html"}, {"file_name": "haystack.html"}]
+        with pytest.raises(ValueError, match="The length of the metadata list must match the number of sources."):
+            converter.run(sources=sources, meta=metadata)
+
+    def test_run_bytestream_metadata(self, test_files_path):
+        """
+        Test if the component runs correctly when metadata is read from the ByteStream object.
+        """
+        converter = HTMLToDocument()
+        with open(test_files_path / "html" / "what_is_haystack.html", "rb") as file:
+            byte_stream = file.read()
+            stream = ByteStream(byte_stream, meta={"content_type": "text/html", "url": "test_url"})
+
+        results = converter.run(sources=[stream])
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        assert "Haystack" in docs[0].content
+        assert docs[0].meta == {"content_type": "text/html", "url": "test_url"}
+
+    def test_run_bytestream_and_doc_metadata(self, test_files_path):
+        """
+        Test if the component runs correctly when metadata is read from the ByteStream object and supplied by the user.
+
+        There is no overlap between the metadata received.
+        """
+        converter = HTMLToDocument()
+        with open(test_files_path / "html" / "what_is_haystack.html", "rb") as file:
+            byte_stream = file.read()
+            stream = ByteStream(byte_stream, meta={"content_type": "text/html", "url": "test_url"})
+
+        metadata = [{"file_name": "what_is_haystack.html"}]
+        results = converter.run(sources=[stream], meta=metadata)
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        assert "Haystack" in docs[0].content
+        assert docs[0].meta == {"file_name": "what_is_haystack.html", "content_type": "text/html", "url": "test_url"}
+
+    def test_run_bytestream_doc_overlapping_metadata(self, test_files_path):
+        """
+        Test if the component runs correctly when metadata is read from the ByteStream object and supplied by the user.
+
+        There is an overlap between the metadata received.
+
+        The component should use the supplied metadata to overwrite the values if there is an overlap between the keys.
+        """
+        converter = HTMLToDocument()
+        with open(test_files_path / "html" / "what_is_haystack.html", "rb") as file:
+            byte_stream = file.read()
+            # ByteStream has "url" present in metadata
+            stream = ByteStream(byte_stream, meta={"content_type": "text/html", "url": "test_url_correct"})
+
+        # "url" supplied by the user overwrites value present in metadata
+        metadata = [{"file_name": "what_is_haystack.html", "url": "test_url_new"}]
+        results = converter.run(sources=[stream], meta=metadata)
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        assert "Haystack" in docs[0].content
+        assert docs[0].meta == {
+            "file_name": "what_is_haystack.html",
+            "content_type": "text/html",
+            "url": "test_url_new",
+        }
+
+    def test_run_wrong_file_type(self, test_files_path, caplog):
+        """
+        Test if the component runs correctly when an input file is not of the expected type.
+        """
+        sources = [test_files_path / "audio" / "answer.wav"]
+        converter = HTMLToDocument()
+        with caplog.at_level(logging.WARNING):
+            results = converter.run(sources=sources)
+            assert "Failed to extract text from" in caplog.text
+
+        assert results["documents"] == []
+
+    def test_run_error_handling(self, caplog):
+        """
+        Test if the component correctly handles errors.
+        """
+        sources = ["non_existing_file.html"]
+        converter = HTMLToDocument()
+        with caplog.at_level(logging.WARNING):
+            results = converter.run(sources=sources)
+            assert "Could not read non_existing_file.html" in caplog.text
+            assert results["documents"] == []
+
+    def test_run_empty_bytestream(self, caplog):
+        """
+        Test that an empty ByteStream is skipped without invoking extraction,
+        so no noisy lxml parse errors are emitted.
+        """
+        empty_stream = ByteStream(data=b"")
+        empty_stream.mime_type = "text/html"
+        converter = HTMLToDocument()
+
+        with patch("haystack.components.converters.html.extract") as mock_extract:
+            with caplog.at_level(logging.WARNING):
+                results = converter.run(sources=[empty_stream])
+
+        assert results["documents"] == []
+        mock_extract.assert_not_called()
+        assert "because it is empty" in caplog.text
+
+    def test_mixed_sources_run(self, test_files_path):
+        """
+        Test if the component runs correctly if the input is a mix of paths and ByteStreams.
+        """
+        sources = [
+            test_files_path / "html" / "what_is_haystack.html",
+            str((test_files_path / "html" / "what_is_haystack.html").absolute()),
+        ]
+        with open(test_files_path / "html" / "what_is_haystack.html", "rb") as f:
+            byte_stream = f.read()
+            sources.append(ByteStream(byte_stream))
+
+        converter = HTMLToDocument()
+        results = converter.run(sources=sources)
+        docs = results["documents"]
+        assert len(docs) == 3
+        for doc in docs:
+            assert "Haystack" in doc.content
+
+    def test_bytestream_encoding_from_meta(self):
+        """
+        Test that a non-UTF-8 ByteStream is decoded using the encoding specified in its meta.
+        """
+        # "caf\xe9" is "café" in latin-1; decoding as utf-8 would raise UnicodeDecodeError.
+        latin1_html = b"<html><body><p>caf\xe9</p></body></html>"
+        bytestream = ByteStream(data=latin1_html, meta={"encoding": "latin-1"})
+
+        converter = HTMLToDocument()
+        results = converter.run(sources=[bytestream])
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        assert "café" in docs[0].content
+
+    def test_bytestream_encoding_from_init(self):
+        """
+        Test that the encoding passed to __init__ is used as a fallback when not set in ByteStream meta.
+        """
+        latin1_html = b"<html><body><p>caf\xe9</p></body></html>"
+        bytestream = ByteStream(data=latin1_html)
+
+        converter = HTMLToDocument(encoding="latin-1")
+        results = converter.run(sources=[bytestream])
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        assert "café" in docs[0].content
+
+    def test_serde(self):
+        """
+        Test if the component runs correctly gets serialized and deserialized.
+        """
+        converter = HTMLToDocument(encoding="latin-1")
+        serde_data = converter.to_dict()
+        new_converter = HTMLToDocument.from_dict(serde_data)
+        assert new_converter.extraction_kwargs == converter.extraction_kwargs
+        assert new_converter.encoding == converter.encoding
+
+    def test_run_difficult_html(self, test_files_path):
+        converter = HTMLToDocument()
+        result = converter.run(sources=[Path(test_files_path / "html" / "paul_graham_superlinear.html")])
+
+        assert len(result["documents"]) == 1
+        assert "Superlinear" in result["documents"][0].content
+
+    @patch("haystack.components.converters.html.extract", return_value="test")
+    def test_run_with_extraction_kwargs(self, mock_extract, test_files_path):
+        sources = [test_files_path / "html" / "what_is_haystack.html"]
+
+        converter = HTMLToDocument()
+        converter.run(sources=sources)
+        assert mock_extract.call_count == 1
+        assert "favor_precision" not in mock_extract.call_args[1]
+
+        precise_converter = HTMLToDocument(extraction_kwargs={"favor_precision": True})
+        mock_extract.reset_mock()
+        precise_converter.run(sources=sources)
+        assert mock_extract.call_count == 1
+        assert mock_extract.call_args[1]["favor_precision"] is True

test/components/converters/test_json.py

@@ -0,0 +1,518 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+import logging
+import os
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from haystack.components.converters import JSONConverter
+from haystack.dataclasses import ByteStream
+
+test_data = [
+    {
+        "year": "1997",
+        "category": "literature",
+        "laureates": [
+            {
+                "id": "674",
+                "firstname": "Dario",
+                "surname": "Fokin",
+                "motivation": "who emulates the jesters of the Middle Ages in scourging authority and upholding the "
+                "dignity of the downtrodden",
+                "share": "1",
+            }
+        ],
+    },
+    {
+        "year": "1986",
+        "category": "medicine",
+        "laureates": [
+            {
+                "id": "434",
+                "firstname": "Stanley",
+                "surname": "Cohen",
+                "motivation": "for their discoveries of growth factors",
+                "share": "2",
+            },
+            {
+                "id": "435",
+                "firstname": "Rita",
+                "surname": "Levi-Montalcini",
+                "motivation": "for their discoveries of growth factors",
+                "share": "2",
+            },
+        ],
+    },
+    {
+        "year": "1938",
+        "category": "physics",
+        "laureates": [
+            {
+                "id": "46",
+                "firstname": "Enrico",
+                "surname": "Fermi",
+                "motivation": "for his demonstrations of the existence of new radioactive elements produced by neutron "
+                "irradiation, and for his related discovery of nuclear reactions brought about by slow "
+                "neutrons",
+                "share": "1",
+            }
+        ],
+    },
+]
+
+
+def test_init_without_jq_schema_and_content_key():
+    with pytest.raises(
+        ValueError, match="No `jq_schema` nor `content_key` specified. Set either or both to extract data."
+    ):
+        JSONConverter()
+
+
+@patch("haystack.components.converters.json.jq_import")
+def test_init_without_jq_schema_and_missing_dependency(jq_import):
+    converter = JSONConverter(content_key="foo")
+    jq_import.check.assert_not_called()
+    assert converter._jq_schema is None
+    assert converter._content_key == "foo"
+    assert converter._meta_fields is None
+
+
+@patch("haystack.components.converters.json.jq_import")
+def test_init_with_jq_schema_and_missing_dependency(jq_import):
+    jq_import.check.side_effect = ImportError
+    with pytest.raises(ImportError):
+        JSONConverter(jq_schema=".laureates[].motivation")
+
+
+def test_init_with_jq_schema():
+    converter = JSONConverter(jq_schema=".")
+    assert converter._jq_schema == "."
+    assert converter._content_key is None
+    assert converter._meta_fields is None
+
+
+def test_to_dict():
+    converter = JSONConverter(
+        jq_schema=".laureates[]", content_key="motivation", extra_meta_fields={"firstname", "surname"}
+    )
+
+    assert converter.to_dict() == {
+        "type": "haystack.components.converters.json.JSONConverter",
+        "init_parameters": {
+            "content_key": "motivation",
+            "jq_schema": ".laureates[]",
+            "extra_meta_fields": {"firstname", "surname"},
+            "store_full_path": False,
+        },
+    }
+
+
+def test_from_dict():
+    data = {
+        "type": "haystack.components.converters.json.JSONConverter",
+        "init_parameters": {
+            "content_key": "motivation",
+            "jq_schema": ".laureates[]",
+            "extra_meta_fields": ["firstname", "surname"],
+            "store_full_path": True,
+        },
+    }
+    converter = JSONConverter.from_dict(data)
+
+    assert converter._jq_schema == ".laureates[]"
+    assert converter._content_key == "motivation"
+    assert converter._meta_fields == ["firstname", "surname"]
+
+
+def test_run(tmpdir):
+    first_test_file = Path(tmpdir / "first_test_file.json")
+    second_test_file = Path(tmpdir / "second_test_file.json")
+
+    first_test_file.write_text(json.dumps(test_data[0]), "utf-8")
+    second_test_file.write_text(json.dumps(test_data[1]), "utf-8")
+    byte_stream = ByteStream.from_string(json.dumps(test_data[2]))
+
+    sources = [str(first_test_file), second_test_file, byte_stream]
+
+    converter = JSONConverter(jq_schema='.laureates[] | .firstname + " " + .surname + " " + .motivation')
+    result = converter.run(sources=sources)
+    assert len(result) == 1
+    assert len(result["documents"]) == 4
+    assert (
+        result["documents"][0].content
+        == "Dario Fokin who emulates the jesters of the Middle Ages in scourging authority and "
+        "upholding the dignity of the downtrodden"
+    )
+    assert result["documents"][0].meta == {"file_path": os.path.basename(first_test_file)}
+    assert result["documents"][1].content == "Stanley Cohen for their discoveries of growth factors"
+    assert result["documents"][1].meta == {"file_path": os.path.basename(second_test_file)}
+    assert result["documents"][2].content == "Rita Levi-Montalcini for their discoveries of growth factors"
+    assert result["documents"][2].meta == {"file_path": os.path.basename(second_test_file)}
+    assert (
+        result["documents"][3].content == "Enrico Fermi for his demonstrations of the existence of new "
+        "radioactive elements produced by neutron irradiation, and for his related discovery of nuclear "
+        "reactions brought about by slow neutrons"
+    )
+    assert result["documents"][3].meta == {}
+
+
+def test_run_with_store_full_path_false(tmpdir):
+    """
+    Test if the component runs correctly with store_full_path=False
+    """
+    first_test_file = Path(tmpdir / "first_test_file.json")
+    second_test_file = Path(tmpdir / "second_test_file.json")
+
+    first_test_file.write_text(json.dumps(test_data[0]), "utf-8")
+    second_test_file.write_text(json.dumps(test_data[1]), "utf-8")
+    byte_stream = ByteStream.from_string(json.dumps(test_data[2]))
+
+    sources = [str(first_test_file), second_test_file, byte_stream]
+
+    converter = JSONConverter(
+        jq_schema='.laureates[] | .firstname + " " + .surname + " " + .motivation', store_full_path=False
+    )
+    result = converter.run(sources=sources)
+    assert len(result) == 1
+    assert len(result["documents"]) == 4
+    assert (
+        result["documents"][0].content
+        == "Dario Fokin who emulates the jesters of the Middle Ages in scourging authority and "
+        "upholding the dignity of the downtrodden"
+    )
+    assert result["documents"][0].meta == {"file_path": "first_test_file.json"}
+    assert result["documents"][1].content == "Stanley Cohen for their discoveries of growth factors"
+    assert result["documents"][1].meta == {"file_path": "second_test_file.json"}
+    assert result["documents"][2].content == "Rita Levi-Montalcini for their discoveries of growth factors"
+    assert result["documents"][2].meta == {"file_path": "second_test_file.json"}
+    assert (
+        result["documents"][3].content == "Enrico Fermi for his demonstrations of the existence of new "
+        "radioactive elements produced by neutron irradiation, and for his related discovery of nuclear "
+        "reactions brought about by slow neutrons"
+    )
+    assert result["documents"][3].meta == {}
+
+
+def test_run_with_non_json_file(tmpdir, caplog):
+    test_file = Path(tmpdir / "test_file.md")
+    test_file.write_text("This is not a JSON file.", "utf-8")
+
+    sources = [test_file]
+    converter = JSONConverter(".laureates | .motivation")
+
+    caplog.clear()
+    with caplog.at_level(logging.WARNING):
+        result = converter.run(sources=sources)
+
+    records = caplog.records
+    assert len(records) == 1
+    assert (
+        records[0].msg
+        == f"Failed to extract text from {test_file}. Skipping it. Error: parse error: Invalid numeric literal at "
+        f"line 1, column 5"
+    )
+    assert result == {"documents": []}
+
+
+def test_run_with_bad_filter(tmpdir, caplog):
+    test_file = Path(tmpdir / "test_file.json")
+    test_file.write_text(json.dumps(test_data[0]), "utf-8")
+
+    sources = [test_file]
+    converter = JSONConverter(".laureates | .motivation")
+
+    caplog.clear()
+    with caplog.at_level(logging.WARNING):
+        result = converter.run(sources=sources)
+
+    records = caplog.records
+    assert len(records) == 1
+    assert (
+        records[0].msg
+        == f'Failed to extract text from {test_file}. Skipping it. Error: Cannot index array with string "motivation"'
+    )
+    assert result == {"documents": []}
+
+
+def test_run_with_bad_encoding(tmpdir, caplog):
+    test_file = Path(tmpdir / "test_file.json")
+    test_file.write_text(json.dumps(test_data[0]), "utf-16")
+
+    sources = [test_file]
+    converter = JSONConverter(".laureates")
+
+    caplog.clear()
+    with caplog.at_level(logging.WARNING):
+        result = converter.run(sources=sources)
+
+    records = caplog.records
+    assert len(records) == 1
+    assert records[0].msg.startswith(
+        f"Failed to extract text from {test_file}. Skipping it. Error: 'utf-8' codec can't decode byte"
+    )
+    assert result == {"documents": []}
+
+
+def test_run_with_single_meta(tmpdir):
+    first_test_file = Path(tmpdir / "first_test_file.json")
+    second_test_file = Path(tmpdir / "second_test_file.json")
+
+    first_test_file.write_text(json.dumps(test_data[0]), "utf-8")
+    second_test_file.write_text(json.dumps(test_data[1]), "utf-8")
+    byte_stream = ByteStream.from_string(json.dumps(test_data[2]))
+
+    sources = [str(first_test_file), second_test_file, byte_stream]
+    meta = {"creation_date": "1945-05-25T00:00:00"}
+    converter = JSONConverter(jq_schema='.laureates[] | .firstname + " " + .surname + " " + .motivation')
+    result = converter.run(sources=sources, meta=meta)
+    assert len(result) == 1
+    assert len(result["documents"]) == 4
+    assert (
+        result["documents"][0].content
+        == "Dario Fokin who emulates the jesters of the Middle Ages in scourging authority and "
+        "upholding the dignity of the downtrodden"
+    )
+    assert result["documents"][0].meta == {
+        "file_path": os.path.basename(first_test_file),
+        "creation_date": "1945-05-25T00:00:00",
+    }
+    assert result["documents"][1].content == "Stanley Cohen for their discoveries of growth factors"
+    assert result["documents"][1].meta == {
+        "file_path": os.path.basename(second_test_file),
+        "creation_date": "1945-05-25T00:00:00",
+    }
+    assert result["documents"][2].content == "Rita Levi-Montalcini for their discoveries of growth factors"
+    assert result["documents"][2].meta == {
+        "file_path": os.path.basename(second_test_file),
+        "creation_date": "1945-05-25T00:00:00",
+    }
+    assert (
+        result["documents"][3].content == "Enrico Fermi for his demonstrations of the existence of new "
+        "radioactive elements produced by neutron irradiation, and for his related discovery of nuclear "
+        "reactions brought about by slow neutrons"
+    )
+    assert result["documents"][3].meta == {"creation_date": "1945-05-25T00:00:00"}
+
+
+def test_run_with_meta_list(tmpdir):
+    first_test_file = Path(tmpdir / "first_test_file.json")
+    second_test_file = Path(tmpdir / "second_test_file.json")
+
+    first_test_file.write_text(json.dumps(test_data[0]), "utf-8")
+    second_test_file.write_text(json.dumps(test_data[1]), "utf-8")
+    byte_stream = ByteStream.from_string(json.dumps(test_data[2]))
+
+    sources = [str(first_test_file), second_test_file, byte_stream]
+    meta = [
+        {"creation_date": "1945-05-25T00:00:00"},
+        {"creation_date": "1943-09-03T00:00:00"},
+        {"creation_date": "1989-11-09T00:00:00"},
+    ]
+    converter = JSONConverter(jq_schema='.laureates[] | .firstname + " " + .surname + " " + .motivation')
+    result = converter.run(sources=sources, meta=meta)
+    assert len(result) == 1
+    assert len(result["documents"]) == 4
+    assert (
+        result["documents"][0].content
+        == "Dario Fokin who emulates the jesters of the Middle Ages in scourging authority and "
+        "upholding the dignity of the downtrodden"
+    )
+    assert result["documents"][0].meta == {
+        "file_path": os.path.basename(first_test_file),
+        "creation_date": "1945-05-25T00:00:00",
+    }
+    assert result["documents"][1].content == "Stanley Cohen for their discoveries of growth factors"
+    assert result["documents"][1].meta == {
+        "file_path": os.path.basename(second_test_file),
+        "creation_date": "1943-09-03T00:00:00",
+    }
+    assert result["documents"][2].content == "Rita Levi-Montalcini for their discoveries of growth factors"
+    assert result["documents"][2].meta == {
+        "file_path": os.path.basename(second_test_file),
+        "creation_date": "1943-09-03T00:00:00",
+    }
+    assert (
+        result["documents"][3].content == "Enrico Fermi for his demonstrations of the existence of new "
+        "radioactive elements produced by neutron irradiation, and for his related discovery of nuclear "
+        "reactions brought about by slow neutrons"
+    )
+    assert result["documents"][3].meta == {"creation_date": "1989-11-09T00:00:00"}
+
+
+def test_run_with_meta_list_of_differing_length(tmpdir):
+    sources = ["random_file.json"]
+
+    meta = [{}, {}]
+    converter = JSONConverter(jq_schema=".")
+    with pytest.raises(ValueError, match="The length of the metadata list must match the number of sources."):
+        converter.run(sources=sources, meta=meta)
+
+
+def test_run_with_jq_schema_and_content_key(tmpdir):
+    first_test_file = Path(tmpdir / "first_test_file.json")
+    second_test_file = Path(tmpdir / "second_test_file.json")
+
+    first_test_file.write_text(json.dumps(test_data[0]), "utf-8")
+    second_test_file.write_text(json.dumps(test_data[1]), "utf-8")
+    byte_stream = ByteStream.from_string(json.dumps(test_data[2]))
+
+    sources = [str(first_test_file), second_test_file, byte_stream]
+    converter = JSONConverter(jq_schema=".laureates[]", content_key="motivation")
+    result = converter.run(sources=sources)
+    assert len(result) == 1
+    assert len(result["documents"]) == 4
+    assert (
+        result["documents"][0].content == "who emulates the jesters of the Middle Ages in scourging authority and "
+        "upholding the dignity of the downtrodden"
+    )
+    assert result["documents"][0].meta == {"file_path": os.path.basename(first_test_file)}
+    assert result["documents"][1].content == "for their discoveries of growth factors"
+    assert result["documents"][1].meta == {"file_path": os.path.basename(second_test_file)}
+    assert result["documents"][2].content == "for their discoveries of growth factors"
+    assert result["documents"][2].meta == {"file_path": os.path.basename(second_test_file)}
+    assert (
+        result["documents"][3].content == "for his demonstrations of the existence of new "
+        "radioactive elements produced by neutron irradiation, and for his related discovery of nuclear "
+        "reactions brought about by slow neutrons"
+    )
+    assert result["documents"][3].meta == {}
+
+
+def test_run_with_jq_schema_content_key_and_extra_meta_fields(tmpdir):
+    first_test_file = Path(tmpdir / "first_test_file.json")
+    second_test_file = Path(tmpdir / "second_test_file.json")
+
+    first_test_file.write_text(json.dumps(test_data[0]), "utf-8")
+    second_test_file.write_text(json.dumps(test_data[1]), "utf-8")
+    byte_stream = ByteStream.from_string(json.dumps(test_data[2]))
+
+    sources = [str(first_test_file), second_test_file, byte_stream]
+    converter = JSONConverter(
+        jq_schema=".laureates[]", content_key="motivation", extra_meta_fields={"firstname", "surname"}
+    )
+    result = converter.run(sources=sources)
+    assert len(result) == 1
+    assert len(result["documents"]) == 4
+    assert (
+        result["documents"][0].content == "who emulates the jesters of the Middle Ages in scourging authority and "
+        "upholding the dignity of the downtrodden"
+    )
+    assert result["documents"][0].meta == {
+        "file_path": os.path.basename(first_test_file),
+        "firstname": "Dario",
+        "surname": "Fokin",
+    }
+    assert result["documents"][1].content == "for their discoveries of growth factors"
+    assert result["documents"][1].meta == {
+        "file_path": os.path.basename(second_test_file),
+        "firstname": "Stanley",
+        "surname": "Cohen",
+    }
+    assert result["documents"][2].content == "for their discoveries of growth factors"
+    assert result["documents"][2].meta == {
+        "file_path": os.path.basename(second_test_file),
+        "firstname": "Rita",
+        "surname": "Levi-Montalcini",
+    }
+    assert (
+        result["documents"][3].content == "for his demonstrations of the existence of new "
+        "radioactive elements produced by neutron irradiation, and for his related discovery of nuclear "
+        "reactions brought about by slow neutrons"
+    )
+    assert result["documents"][3].meta == {"firstname": "Enrico", "surname": "Fermi"}
+
+
+def test_run_with_content_key(tmpdir):
+    first_test_file = Path(tmpdir / "first_test_file.json")
+    second_test_file = Path(tmpdir / "second_test_file.json")
+
+    first_test_file.write_text(json.dumps(test_data[0]), "utf-8")
+    second_test_file.write_text(json.dumps(test_data[1]), "utf-8")
+    byte_stream = ByteStream.from_string(json.dumps(test_data[2]))
+
+    sources = [str(first_test_file), second_test_file, byte_stream]
+    converter = JSONConverter(content_key="category")
+    result = converter.run(sources=sources)
+    assert len(result) == 1
+    assert len(result["documents"]) == 3
+    assert result["documents"][0].content == "literature"
+    assert result["documents"][0].meta == {"file_path": os.path.basename(first_test_file)}
+    assert result["documents"][1].content == "medicine"
+    assert result["documents"][1].meta == {"file_path": os.path.basename(second_test_file)}
+    assert result["documents"][2].content == "physics"
+    assert result["documents"][2].meta == {}
+
+
+def test_run_with_content_key_and_extra_meta_fields(tmpdir):
+    first_test_file = Path(tmpdir / "first_test_file.json")
+    second_test_file = Path(tmpdir / "second_test_file.json")
+
+    first_test_file.write_text(json.dumps(test_data[0]), "utf-8")
+    second_test_file.write_text(json.dumps(test_data[1]), "utf-8")
+    byte_stream = ByteStream.from_string(json.dumps(test_data[2]))
+
+    sources = [str(first_test_file), second_test_file, byte_stream]
+    converter = JSONConverter(content_key="category", extra_meta_fields={"year"})
+    result = converter.run(sources=sources)
+    assert len(result) == 1
+    assert len(result["documents"]) == 3
+    assert result["documents"][0].content == "literature"
+    assert result["documents"][0].meta == {"file_path": os.path.basename(first_test_file), "year": "1997"}
+    assert result["documents"][1].content == "medicine"
+    assert result["documents"][1].meta == {"file_path": os.path.basename(second_test_file), "year": "1986"}
+    assert result["documents"][2].content == "physics"
+    assert result["documents"][2].meta == {"year": "1938"}
+
+
+def test_run_with_jq_schema_content_key_and_extra_meta_fields_literal(tmpdir):
+    first_test_file = Path(tmpdir / "first_test_file.json")
+    second_test_file = Path(tmpdir / "second_test_file.json")
+
+    first_test_file.write_text(json.dumps(test_data[0]), "utf-8")
+    second_test_file.write_text(json.dumps(test_data[1]), "utf-8")
+    byte_stream = ByteStream.from_string(json.dumps(test_data[2]))
+
+    sources = [str(first_test_file), second_test_file, byte_stream]
+    converter = JSONConverter(jq_schema=".laureates[]", content_key="motivation", extra_meta_fields="*")
+    result = converter.run(sources=sources)
+    assert len(result) == 1
+    assert len(result["documents"]) == 4
+    assert (
+        result["documents"][0].content
+        == "who emulates the jesters of the Middle Ages in scourging authority and upholding the dignity of the "
+        "downtrodden"
+    )
+    assert result["documents"][0].meta == {
+        "file_path": os.path.basename(first_test_file),
+        "id": "674",
+        "firstname": "Dario",
+        "surname": "Fokin",
+        "share": "1",
+    }
+    assert result["documents"][1].content == "for their discoveries of growth factors"
+    assert result["documents"][1].meta == {
+        "file_path": os.path.basename(second_test_file),
+        "id": "434",
+        "firstname": "Stanley",
+        "surname": "Cohen",
+        "share": "2",
+    }
+    assert result["documents"][2].content == "for their discoveries of growth factors"
+    assert result["documents"][2].meta == {
+        "file_path": os.path.basename(second_test_file),
+        "id": "435",
+        "firstname": "Rita",
+        "surname": "Levi-Montalcini",
+        "share": "2",
+    }
+    assert (
+        result["documents"][3].content
+        == "for his demonstrations of the existence of new radioactive elements produced by neutron irradiation, "
+        "and for his related discovery of nuclear reactions brought about by slow neutrons"
+    )
+    assert result["documents"][3].meta == {"id": "46", "firstname": "Enrico", "surname": "Fermi", "share": "1"}

test/components/converters/test_markdown_to_document.py

@@ -0,0 +1,233 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+from unittest.mock import patch
+
+import pytest
+
+from haystack.components.converters.markdown import MarkdownToDocument
+from haystack.dataclasses import ByteStream
+
+
+class TestMarkdownToDocument:
+    def test_init_params_default(self):
+        converter = MarkdownToDocument()
+        assert converter.table_to_single_line is False
+        assert converter.progress_bar is True
+        assert converter.encoding == "utf-8"
+        assert converter.extract_frontmatter is False
+
+    def test_init_params_custom(self):
+        converter = MarkdownToDocument(table_to_single_line=True, progress_bar=False, store_full_path=False)
+        assert converter.table_to_single_line is True
+        assert converter.progress_bar is False
+        assert converter.store_full_path is False
+
+    @pytest.mark.integration
+    def test_run(self, test_files_path):
+        converter = MarkdownToDocument()
+        sources = [test_files_path / "markdown" / "sample.md"]
+        results = converter.run(sources=sources)
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        for doc in docs:
+            assert "What to build with Haystack" in doc.content
+            assert "# git clone https://github.com/deepset-ai/haystack.git" in doc.content
+
+    def test_run_with_store_full_path_false(self, test_files_path):
+        """
+        Test if the component runs correctly with store_full_path=False
+        """
+        converter = MarkdownToDocument(store_full_path=False)
+        sources = [test_files_path / "markdown" / "sample.md"]
+        results = converter.run(sources=sources)
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        for doc in docs:
+            assert "What to build with Haystack" in doc.content
+            assert "# git clone https://github.com/deepset-ai/haystack.git" in doc.content
+            assert doc.meta["file_path"] == "sample.md"
+
+    def test_run_calls_normalize_metadata(self, test_files_path):
+        bytestream = ByteStream(data=b"test", meta={"author": "test_author", "language": "en"})
+
+        converter = MarkdownToDocument()
+
+        with patch("haystack.components.converters.markdown.normalize_metadata") as normalize_metadata:
+            normalize_metadata.return_value = [{"language": "it"}, {"language": "it"}]
+
+            converter.run(sources=[bytestream, test_files_path / "markdown" / "sample.md"], meta={"language": "it"})
+
+        # check that the metadata normalizer is called properly
+        normalize_metadata.assert_called_with(meta={"language": "it"}, sources_count=2)
+
+    def test_run_with_meta(self, test_files_path):
+        bytestream = ByteStream(data=b"test", meta={"author": "test_author", "language": "en"})
+
+        converter = MarkdownToDocument()
+
+        with patch("haystack.components.converters.markdown.MarkdownIt.render") as mock_render:
+            mock_render.return_value = "test"
+            output = converter.run(
+                sources=[bytestream, test_files_path / "markdown" / "sample.md"], meta={"language": "it"}
+            )
+
+        # check that the metadata from the bytestream is merged with that from the meta parameter
+        assert output["documents"][0].meta["author"] == "test_author"
+        assert output["documents"][0].meta["language"] == "it"
+        assert output["documents"][1].meta["language"] == "it"
+
+    def test_run_extracts_yaml_frontmatter_into_metadata(self):
+        bytestream = ByteStream(
+            data=(
+                b"---\n"
+                b"ticker: AAPL\n"
+                b"date: 2026-06-12\n"
+                b"rating_score: 4\n"
+                b"source: earnings_call\n"
+                b"tags:\n"
+                b"  - guidance\n"
+                b"---\n"
+                b"# Thesis\n"
+                b"Revenue guidance improved.\n"
+            ),
+            meta={"file_path": "/tmp/aapl.md"},
+        )
+
+        converter = MarkdownToDocument(progress_bar=False, extract_frontmatter=True)
+        output = converter.run(sources=[bytestream])
+        document = output["documents"][0]
+
+        assert "Revenue guidance improved." in document.content
+        assert "ticker: AAPL" not in document.content
+        assert document.meta["ticker"] == "AAPL"
+        assert document.meta["date"] == "2026-06-12"
+        assert document.meta["rating_score"] == 4
+        assert document.meta["source"] == "earnings_call"
+        assert document.meta["tags"] == ["guidance"]
+        assert document.meta["file_path"] == "aapl.md"
+
+    def test_run_keeps_frontmatter_as_content_by_default(self):
+        bytestream = ByteStream(data=b"---\nticker: AAPL\n---\n# Thesis\n")
+
+        converter = MarkdownToDocument(progress_bar=False)
+        output = converter.run(sources=[bytestream])
+        document = output["documents"][0]
+
+        assert "ticker: AAPL" in document.content
+        assert "ticker" not in document.meta
+
+    def test_run_meta_overrides_frontmatter_metadata(self):
+        bytestream = ByteStream(
+            data=b"---\nticker: AAPL\nsource: filing\n---\n# Thesis\n", meta={"source": "bytestream"}
+        )
+
+        converter = MarkdownToDocument(progress_bar=False, extract_frontmatter=True)
+        output = converter.run(sources=[bytestream], meta={"ticker": "MSFT"})
+        document = output["documents"][0]
+
+        assert document.meta["ticker"] == "MSFT"
+        assert document.meta["source"] == "filing"
+
+    def test_run_keeps_malformed_frontmatter_as_content_and_logs_warning(self, caplog):
+        bytestream = ByteStream(data=b"---\nticker: [AAPL\n---\n# Thesis\n")
+
+        converter = MarkdownToDocument(progress_bar=False, extract_frontmatter=True)
+        with caplog.at_level(logging.WARNING):
+            output = converter.run(sources=[bytestream])
+
+        document = output["documents"][0]
+        assert "ticker: [AAPL" in document.content
+        assert "ticker" not in document.meta
+        assert "Could not parse YAML frontmatter" in caplog.text
+
+    def test_run_keeps_unserializable_frontmatter_as_content_and_logs_warning(self, caplog):
+        bytestream = ByteStream(data=b"---\ncycle: &cycle\n  - *cycle\n---\n# Thesis\n")
+
+        converter = MarkdownToDocument(progress_bar=False, extract_frontmatter=True)
+        with caplog.at_level(logging.WARNING):
+            output = converter.run(sources=[bytestream])
+
+        document = output["documents"][0]
+        assert "cycle:" in document.content
+        assert "cycle" not in document.meta
+        assert "Could not convert YAML frontmatter" in caplog.text
+
+    @pytest.mark.integration
+    def test_run_wrong_file_type(self, test_files_path, caplog):
+        """
+        Test if the component runs correctly when an input file is not of the expected type.
+        """
+        sources = [test_files_path / "audio" / "answer.wav"]
+        converter = MarkdownToDocument()
+        with caplog.at_level(logging.WARNING):
+            output = converter.run(sources=sources)
+            assert "codec can't decode byte" in caplog.text
+
+        docs = output["documents"]
+        assert not docs
+
+    @pytest.mark.integration
+    def test_run_error_handling(self, caplog):
+        """
+        Test if the component correctly handles errors.
+        """
+        sources = ["non_existing_file.md"]
+        converter = MarkdownToDocument()
+        with caplog.at_level(logging.WARNING):
+            result = converter.run(sources=sources)
+            assert "Could not read non_existing_file.md" in caplog.text
+            assert not result["documents"]
+
+    def test_mixed_sources_run(self, test_files_path):
+        """
+        Test if the component runs correctly if the input is a mix of strings, paths and ByteStreams.
+        """
+        sources = [
+            test_files_path / "markdown" / "sample.md",
+            str((test_files_path / "markdown" / "sample.md").absolute()),
+        ]
+        with open(test_files_path / "markdown" / "sample.md", "rb") as f:
+            byte_stream = f.read()
+            sources.append(ByteStream(byte_stream))
+
+        converter = MarkdownToDocument()
+        output = converter.run(sources=sources)
+        docs = output["documents"]
+        assert len(docs) == 3
+        for doc in docs:
+            assert "What to build with Haystack" in doc.content
+            assert "# git clone https://github.com/deepset-ai/haystack.git" in doc.content
+
+    def test_bytestream_encoding_from_meta(self):
+        """
+        Test that a non-UTF-8 ByteStream is decoded using the encoding specified in its meta.
+        """
+        # "caf\xe9" is "café" in latin-1; decoding as utf-8 would raise UnicodeDecodeError.
+        latin1_md = "# caf\xe9".encode("latin-1")
+        bytestream = ByteStream(data=latin1_md, meta={"encoding": "latin-1"})
+
+        converter = MarkdownToDocument(progress_bar=False)
+        output = converter.run(sources=[bytestream])
+        docs = output["documents"]
+
+        assert len(docs) == 1
+        assert "café" in docs[0].content
+
+    def test_bytestream_encoding_from_init(self):
+        """
+        Test that the encoding passed to __init__ is used as a fallback when not set in ByteStream meta.
+        """
+        latin1_md = "# caf\xe9".encode("latin-1")
+        bytestream = ByteStream(data=latin1_md)
+
+        converter = MarkdownToDocument(encoding="latin-1", progress_bar=False)
+        output = converter.run(sources=[bytestream])
+        docs = output["documents"]
+
+        assert len(docs) == 1
+        assert "café" in docs[0].content

test/components/converters/test_msg_to_document.py

@@ -0,0 +1,38 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from haystack.components.converters.msg import MSGToDocument
+
+
+class TestMSGToDocument:
+    def test_run(self, test_files_path):
+        converter = MSGToDocument(store_full_path=True)
+        paths = [test_files_path / "msg" / "sample.msg"]
+        result = converter.run(sources=paths, meta={"date_added": "2021-09-01T00:00:00"})
+        assert len(result["documents"]) == 1
+        assert result["documents"][0].content.startswith('From: "Sebastian Lee"')
+        assert result["documents"][0].meta == {
+            "date_added": "2021-09-01T00:00:00",
+            "file_path": str(test_files_path / "msg" / "sample.msg"),
+        }
+        assert len(result["attachments"]) == 1
+        assert result["attachments"][0].mime_type == "application/pdf"
+        assert result["attachments"][0].meta == {
+            "date_added": "2021-09-01T00:00:00",
+            "parent_file_path": str(test_files_path / "msg" / "sample.msg"),
+            "file_path": "sample_pdf_1.pdf",
+        }
+
+    def test_run_wrong_file_type(self, test_files_path, caplog):
+        converter = MSGToDocument(store_full_path=False)
+        paths = [test_files_path / "pdf" / "sample_pdf_1.pdf"]
+        result = converter.run(sources=paths, meta={"date_added": "2021-09-01T00:00:00"})
+        assert len(result["documents"]) == 0
+        assert "msg_file is not an Outlook MSG file" in caplog.text
+
+    def test_run_empty_sources(self, test_files_path):
+        converter = MSGToDocument(store_full_path=False)
+        result = converter.run(sources=[])
+        assert len(result["documents"]) == 0
+        assert len(result["attachments"]) == 0

test/components/converters/test_multi_file_converter.py

@@ -0,0 +1,145 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+from haystack import Document, Pipeline
+from haystack.components.converters.multi_file_converter import MultiFileConverter
+from haystack.core.component.component import Component
+from haystack.core.pipeline.base import component_from_dict, component_to_dict
+from haystack.dataclasses import ByteStream
+
+
+@pytest.fixture
+def converter():
+    converter = MultiFileConverter()
+    converter.warm_up()
+    return converter
+
+
+class TestMultiFileConverter:
+    def test_init_default_params(self, converter):
+        """Test initialization with default parameters"""
+        assert converter.encoding == "utf-8"
+        assert converter.json_content_key == "content"
+        assert isinstance(converter, Component)
+
+    def test_init_custom_params(self):
+        """Test initialization with custom parameters"""
+        converter = MultiFileConverter(encoding="latin-1", json_content_key="text")
+        assert converter.encoding == "latin-1"
+        assert converter.json_content_key == "text"
+
+    def test_to_dict(self, converter):
+        """Test serialization to dictionary"""
+        data = component_to_dict(converter, "converter")
+        assert data == {
+            "type": "haystack.components.converters.multi_file_converter.MultiFileConverter",
+            "init_parameters": {"encoding": "utf-8", "json_content_key": "content"},
+        }
+
+    def test_from_dict(self):
+        """Test deserialization from dictionary"""
+        data = {
+            "type": "haystack.components.converters.multi_file_converter.MultiFileConverter",
+            "init_parameters": {"encoding": "latin-1", "json_content_key": "text"},
+        }
+        conv = component_from_dict(MultiFileConverter, data, "converter")
+        assert conv.encoding == "latin-1"
+        assert conv.json_content_key == "text"
+
+    @pytest.mark.parametrize(
+        "suffix,file_path",
+        [
+            ("csv", "csv/sample_1.csv"),
+            ("docx", "docx/sample_docx.docx"),
+            ("html", "html/what_is_haystack.html"),
+            ("json", "json/json_conversion_testfile.json"),
+            ("md", "markdown/sample.md"),
+            ("pdf", "pdf/sample_pdf_1.pdf"),
+            ("pptx", "pptx/sample_pptx.pptx"),
+            ("txt", "txt/doc_1.txt"),
+            ("xlsx", "xlsx/table_empty_rows_and_columns.xlsx"),
+        ],
+    )
+    @pytest.mark.integration
+    def test_run(self, test_files_path, converter, suffix, file_path):
+        unclassified_bytestream = ByteStream(b"unclassified content")
+        unclassified_bytestream.meta["content_type"] = "unknown_type"
+
+        paths = [test_files_path / file_path, unclassified_bytestream]
+
+        output = converter.run(sources=paths)
+        docs = output["documents"]
+        unclassified = output["unclassified"]
+
+        assert len(docs) == 1
+        assert isinstance(docs[0], Document)
+        assert docs[0].content is not None
+        assert docs[0].meta["file_path"].endswith(suffix)
+
+        assert len(unclassified) == 1
+        assert isinstance(unclassified[0], ByteStream)
+        assert unclassified[0].meta["content_type"] == "unknown_type"
+
+    def test_run_with_meta(self, test_files_path, converter):
+        """Test conversion with metadata"""
+        paths = [test_files_path / "txt" / "doc_1.txt"]
+        meta = {"language": "en", "author": "test"}
+        output = converter.run(sources=paths, meta=meta)
+        docs = output["documents"]
+        assert docs[0].meta["language"] == "en"
+        assert docs[0].meta["author"] == "test"
+
+    def test_run_with_bytestream(self, converter):
+        """Test converting ByteStream input"""
+        bytestream = ByteStream(data=b"test content", mime_type="text/plain", meta={"file_path": "test.txt"})
+        output = converter.run(sources=[bytestream])
+        docs = output["documents"]
+        assert len(docs) == 1
+        assert docs[0].content == "test content"
+        assert docs[0].meta["file_path"] == "test.txt"
+
+    def test_run_error_handling(self, test_files_path, converter, caplog):
+        """Test error handling for non-existent files"""
+        paths = [test_files_path / "non_existent.txt"]
+        with caplog.at_level("WARNING"):
+            output = converter.run(sources=paths)
+            assert "File not found" in caplog.text
+            assert len(output["failed"]) == 1
+
+    @pytest.mark.integration
+    def test_run_all_file_types(self, test_files_path, converter):
+        """Test converting all supported file types in parallel"""
+        paths = [
+            test_files_path / "csv" / "sample_1.csv",
+            test_files_path / "docx" / "sample_docx.docx",
+            test_files_path / "html" / "what_is_haystack.html",
+            test_files_path / "json" / "json_conversion_testfile.json",
+            test_files_path / "markdown" / "sample.md",
+            test_files_path / "txt" / "doc_1.txt",
+            test_files_path / "pdf" / "sample_pdf_1.pdf",
+            test_files_path / "pptx" / "sample_pptx.pptx",
+            test_files_path / "xlsx" / "table_empty_rows_and_columns.xlsx",
+        ]
+        output = converter.run(sources=paths)
+        docs = output["documents"]
+
+        # Verify we got a document for each file
+        assert len(docs) == len(paths)
+        assert all(isinstance(doc, Document) for doc in docs)
+
+    @pytest.mark.integration
+    def test_run_in_pipeline(self, test_files_path, converter):
+        pipeline = Pipeline(max_runs_per_component=1)
+        pipeline.add_component("converter", converter)
+
+        paths = [test_files_path / "txt" / "doc_1.txt", test_files_path / "pdf" / "sample_pdf_1.pdf"]
+
+        output = pipeline.run(data={"sources": paths})
+        docs = output["converter"]["documents"]
+
+        assert len(docs) == 2
+        assert all(isinstance(doc, Document) for doc in docs)
+        assert all(doc.content is not None for doc in docs)

test/components/converters/test_output_adapter.py

@@ -0,0 +1,219 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+from typing import Any, List
+
+import pytest
+
+from haystack import Pipeline, component
+from haystack.components.converters import OutputAdapter
+from haystack.components.converters.output_adapter import OutputAdaptationException
+from haystack.core.component.sockets import InputSocket
+from haystack.dataclasses import Document
+
+
+def custom_filter_to_sede(value):
+    return value.upper()
+
+
+def another_custom_filter(value):
+    return value.upper()
+
+
+class TestOutputAdapter:
+    #  OutputAdapter can be initialized with a valid Jinja2 template string and output type.
+    def test_initialized_with_valid_template_and_output_type(self):
+        template = "{{ documents[0].content }}"
+        output_type = str
+        adapter = OutputAdapter(template="{{ documents[0].content }}", output_type=str)
+
+        assert adapter.template == template
+        assert adapter.__haystack_output__.output.name == "output"
+        assert adapter.__haystack_output__.output.type == output_type
+
+    #  OutputAdapter can adapt the output of one component to be compatible with the input of another
+    #  component using Jinja2 template expressions.
+    def test_output_adaptation(self):
+        adapter = OutputAdapter(template="{{ documents[0].content }}", output_type=str)
+
+        input_data = {"documents": [{"content": "Test content"}]}
+        expected_output = {"output": "Test content"}
+
+        assert adapter.run(**input_data) == expected_output
+
+    #  OutputAdapter can add filter 'json_loads' and use it
+    def test_predefined_filters(self):
+        adapter = OutputAdapter(
+            template="{{ documents[0].content|json_loads }}",
+            output_type=dict,
+            custom_filters={"json_loads": lambda s: json.loads(str(s))},
+        )
+
+        input_data = {"documents": [{"content": '{"key": "value"}'}]}
+        expected_output = {"output": {"key": "value"}}
+
+        assert adapter.run(**input_data) == expected_output
+
+    #  OutputAdapter can handle custom filters provided in the component configuration.
+    def test_custom_filters(self):
+        def custom_filter(value):
+            return value.upper()
+
+        custom_filters = {"custom_filter": custom_filter}
+        adapter = OutputAdapter(
+            template="{{ documents[0].content|custom_filter }}", output_type=str, custom_filters=custom_filters
+        )
+
+        input_data = {"documents": [{"content": "test content"}]}
+        expected_output = {"output": "TEST CONTENT"}
+
+        assert adapter.run(**input_data) == expected_output
+
+    #  OutputAdapter raises an exception on init if the Jinja2 template string is invalid.
+    def test_invalid_template_string(self):
+        with pytest.raises(ValueError):
+            OutputAdapter(template="{{ documents[0].content }", output_type=str)
+
+    #  OutputAdapter raises an exception if no input data is provided for output adaptation.
+    def test_no_input_data_provided(self):
+        adapter = OutputAdapter(template="{{ documents[0].content }}", output_type=str)
+        with pytest.raises(ValueError):
+            adapter.run()
+
+    #  OutputAdapter raises an exception if there's an error during the adaptation process.
+    def test_error_during_adaptation(self):
+        adapter = OutputAdapter(template="{{ documents[0].content }}", output_type=str)
+        input_data = {"documents": [{"title": "Test title"}]}
+
+        with pytest.raises(OutputAdaptationException):
+            adapter.run(**input_data)
+
+    # OutputAdapter can be serialized to a dictionary and deserialized back to an OutputAdapter instance.
+    def test_sede(self):
+        adapter = OutputAdapter(template="{{ documents[0].content }}", output_type=str)
+        adapter_dict = adapter.to_dict()
+        deserialized_adapter = OutputAdapter.from_dict(adapter_dict)
+
+        assert adapter.template == deserialized_adapter.template
+        assert adapter.output_type == deserialized_adapter.output_type
+
+    # OutputAdapter can be serialized to a dictionary and deserialized along with custom filters
+    def test_sede_with_custom_filters(self):
+        # NOTE: filters need to be declared in a namespace visible to the deserialization function
+        custom_filters = {"custom_filter": custom_filter_to_sede}
+        adapter = OutputAdapter(
+            template="{{ documents[0].content|custom_filter }}", output_type=str, custom_filters=custom_filters
+        )
+        adapter_dict = adapter.to_dict()
+        deserialized_adapter = OutputAdapter.from_dict(adapter_dict)
+
+        assert adapter.template == deserialized_adapter.template
+        assert adapter.output_type == deserialized_adapter.output_type
+        assert adapter.custom_filters == deserialized_adapter.custom_filters == custom_filters
+
+        # invoke the custom filter to check if it is deserialized correctly
+        assert deserialized_adapter.custom_filters["custom_filter"]("test") == "TEST"
+
+    # OutputAdapter can be serialized to a dictionary and deserialized along with multiple custom filters
+    def test_sede_with_multiple_custom_filters(self):
+        # NOTE: filters need to be declared in a namespace visible to the deserialization function
+        custom_filters = {"custom_filter": custom_filter_to_sede, "another_custom_filter": another_custom_filter}
+        adapter = OutputAdapter(
+            template="{{ documents[0].content|custom_filter }}", output_type=str, custom_filters=custom_filters
+        )
+        adapter_dict = adapter.to_dict()
+        deserialized_adapter = OutputAdapter.from_dict(adapter_dict)
+
+        assert adapter.template == deserialized_adapter.template
+        assert adapter.output_type == deserialized_adapter.output_type
+        assert adapter.custom_filters == deserialized_adapter.custom_filters == custom_filters
+
+        # invoke the custom filter to check if it is deserialized correctly
+        assert deserialized_adapter.custom_filters["custom_filter"]("test") == "TEST"
+
+    def test_sede_with_list_output_type_in_pipeline(self):
+        pipe = Pipeline()
+        pipe.add_component("adapter", OutputAdapter(template="{{ test }}", output_type=list[str]))
+        serialized_pipe = pipe.dumps()
+
+        # we serialize the pipeline and check if the output type is serialized correctly (as list[str])
+        assert "list[str]" in serialized_pipe
+
+        deserialized_pipe = Pipeline.loads(serialized_pipe)
+        assert deserialized_pipe.get_component("adapter").output_type == list[str]
+
+    def test_sede_with_typing_list_output_type_in_pipeline(self):
+        pipe = Pipeline()
+        pipe.add_component("adapter", OutputAdapter(template="{{ test }}", output_type=List[str]))
+        serialized_pipe = pipe.dumps()
+
+        # we serialize the pipeline and check if the output type is serialized correctly (as list[str])
+        assert "typing.List[str]" in serialized_pipe
+
+        deserialized_pipe = Pipeline.loads(serialized_pipe)
+        assert deserialized_pipe.get_component("adapter").output_type == List[str]
+
+    def test_output_adapter_from_dict_custom_filters_none(self):
+        component = OutputAdapter.from_dict(
+            data={
+                "type": "haystack.components.converters.output_adapter.OutputAdapter",
+                "init_parameters": {
+                    "template": "{{ documents[0].content}}",
+                    "output_type": "str",
+                    "custom_filters": None,
+                    "unsafe": False,
+                },
+            }
+        )
+
+        assert component.template == "{{ documents[0].content}}"
+        assert component.output_type == str
+        assert component.custom_filters == {}
+        assert not component._unsafe
+
+    def test_output_adapter_in_pipeline(self):
+        @component
+        class DocumentProducer:
+            @component.output_types(documents=dict)
+            def run(self):
+                return {"documents": [{"content": '{"framework": "Haystack"}'}]}
+
+        pipe = Pipeline()
+        pipe.add_component(
+            name="output_adapter",
+            instance=OutputAdapter(
+                template="{{ documents[0].content | json_loads}}",
+                output_type=str,
+                custom_filters={"json_loads": lambda s: json.loads(str(s))},
+            ),
+        )
+        pipe.add_component(name="document_producer", instance=DocumentProducer())
+        pipe.connect("document_producer", "output_adapter")
+        result = pipe.run(data={})
+        assert result
+        assert result["output_adapter"]["output"] == {"framework": "Haystack"}
+
+    def test_unsafe(self):
+        adapter = OutputAdapter(template="{{ documents[0] }}", output_type=Document, unsafe=True)
+        documents = [
+            Document(content="Test document"),
+            Document(content="Another test document"),
+            Document(content="Yet another test document"),
+        ]
+        res = adapter.run(documents=documents)
+        assert res["output"] == documents[0]
+
+    def test_variables_correct_with_assignment(self) -> None:
+        template = """{% if control == 'something' %}
+    {% set output = 1 %}
+{% else %}
+    {% set output = 3 %}
+{% endif %}
+{{ output }}
+"""
+        adapter = OutputAdapter(template=template, output_type=int)
+        assert adapter.__haystack_input__._sockets_dict == {"control": InputSocket(name="control", type=Any)}
+        res = adapter.run(control="something")
+        assert res["output"] == 1

test/components/converters/test_pdfminer_to_document.py

@@ -0,0 +1,242 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+from unittest.mock import patch
+
+import pytest
+
+from haystack import Document
+from haystack.components.converters.pdfminer import PDFMinerToDocument
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.dataclasses import ByteStream
+
+
+class TestPDFMinerToDocument:
+    def test_run(self, test_files_path):
+        """
+        Test if the component runs correctly.
+        """
+        converter = PDFMinerToDocument()
+        sources = [test_files_path / "pdf" / "sample_pdf_1.pdf"]
+        results = converter.run(sources=sources)
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        for doc in docs:
+            assert "the page 3 is empty" in doc.content
+            assert "Page 4 of Sample PDF" in doc.content
+
+    def test_init_params_custom(self, test_files_path):
+        """
+        Test if init arguments are passed successfully to PDFMinerToDocument layout parameters
+        """
+        converter = PDFMinerToDocument(char_margin=0.5, all_texts=True, store_full_path=False)
+        assert converter.layout_params.char_margin == 0.5
+        assert converter.layout_params.all_texts is True
+        assert converter.store_full_path is False
+
+    def test_run_with_store_full_path_false(self, test_files_path):
+        """
+        Test if the component runs correctly with store_full_path=False
+        """
+        converter = PDFMinerToDocument(store_full_path=False)
+        sources = [test_files_path / "pdf" / "sample_pdf_1.pdf"]
+        results = converter.run(sources=sources)
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        for doc in docs:
+            assert "the page 3 is empty" in doc.content
+            assert "Page 4 of Sample PDF" in doc.content
+            assert doc.meta["file_path"] == "sample_pdf_1.pdf"
+
+    def test_run_wrong_file_type(self, test_files_path, caplog):
+        """
+        Test if the component runs correctly when an input file is not of the expected type.
+        """
+        sources = [test_files_path / "audio" / "answer.wav"]
+        converter = PDFMinerToDocument()
+
+        with caplog.at_level(logging.WARNING):
+            output = converter.run(sources=sources)
+            assert "Is this really a PDF?" in caplog.text
+
+        docs = output["documents"]
+        assert not docs
+
+    def test_arg_is_none(self, test_files_path):
+        """
+        Test if the component runs correctly when an argument is None.
+        """
+        converter = PDFMinerToDocument(char_margin=None)
+        assert converter.layout_params.char_margin is None
+
+    def test_run_doc_metadata(self, test_files_path):
+        """
+        Test if the component runs correctly when metadata is supplied by the user.
+        """
+        converter = PDFMinerToDocument()
+        sources = [test_files_path / "pdf" / "sample_pdf_2.pdf"]
+        metadata = [{"file_name": "sample_pdf_2.pdf"}]
+        results = converter.run(sources=sources, meta=metadata)
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        assert "Ward Cunningham" in docs[0].content
+        assert docs[0].meta["file_name"] == "sample_pdf_2.pdf"
+
+    def test_incorrect_meta(self, test_files_path):
+        """
+        Test if the component raises an error when incorrect metadata is supplied by the user.
+        """
+        converter = PDFMinerToDocument()
+        sources = [test_files_path / "pdf" / "sample_pdf_3.pdf"]
+        metadata = [{"file_name": "sample_pdf_3.pdf"}, {"file_name": "sample_pdf_2.pdf"}]
+        with pytest.raises(ValueError, match="The length of the metadata list must match the number of sources."):
+            converter.run(sources=sources, meta=metadata)
+
+    def test_run_bytestream_metadata(self, test_files_path):
+        """
+        Test if the component runs correctly when metadata is read from the ByteStream object.
+        """
+        converter = PDFMinerToDocument()
+        with open(test_files_path / "pdf" / "sample_pdf_2.pdf", "rb") as file:
+            byte_stream = file.read()
+            stream = ByteStream(byte_stream, meta={"content_type": "text/pdf", "url": "test_url"})
+
+        results = converter.run(sources=[stream])
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        assert "Ward Cunningham" in docs[0].content
+        assert docs[0].meta == {"content_type": "text/pdf", "url": "test_url"}
+
+    def test_run_bytestream_doc_overlapping_metadata(self, test_files_path):
+        """
+        Test if the component runs correctly when metadata is read from the ByteStream object and supplied by the user.
+
+        There is an overlap between the metadata received.
+
+        The component should use the supplied metadata to overwrite the values if there is an overlap between the keys.
+        """
+        converter = PDFMinerToDocument()
+        with open(test_files_path / "pdf" / "sample_pdf_2.pdf", "rb") as file:
+            byte_stream = file.read()
+            # ByteStream has "url" present in metadata
+            stream = ByteStream(byte_stream, meta={"content_type": "text/pdf", "url": "test_url_correct"})
+
+        # "url" supplied by the user overwrites value present in metadata
+        metadata = [{"file_name": "sample_pdf_2.pdf", "url": "test_url_new"}]
+        results = converter.run(sources=[stream], meta=metadata)
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        assert "Ward Cunningham" in docs[0].content
+        assert docs[0].meta == {"file_name": "sample_pdf_2.pdf", "content_type": "text/pdf", "url": "test_url_new"}
+
+    def test_run_error_handling(self, caplog):
+        """
+        Test if the component correctly handles errors.
+        """
+        sources = ["non_existing_file.pdf"]
+        converter = PDFMinerToDocument()
+        with caplog.at_level(logging.WARNING):
+            results = converter.run(sources=sources)
+            assert "Could not read non_existing_file.pdf" in caplog.text
+            assert results["documents"] == []
+
+    def test_run_empty_document(self, caplog, test_files_path):
+        sources = [test_files_path / "pdf" / "non_text_searchable.pdf"]
+        converter = PDFMinerToDocument()
+        with caplog.at_level(logging.WARNING):
+            results = converter.run(sources=sources)
+            assert "PDFMinerToDocument could not extract text from the file" in caplog.text
+            assert results["documents"][0].content == ""
+
+            # Check that not only content is used when the returned document is initialized and doc id is generated
+            assert results["documents"][0].meta["file_path"] == "non_text_searchable.pdf"
+            assert results["documents"][0].id != Document(content="").id
+
+    def test_run_detect_pages_and_split_by_passage(self, test_files_path):
+        converter = PDFMinerToDocument()
+        sources = [test_files_path / "pdf" / "sample_pdf_2.pdf"]
+        pdf_doc = converter.run(sources=sources)
+        splitter = DocumentSplitter(split_length=1, split_by="page")
+        docs = splitter.run(pdf_doc["documents"])
+        assert len(docs["documents"]) == 4
+
+    def test_run_detect_paragraphs_to_be_used_in_split_passage(self, test_files_path):
+        converter = PDFMinerToDocument()
+        sources = [test_files_path / "pdf" / "sample_pdf_2.pdf"]
+        pdf_doc = converter.run(sources=sources)
+        splitter = DocumentSplitter(split_length=1, split_by="passage")
+        docs = splitter.run(pdf_doc["documents"])
+
+        assert len(docs["documents"]) == 29
+
+        expected = (
+            "\nA wiki (/ˈwɪki/ (About this soundlisten) WIK-ee) is a hypertext publication collaboratively"
+            " \nedited and managed by its own audience directly using a web browser. A typical wiki \ncontains "
+            "multiple pages for the subjects or scope of the project and may be either open \nto the public or "
+            "limited to use within an organization for maintaining its internal knowledge \nbase. Wikis are "
+            "enabled by wiki software, otherwise known as wiki engines. A wiki engine, \nbeing a form of a "
+            "content management system, differs from other web-based systems \nsuch as blog software, in that "
+            "the content is created without any defined owner or leader, \nand wikis have little inherent "
+            "structure, allowing structure to emerge according to the \nneeds of the users.[1] \n\n"
+        )
+        assert docs["documents"][6].content == expected
+
+    def test_detect_undecoded_cid_characters(self):
+        """
+        Test if the component correctly detects and reports undecoded CID characters in text.
+        """
+        converter = PDFMinerToDocument()
+
+        # Test text with no CID characters
+        text = "This is a normal text without any CID characters."
+        result = converter.detect_undecoded_cid_characters(text)
+        assert result["total_chars"] == len(text)
+        assert result["cid_chars"] == 0
+        assert result["percentage"] == 0
+
+        # Test text with CID characters
+        text = "Some text with (cid:123) and (cid:456) characters"
+        result = converter.detect_undecoded_cid_characters(text)
+        assert result["total_chars"] == len(text)
+        assert result["cid_chars"] == len("(cid:123)") + len("(cid:456)")  # 18 characters total
+        assert result["percentage"] == round((18 / len(text)) * 100, 2)
+
+        # Test text with multiple consecutive CID characters
+        text = "(cid:123)(cid:456)(cid:789)"
+        result = converter.detect_undecoded_cid_characters(text)
+        assert result["total_chars"] == len(text)
+        assert result["cid_chars"] == len("(cid:123)(cid:456)(cid:789)")
+        assert result["percentage"] == 100.0
+
+        # Test empty text
+        text = ""
+        result = converter.detect_undecoded_cid_characters(text)
+        assert result["total_chars"] == 0
+        assert result["cid_chars"] == 0
+        assert result["percentage"] == 0
+
+    def test_pdfminer_logs_warning_for_cid_characters(self, caplog, monkeypatch):
+        """
+        Test if the component correctly logs a warning when undecoded CID characters are detected.
+        """
+        test_data = ByteStream(data=b"fake", meta={"file_path": "test.pdf"})
+
+        def mock_converter(*args, **kwargs):
+            return "This is text with (cid:123) and (cid:456) characters"
+
+        def mock_extract_pages(*args, **kwargs):
+            return ["mocked page"]
+
+        with patch("haystack.components.converters.pdfminer.extract_pages", side_effect=mock_extract_pages):
+            with patch.object(PDFMinerToDocument, "_converter", side_effect=mock_converter):
+                with caplog.at_level(logging.WARNING):
+                    converter = PDFMinerToDocument()
+                    converter.run(sources=[test_data])
+                    assert "Detected 18 undecoded CID characters in 52 characters (34.62%)" in caplog.text

test/components/converters/test_pptx_to_document.py

@@ -0,0 +1,123 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import os
+
+import pytest
+
+from haystack.components.converters.pptx import PPTXToDocument
+from haystack.dataclasses import ByteStream
+
+
+class TestPPTXToDocument:
+    def test_run(self, test_files_path):
+        """
+        Test if the component runs correctly.
+        """
+        bytestream = ByteStream.from_file_path(test_files_path / "pptx" / "sample_pptx.pptx")
+        bytestream.meta["file_path"] = str(test_files_path / "pptx" / "sample_pptx.pptx")
+        bytestream.meta["key"] = "value"
+        files = [str(test_files_path / "pptx" / "sample_pptx.pptx"), bytestream]
+        converter = PPTXToDocument()
+        output = converter.run(sources=files)
+        docs = output["documents"]
+
+        assert len(docs) == 2
+        assert (
+            "Sample Title Slide\nJane Doe\fTitle of First Slide\nThis is a bullet point\nThis is another bullet point"
+            in docs[0].content
+        )
+        assert (
+            "Sample Title Slide\nJane Doe\fTitle of First Slide\nThis is a bullet point\nThis is another bullet point"
+            in docs[0].content
+        )
+        assert docs[0].meta["file_path"] == os.path.basename(files[0])
+        assert docs[1].meta == {"file_path": os.path.basename(bytestream.meta["file_path"]), "key": "value"}
+
+    def test_run_error_non_existent_file(self, caplog):
+        sources = ["non_existing_file.pptx"]
+        converter = PPTXToDocument()
+        with caplog.at_level(logging.WARNING):
+            results = converter.run(sources=sources)
+            assert "Could not read non_existing_file.pptx" in caplog.text
+            assert results["documents"] == []
+
+    def test_run_error_wrong_file_type(self, caplog, test_files_path):
+        sources = [str(test_files_path / "txt" / "doc_1.txt")]
+        converter = PPTXToDocument()
+        with caplog.at_level(logging.WARNING):
+            results = converter.run(sources=sources)
+            assert "doc_1.txt and convert it" in caplog.text
+            assert results["documents"] == []
+
+    def test_run_with_meta(self, test_files_path):
+        bytestream = ByteStream.from_file_path(test_files_path / "pptx" / "sample_pptx.pptx")
+        bytestream.meta["file_path"] = str(test_files_path / "pptx" / "sample_pptx.pptx")
+        bytestream.meta["key"] = "value"
+
+        converter = PPTXToDocument()
+        output = converter.run(sources=[bytestream], meta=[{"language": "it"}])
+        document = output["documents"][0]
+
+        assert document.meta == {
+            "file_path": os.path.basename(test_files_path / "pptx" / "sample_pptx.pptx"),
+            "key": "value",
+            "language": "it",
+        }
+
+    def test_run_with_store_full_path_false(self, test_files_path):
+        """
+        Test if the component runs correctly with store_full_path=False
+        """
+        bytestream = ByteStream.from_file_path(test_files_path / "pptx" / "sample_pptx.pptx")
+        bytestream.meta["file_path"] = str(test_files_path / "pptx" / "sample_pptx.pptx")
+        bytestream.meta["key"] = "value"
+
+        converter = PPTXToDocument(store_full_path=False)
+        output = converter.run(sources=[bytestream], meta=[{"language": "it"}])
+        document = output["documents"][0]
+
+        assert document.meta == {"file_path": "sample_pptx.pptx", "key": "value", "language": "it"}
+
+    def test_to_dict(self):
+        converter = PPTXToDocument(link_format="markdown", store_full_path=True)
+        data = converter.to_dict()
+        assert data == {
+            "type": "haystack.components.converters.pptx.PPTXToDocument",
+            "init_parameters": {"link_format": "markdown", "store_full_path": True},
+        }
+
+    def test_to_dict_defaults(self):
+        converter = PPTXToDocument()
+        data = converter.to_dict()
+        assert data == {
+            "type": "haystack.components.converters.pptx.PPTXToDocument",
+            "init_parameters": {"link_format": "none", "store_full_path": False},
+        }
+
+    def test_link_format_invalid(self):
+        with pytest.raises(ValueError, match="Unknown link format"):
+            PPTXToDocument(link_format="invalid")
+
+    @pytest.mark.parametrize("link_format", ["markdown", "plain"])
+    def test_link_extraction(self, test_files_path, link_format):
+        converter = PPTXToDocument(link_format=link_format)
+        paths = [test_files_path / "pptx" / "sample_pptx_with_link.pptx"]
+        output = converter.run(sources=paths)
+        content = output["documents"][0].content
+
+        if link_format == "markdown":
+            assert "[Example](https://example.com)" in content
+        else:
+            assert "Example (https://example.com)" in content
+
+    def test_no_link_extraction(self, test_files_path):
+        converter = PPTXToDocument()
+        paths = [test_files_path / "pptx" / "sample_pptx_with_link.pptx"]
+        output = converter.run(sources=paths)
+        content = output["documents"][0].content
+
+        assert "https://example.com" not in content
+        assert "Example" in content

test/components/converters/test_pypdf_to_document.py

@@ -0,0 +1,238 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+from unittest.mock import Mock, patch
+
+import pytest
+
+from haystack import Document
+from haystack.components.converters.pypdf import PyPDFExtractionMode, PyPDFToDocument
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.dataclasses import ByteStream
+
+
+@pytest.fixture
+def pypdf_component():
+    return PyPDFToDocument()
+
+
+class TestPyPDFToDocument:
+    def test_init(self, pypdf_component):
+        assert pypdf_component.extraction_mode == PyPDFExtractionMode.PLAIN
+        assert pypdf_component.plain_mode_orientations == (0, 90, 180, 270)
+        assert pypdf_component.plain_mode_space_width == 200.0
+        assert pypdf_component.layout_mode_space_vertically is True
+        assert pypdf_component.layout_mode_scale_weight == 1.25
+        assert pypdf_component.layout_mode_strip_rotated is True
+        assert pypdf_component.layout_mode_font_height_weight == 1.0
+
+    def test_init_custom_params(self):
+        pypdf_component = PyPDFToDocument(
+            extraction_mode="layout",
+            plain_mode_orientations=(0, 90),
+            plain_mode_space_width=150.0,
+            layout_mode_space_vertically=False,
+            layout_mode_scale_weight=2.0,
+            layout_mode_strip_rotated=False,
+            layout_mode_font_height_weight=0.5,
+        )
+
+        assert pypdf_component.extraction_mode == PyPDFExtractionMode.LAYOUT
+        assert pypdf_component.plain_mode_orientations == (0, 90)
+        assert pypdf_component.plain_mode_space_width == 150.0
+        assert pypdf_component.layout_mode_space_vertically is False
+        assert pypdf_component.layout_mode_scale_weight == 2.0
+        assert pypdf_component.layout_mode_strip_rotated is False
+        assert pypdf_component.layout_mode_font_height_weight == 0.5
+
+    def test_init_invalid_extraction_mode(self):
+        with pytest.raises(ValueError):
+            PyPDFToDocument(extraction_mode="invalid")
+
+    def test_to_dict(self, pypdf_component):
+        data = pypdf_component.to_dict()
+        assert data == {
+            "type": "haystack.components.converters.pypdf.PyPDFToDocument",
+            "init_parameters": {
+                "extraction_mode": "plain",
+                "plain_mode_orientations": (0, 90, 180, 270),
+                "plain_mode_space_width": 200.0,
+                "layout_mode_space_vertically": True,
+                "layout_mode_scale_weight": 1.25,
+                "layout_mode_strip_rotated": True,
+                "layout_mode_font_height_weight": 1.0,
+                "store_full_path": False,
+            },
+        }
+
+    def test_from_dict(self):
+        data = {
+            "type": "haystack.components.converters.pypdf.PyPDFToDocument",
+            "init_parameters": {
+                "extraction_mode": "plain",
+                "plain_mode_orientations": (0, 90, 180, 270),
+                "plain_mode_space_width": 200.0,
+                "layout_mode_space_vertically": True,
+                "layout_mode_scale_weight": 1.25,
+                "layout_mode_strip_rotated": True,
+                "layout_mode_font_height_weight": 1.0,
+            },
+        }
+
+        instance = PyPDFToDocument.from_dict(data)
+        assert isinstance(instance, PyPDFToDocument)
+        assert instance.extraction_mode == PyPDFExtractionMode.PLAIN
+        assert instance.plain_mode_orientations == (0, 90, 180, 270)
+        assert instance.plain_mode_space_width == 200.0
+        assert instance.layout_mode_space_vertically is True
+        assert instance.layout_mode_scale_weight == 1.25
+        assert instance.layout_mode_strip_rotated is True
+        assert instance.layout_mode_font_height_weight == 1.0
+
+    def test_from_dict_defaults(self):
+        data = {"type": "haystack.components.converters.pypdf.PyPDFToDocument", "init_parameters": {}}
+        instance = PyPDFToDocument.from_dict(data)
+        assert isinstance(instance, PyPDFToDocument)
+        assert instance.extraction_mode == PyPDFExtractionMode.PLAIN
+
+    def test_default_convert(self):
+        mock_page1 = Mock()
+        mock_page2 = Mock()
+        mock_page1.extract_text.return_value = "Page 1 content"
+        mock_page2.extract_text.return_value = "Page 2 content"
+        mock_reader = Mock()
+        mock_reader.pages = [mock_page1, mock_page2]
+
+        converter = PyPDFToDocument(
+            extraction_mode="layout",
+            plain_mode_orientations=(0, 90),
+            plain_mode_space_width=150.0,
+            layout_mode_space_vertically=False,
+            layout_mode_scale_weight=2.0,
+            layout_mode_strip_rotated=False,
+            layout_mode_font_height_weight=1.5,
+        )
+
+        text = converter._default_convert(mock_reader)
+        assert text == "Page 1 content\fPage 2 content"
+
+        expected_params = {
+            "extraction_mode": "layout",
+            "orientations": (0, 90),
+            "space_width": 150.0,
+            "layout_mode_space_vertically": False,
+            "layout_mode_scale_weight": 2.0,
+            "layout_mode_strip_rotated": False,
+            "layout_mode_font_height_weight": 1.5,
+        }
+        for mock_page in mock_reader.pages:
+            mock_page.extract_text.assert_called_once_with(**expected_params)
+
+    @pytest.mark.integration
+    def test_run(self, test_files_path, pypdf_component):
+        """
+        Test if the component runs correctly.
+        """
+        paths = [test_files_path / "pdf" / "sample_pdf_1.pdf"]
+        output = pypdf_component.run(sources=paths)
+        docs = output["documents"]
+        assert len(docs) == 1
+        assert "History" in docs[0].content
+
+    @pytest.mark.integration
+    def test_page_breaks_added(self, test_files_path, pypdf_component):
+        paths = [test_files_path / "pdf" / "sample_pdf_1.pdf"]
+        output = pypdf_component.run(sources=paths)
+        docs = output["documents"]
+        assert len(docs) == 1
+        assert docs[0].content.count("\f") == 3
+
+    def test_run_with_meta(self, test_files_path, pypdf_component):
+        bytestream = ByteStream(data=b"test", meta={"author": "test_author", "language": "en"})
+
+        with patch("haystack.components.converters.pypdf.PdfReader"):
+            output = pypdf_component.run(
+                sources=[bytestream, test_files_path / "pdf" / "sample_pdf_1.pdf"], meta={"language": "it"}
+            )
+
+        # check that the metadata from the bytestream is merged with that from the meta parameter
+        assert output["documents"][0].meta["author"] == "test_author"
+        assert output["documents"][0].meta["language"] == "it"
+        assert output["documents"][1].meta["language"] == "it"
+
+    def test_run_with_store_full_path_false(self, test_files_path):
+        """
+        Test if the component runs correctly with store_full_path=False
+        """
+        sources = [test_files_path / "pdf" / "sample_pdf_1.pdf"]
+        converter = PyPDFToDocument(store_full_path=True)
+        results = converter.run(sources=sources)
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        assert docs[0].meta["file_path"] == str(sources[0])
+
+        converter = PyPDFToDocument(store_full_path=False)
+        results = converter.run(sources=sources)
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        assert docs[0].meta["file_path"] == "sample_pdf_1.pdf"
+
+    def test_run_error_handling(self, test_files_path, pypdf_component, caplog):
+        """
+        Test if the component correctly handles errors.
+        """
+        paths = ["non_existing_file.pdf"]
+        with caplog.at_level(logging.WARNING):
+            pypdf_component.run(sources=paths)
+            assert "Could not read non_existing_file.pdf" in caplog.text
+
+    @pytest.mark.integration
+    def test_mixed_sources_run(self, test_files_path, pypdf_component):
+        """
+        Test if the component runs correctly when mixed sources are provided.
+        """
+        paths = [test_files_path / "pdf" / "sample_pdf_1.pdf"]
+        with open(test_files_path / "pdf" / "sample_pdf_1.pdf", "rb") as f:
+            paths.append(ByteStream(f.read()))
+
+        output = pypdf_component.run(sources=paths)
+        docs = output["documents"]
+        assert len(docs) == 2
+        assert "History and standardization" in docs[0].content
+        assert "History and standardization" in docs[1].content
+
+    def test_run_empty_document(self, caplog, test_files_path):
+        paths = [test_files_path / "pdf" / "non_text_searchable.pdf"]
+        with caplog.at_level(logging.WARNING):
+            output = PyPDFToDocument().run(sources=paths)
+            assert "PyPDFToDocument could not extract text from the file" in caplog.text
+            assert output["documents"][0].content == ""
+
+            # Check that meta is used when the returned document is initialized and thus when doc id is generated
+            assert output["documents"][0].meta["file_path"] == "non_text_searchable.pdf"
+            assert output["documents"][0].id != Document(content="").id
+
+    def test_run_detect_paragraphs_to_be_used_in_split_passage(self, test_files_path):
+        converter = PyPDFToDocument(extraction_mode=PyPDFExtractionMode.LAYOUT)
+        sources = [test_files_path / "pdf" / "sample_pdf_2.pdf"]
+        pdf_doc = converter.run(sources=sources)
+        splitter = DocumentSplitter(split_length=1, split_by="passage")
+        docs = splitter.run(pdf_doc["documents"])
+
+        assert len(docs["documents"]) == 51
+
+        expected = (
+            "A wiki (/ˈwɪki/ (About this soundlisten) WIK-ee) is a hypertext publication collaboratively\n"
+            "edited and managed by its own audience directly using a web browser. A typical wiki\ncontains "
+            "multiple pages for the subjects or scope of the project and may be either open\nto the public or "
+            "limited to use within an organization for maintaining its internal knowledge\nbase. Wikis are "
+            "enabled by wiki software, otherwise known as wiki engines. A wiki engine,\nbeing a form of a "
+            "content management system, differs from other web-based systems\nsuch as blog software, in that "
+            "the content is created without any defined owner or leader,\nand wikis have little inherent "
+            "structure, allowing structure to emerge according to the\nneeds of the users.[1]\n\n"
+        )
+        assert docs["documents"][2].content == expected

test/components/converters/test_textfile_to_document.py

@@ -0,0 +1,87 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import os
+
+from haystack.components.converters.txt import TextFileToDocument
+from haystack.dataclasses import ByteStream
+
+
+class TestTextfileToDocument:
+    def test_run(self, test_files_path):
+        """
+        Test if the component runs correctly.
+        """
+        bytestream = ByteStream.from_file_path(test_files_path / "txt" / "doc_3.txt")
+        bytestream.meta["file_path"] = str(test_files_path / "txt" / "doc_3.txt")
+        bytestream.meta["key"] = "value"
+        files = [str(test_files_path / "txt" / "doc_1.txt"), test_files_path / "txt" / "doc_2.txt", bytestream]
+        converter = TextFileToDocument()
+        output = converter.run(sources=files)
+        docs = output["documents"]
+        assert len(docs) == 3
+        assert "Some text for testing." in docs[0].content
+        assert "This is a test line." in docs[1].content
+        assert "That's yet another file!" in docs[2].content
+        assert docs[0].meta["file_path"] == os.path.basename(files[0])
+        assert docs[1].meta["file_path"] == os.path.basename(files[1])
+        assert docs[2].meta == {"file_path": os.path.basename(bytestream.meta["file_path"]), "key": "value"}
+
+    def test_run_with_store_full_path(self, test_files_path):
+        """
+        Test if the component runs correctly with store_full_path= False.
+        """
+        bytestream = ByteStream.from_file_path(test_files_path / "txt" / "doc_3.txt")
+        bytestream.meta["file_path"] = str(test_files_path / "txt" / "doc_3.txt")
+        bytestream.meta["key"] = "value"
+        files = [str(test_files_path / "txt" / "doc_1.txt"), bytestream]
+        converter = TextFileToDocument(store_full_path=False)
+        output = converter.run(sources=files)
+        docs = output["documents"]
+        assert len(docs) == 2
+        assert "Some text for testing." in docs[0].content
+        assert "That's yet another file!" in docs[1].content
+        assert docs[0].meta["file_path"] == "doc_1.txt"
+        assert docs[1].meta["file_path"] == "doc_3.txt"
+
+    def test_run_error_handling(self, test_files_path, caplog):
+        """
+        Test if the component correctly handles errors.
+        """
+        paths = [test_files_path / "txt" / "doc_1.txt", "non_existing_file.txt", test_files_path / "txt" / "doc_3.txt"]
+        converter = TextFileToDocument()
+        with caplog.at_level(logging.WARNING):
+            output = converter.run(sources=paths)
+            assert "non_existing_file.txt" in caplog.text
+        docs = output["documents"]
+        assert len(docs) == 2
+        assert docs[0].meta["file_path"] == os.path.basename(paths[0])
+        assert docs[1].meta["file_path"] == os.path.basename(paths[2])
+
+    def test_encoding_override(self, test_files_path):
+        """
+        Test if the encoding metadata field is used properly
+        """
+        bytestream = ByteStream.from_file_path(test_files_path / "txt" / "doc_1.txt")
+        bytestream.meta["key"] = "value"
+
+        converter = TextFileToDocument(encoding="utf-16")
+        output = converter.run(sources=[bytestream])
+        assert "Some text for testing." not in output["documents"][0].content
+
+        bytestream.meta["encoding"] = "utf-8"
+        output = converter.run(sources=[bytestream])
+        assert "Some text for testing." in output["documents"][0].content
+
+    def test_run_with_meta(self):
+        bytestream = ByteStream(data=b"test", meta={"author": "test_author", "language": "en"})
+
+        converter = TextFileToDocument()
+
+        output = converter.run(sources=[bytestream], meta=[{"language": "it"}])
+        document = output["documents"][0]
+
+        # check that the metadata from the bytestream is merged with that from the meta parameter
+        assert document.meta == {"author": "test_author", "language": "it"}

test/components/converters/test_utils.py

@@ -0,0 +1,72 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import pytest
+
+from haystack.components.converters.utils import get_bytestream_from_source, normalize_metadata
+from haystack.dataclasses import ByteStream
+
+
+def test_normalize_metadata_None():
+    assert normalize_metadata(None, sources_count=1) == [{}]
+    assert normalize_metadata(None, sources_count=3) == [{}, {}, {}]
+
+
+def test_normalize_metadata_single_dict():
+    assert normalize_metadata({"a": 1}, sources_count=1) == [{"a": 1}]
+    assert normalize_metadata({"a": 1}, sources_count=3) == [{"a": 1}, {"a": 1}, {"a": 1}]
+
+
+def test_normalize_metadata_list_of_right_size():
+    assert normalize_metadata([{"a": 1}], sources_count=1) == [{"a": 1}]
+    assert normalize_metadata([{"a": 1}, {"b": 2}, {"c": 3}], sources_count=3) == [{"a": 1}, {"b": 2}, {"c": 3}]
+
+
+def test_normalize_metadata_list_of_wrong_size():
+    with pytest.raises(ValueError, match="The length of the metadata list must match the number of sources."):
+        normalize_metadata([{"a": 1}], sources_count=3)
+    with pytest.raises(ValueError, match="The length of the metadata list must match the number of sources."):
+        assert normalize_metadata([{"a": 1}, {"b": 2}, {"c": 3}], sources_count=1)
+
+
+def test_normalize_metadata_other_type():
+    with pytest.raises(ValueError, match="meta must be either None, a dictionary or a list of dictionaries."):
+        normalize_metadata(({"a": 1},), sources_count=1)
+
+
+def test_get_bytestream_from_path_object(tmp_path):
+    bytes_ = b"hello world"
+    source = tmp_path / "test.txt"
+    source.write_bytes(bytes_)
+
+    bs = get_bytestream_from_source(source, guess_mime_type=True)
+
+    assert isinstance(bs, ByteStream)
+    assert bs.data == bytes_
+    assert bs.mime_type == "text/plain"
+    assert bs.meta["file_path"].endswith("test.txt")
+
+
+def test_get_bytestream_from_string_path(tmp_path):
+    bytes_ = b"hello world"
+    source = tmp_path / "test.txt"
+    source.write_bytes(bytes_)
+
+    bs = get_bytestream_from_source(str(source), guess_mime_type=True)
+
+    assert isinstance(bs, ByteStream)
+    assert bs.data == bytes_
+    assert bs.mime_type == "text/plain"
+    assert bs.meta["file_path"].endswith("test.txt")
+
+
+def test_get_bytestream_from_source_invalid_type():
+    with pytest.raises(ValueError, match="Unsupported source type"):
+        get_bytestream_from_source(123)
+
+
+def test_get_bytestream_from_source_bytestream_passthrough():
+    bs = ByteStream(data=b"spam", mime_type="text/custom", meta={"spam": "eggs"})
+    result = get_bytestream_from_source(bs)
+    assert result is bs

test/components/converters/test_xlsx_to_document.py

@@ -0,0 +1,170 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+
+import pytest
+
+from haystack.components.converters.xlsx import XLSXToDocument
+
+
+class TestXLSXToDocument:
+    def test_init(self) -> None:
+        converter = XLSXToDocument()
+        assert converter.sheet_name is None
+        assert converter.read_excel_kwargs == {}
+        assert converter.table_format == "csv"
+        assert converter.link_format == "none"
+        assert converter.table_format_kwargs == {}
+
+    def test_run_basic_tables(self, test_files_path) -> None:
+        converter = XLSXToDocument(store_full_path=True)
+        paths = [test_files_path / "xlsx" / "basic_tables_two_sheets.xlsx"]
+        results = converter.run(sources=paths, meta={"date_added": "2022-01-01T00:00:00"})
+        documents = results["documents"]
+        assert len(documents) == 2
+        assert documents[0].content == ",A,B\n1,col_a,col_b\n2,1.5,test\n"
+        assert documents[0].meta == {
+            "date_added": "2022-01-01T00:00:00",
+            "file_path": str(test_files_path / "xlsx" / "basic_tables_two_sheets.xlsx"),
+            "xlsx": {"sheet_name": "Basic Table"},
+        }
+        assert documents[1].content == ",A,B\n1,col_c,col_d\n2,True,\n"
+        assert documents[1].meta == {
+            "date_added": "2022-01-01T00:00:00",
+            "file_path": str(test_files_path / "xlsx" / "basic_tables_two_sheets.xlsx"),
+            "xlsx": {"sheet_name": "Table Missing Value"},
+        }
+
+    def test_run_table_empty_rows_and_columns(self, test_files_path) -> None:
+        converter = XLSXToDocument(store_full_path=False)
+        paths = [test_files_path / "xlsx" / "table_empty_rows_and_columns.xlsx"]
+        results = converter.run(sources=paths, meta={"date_added": "2022-01-01T00:00:00"})
+        documents = results["documents"]
+        assert len(documents) == 1
+        assert documents[0].content == ",A,B,C\n1,,,\n2,,,\n3,,,\n4,,col_a,col_b\n5,,1.5,test\n"
+        assert documents[0].meta == {
+            "date_added": "2022-01-01T00:00:00",
+            "file_path": "table_empty_rows_and_columns.xlsx",
+            "xlsx": {"sheet_name": "Sheet1"},
+        }
+
+    def test_run_multiple_tables_in_one_sheet(self, test_files_path) -> None:
+        converter = XLSXToDocument(store_full_path=True)
+        paths = [test_files_path / "xlsx" / "multiple_tables.xlsx"]
+        results = converter.run(sources=paths, meta={"date_added": "2022-01-01T00:00:00"})
+        documents = results["documents"]
+        assert len(documents) == 1
+        assert (
+            documents[0].content
+            == ",A,B,C,D,E,F\n1,,,,,,\n2,,,,,,\n3,,col_a,col_b,,,\n4,,1.5,test,,col_c,col_d\n5,,,,,3,True\n"
+        )
+        assert documents[0].meta == {
+            "date_added": "2022-01-01T00:00:00",
+            "file_path": str(test_files_path / "xlsx" / "multiple_tables.xlsx"),
+            "xlsx": {"sheet_name": "Sheet1"},
+        }
+
+    def test_run_markdown(self, test_files_path) -> None:
+        converter = XLSXToDocument(table_format="markdown", store_full_path=True)
+        paths = [test_files_path / "xlsx" / "basic_tables_two_sheets.xlsx"]
+        results = converter.run(sources=paths, meta={"date_added": "2022-01-01T00:00:00"})
+        documents = results["documents"]
+        assert len(documents) == 2
+        assert (
+            documents[0].content
+            == "|    | A     | B     |\n|---:|:------|:------|\n|  1 | col_a | col_b |\n|  2 | 1.5   | test  |"
+        )
+        assert documents[0].meta == {
+            "date_added": "2022-01-01T00:00:00",
+            "file_path": str(test_files_path / "xlsx" / "basic_tables_two_sheets.xlsx"),
+            "xlsx": {"sheet_name": "Basic Table"},
+        }
+        assert (
+            documents[1].content
+            == "|    | A     | B     |\n|---:|:------|:------|\n|  1 | col_c | col_d |\n|  2 | True  | nan   |"
+        )
+        assert documents[1].meta == {
+            "date_added": "2022-01-01T00:00:00",
+            "file_path": str(test_files_path / "xlsx" / "basic_tables_two_sheets.xlsx"),
+            "xlsx": {"sheet_name": "Table Missing Value"},
+        }
+
+    @pytest.mark.parametrize(
+        "sheet_name, expected_sheet_name, expected_content",
+        [
+            ("Basic Table", "Basic Table", ",A,B\n1,col_a,col_b\n2,1.5,test\n"),
+            ("Table Missing Value", "Table Missing Value", ",A,B\n1,col_c,col_d\n2,True,\n"),
+            (0, 0, ",A,B\n1,col_a,col_b\n2,1.5,test\n"),
+            (1, 1, ",A,B\n1,col_c,col_d\n2,True,\n"),
+        ],
+    )
+    def test_run_sheet_name(
+        self, sheet_name: int | str, expected_sheet_name: str, expected_content: str, test_files_path
+    ) -> None:
+        converter = XLSXToDocument(sheet_name=sheet_name, store_full_path=True)
+        paths = [test_files_path / "xlsx" / "basic_tables_two_sheets.xlsx"]
+        results = converter.run(sources=paths)
+        documents = results["documents"]
+        assert len(documents) == 1
+        assert documents[0].content == expected_content
+        assert documents[0].meta == {
+            "file_path": str(test_files_path / "xlsx" / "basic_tables_two_sheets.xlsx"),
+            "xlsx": {"sheet_name": expected_sheet_name},
+        }
+
+    def test_run_with_read_excel_kwargs(self, test_files_path) -> None:
+        converter = XLSXToDocument(sheet_name="Basic Table", read_excel_kwargs={"skiprows": 1}, store_full_path=True)
+        paths = [test_files_path / "xlsx" / "basic_tables_two_sheets.xlsx"]
+        results = converter.run(sources=paths, meta={"date_added": "2022-01-01T00:00:00"})
+        documents = results["documents"]
+        assert len(documents) == 1
+        assert documents[0].content == ",A,B\n1,1.5,test\n"
+        assert documents[0].meta == {
+            "date_added": "2022-01-01T00:00:00",
+            "file_path": str(test_files_path / "xlsx" / "basic_tables_two_sheets.xlsx"),
+            "xlsx": {"sheet_name": "Basic Table"},
+        }
+
+    def test_run_error_wrong_file_type(self, caplog: pytest.LogCaptureFixture, test_files_path) -> None:
+        converter = XLSXToDocument()
+        sources = [test_files_path / "pdf" / "sample_pdf_1.pdf"]
+        with caplog.at_level(logging.WARNING):
+            results = converter.run(sources=sources)
+            assert "sample_pdf_1.pdf and convert it" in caplog.text
+            assert results["documents"] == []
+
+    def test_run_error_non_existent_file(self, caplog: pytest.LogCaptureFixture) -> None:
+        converter = XLSXToDocument()
+        paths = ["non_existing_file.docx"]
+        with caplog.at_level(logging.WARNING):
+            converter.run(sources=paths)
+            assert "Could not read non_existing_file.docx" in caplog.text
+
+    def test_link_format_invalid(self) -> None:
+        with pytest.raises(ValueError, match="Unknown link format"):
+            XLSXToDocument(link_format="invalid")
+
+    @pytest.mark.parametrize("link_format", ["markdown", "plain"])
+    def test_link_extraction(self, test_files_path, link_format) -> None:
+        converter = XLSXToDocument(link_format=link_format)
+        paths = [test_files_path / "xlsx" / "spreadsheet_with_links.xlsx"]
+        results = converter.run(sources=paths)
+        content = results["documents"][0].content
+
+        if link_format == "markdown":
+            assert "[Click here](https://example.com)" in content
+            assert "[Docs](https://python.org)" in content
+        else:
+            assert "Click here (https://example.com)" in content
+            assert "Docs (https://python.org)" in content
+
+    def test_no_link_extraction(self, test_files_path) -> None:
+        converter = XLSXToDocument()
+        paths = [test_files_path / "xlsx" / "spreadsheet_with_links.xlsx"]
+        results = converter.run(sources=paths)
+        content = results["documents"][0].content
+
+        assert "https://example.com" not in content
+        assert "Click here" in content