chore: import upstream snapshot with attribution

haystack/components/extractors/__init__.py

@@ -0,0 +1,17 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {"llm_metadata_extractor": ["LLMMetadataExtractor"], "regex_text_extractor": ["RegexTextExtractor"]}
+
+if TYPE_CHECKING:
+    from .llm_metadata_extractor import LLMMetadataExtractor as LLMMetadataExtractor
+    from .regex_text_extractor import RegexTextExtractor as RegexTextExtractor
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/extractors/image/__init__.py

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import sys
+from typing import TYPE_CHECKING
+
+from lazy_imports import LazyImporter
+
+_import_structure = {"llm_document_content_extractor": ["LLMDocumentContentExtractor"]}
+
+if TYPE_CHECKING:
+    from .llm_document_content_extractor import LLMDocumentContentExtractor as LLMDocumentContentExtractor
+
+else:
+    sys.modules[__name__] = LazyImporter(name=__name__, module_file=__file__, import_structure=_import_structure)

haystack/components/extractors/image/llm_document_content_extractor.py

@@ -0,0 +1,422 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import asyncio
+import json
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import replace
+from typing import Any, Literal
+
+from jinja2 import meta
+from jinja2.sandbox import SandboxedEnvironment
+
+from haystack import Document, component, default_from_dict, default_to_dict, logging
+from haystack.components.converters.image.document_to_image import DocumentToImageContent
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.core.serialization import component_to_dict
+from haystack.dataclasses import ImageContent, TextContent
+from haystack.dataclasses.chat_message import ChatMessage
+from haystack.utils import deserialize_chatgenerator_inplace
+from haystack.utils.async_utils import _execute_component_async
+from haystack.utils.misc import _parse_dict_from_json
+
+logger = logging.getLogger(__name__)
+
+
+# Reserved key in the LLM JSON response that holds the main document text.
+DOCUMENT_CONTENT_KEY = "document_content"
+
+
+DEFAULT_PROMPT_TEMPLATE = """
+You are part of an information extraction pipeline that extracts the content of image-based documents.
+
+Extract the content from the provided image.
+You need to extract the content exactly.
+Format everything as markdown.
+Make sure to retain the reading order of the document.
+
+**Visual Elements**
+Do not extract figures, drawings, maps, graphs or any other visual elements.
+Instead, add a caption that describes briefly what you see in the visual element.
+You must describe each visual element.
+If you only see a visual element without other content, you must describe this visual element.
+Enclose each image caption with [img-caption][/img-caption]
+
+**Tables**
+Make sure to format the table in markdown.
+Add a short caption below the table that describes the table's content.
+Enclose each table caption with [table-caption][/table-caption].
+The caption must be placed below the extracted table.
+
+**Forms**
+Reproduce checkbox selections with markdown.
+
+Return a single JSON object. It must contain the key "document_content" with the extracted text as value.
+
+No markdown, no code fence, only raw JSON.
+
+Document:"""
+
+
+@component
+class LLMDocumentContentExtractor:
+    """
+    Extracts textual content and optionally metadata from image-based documents using a vision-enabled LLM.
+
+    One prompt and one LLM call per document. The component converts each document to an image via
+    DocumentToImageContent and sends it to the ChatGenerator. The prompt must not contain Jinja variables.
+
+    Response handling:
+    - If the LLM returns a **plain string** (non-JSON or not a JSON object), it is written to the document's content.
+    - If the LLM returns a **JSON object with only the key** `document_content`, that value is written to content.
+    - If the LLM returns a **JSON object with multiple keys**, the value of ``document_content`` (if present) is
+      written to content and all other keys are merged into the document's metadata.
+
+    The ChatGenerator can be configured to return JSON (e.g. ``response_format={"type": "json_object"}``
+    in ``generation_kwargs``).
+
+    Documents that fail extraction are returned in ``failed_documents`` with ``content_extraction_error`` in metadata.
+
+    ### Usage example
+
+    ```python
+    from haystack import Document
+    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.components.extractors.image import LLMDocumentContentExtractor
+
+    prompt = \"\"\"
+    Extract the content from the provided image.
+    Format everything as markdown. Return only the extracted content as a JSON object with the key 'document_content'.
+    No markdown, no code fence, only raw JSON.
+
+    Extract metadata about the image like source of the image, date of creation, etc. if you can.
+    Return this metadata as additional key-value pairs in the same JSON object.
+    \"\"\"
+
+    chat_generator = OpenAIChatGenerator(
+            generation_kwargs={
+                    "response_format": {
+                        "type": "json_schema",
+                        "json_schema": {
+                            "name": "entity_extraction",
+                            "schema": {
+                                "type": "object",
+                                "properties": {
+                                    "document_content": {"type": "string"},
+                                    "author": {"type": "string"},
+                                    "date": {"type": "string"},
+                                    "document_type": {"type": "string"},
+                                    "title": {"type": "string"},
+                                },
+                                "additionalProperties": False,
+                            },
+                        },
+                    }
+                }
+            )
+
+    extractor = LLMDocumentContentExtractor(
+        chat_generator=chat_generator,
+        file_path_meta_field="file_path",
+        raise_on_failure=False
+    )
+
+    documents = [
+        Document(content="", meta={"file_path": "test/test_files/images/image_metadata.png"}),
+        Document(content="", meta={"file_path": "test/test_files/images/apple.jpg", "page_number": 1})
+    ]
+    result = extractor.run(documents=documents)
+    updated_documents = result["documents"]
+    ```
+    """
+
+    def __init__(
+        self,
+        *,
+        chat_generator: ChatGenerator,
+        prompt: str = DEFAULT_PROMPT_TEMPLATE,
+        file_path_meta_field: str = "file_path",
+        root_path: str | None = None,
+        detail: Literal["auto", "high", "low"] | None = None,
+        size: tuple[int, int] | None = None,
+        raise_on_failure: bool = False,
+        max_workers: int = 3,
+    ) -> None:
+        """
+        Initialize the LLMDocumentContentExtractor component.
+
+        :param chat_generator: A ChatGenerator that supports vision input. Optionally configured for JSON
+            (e.g. ``response_format={"type": "json_object"}`` in ``generation_kwargs``).
+        :param prompt: Prompt for extraction. Must not contain Jinja variables.
+        :param file_path_meta_field: The metadata field in the Document that contains the file path to the image or PDF.
+        :param root_path: The root directory path where document files are located. If provided, file paths in
+            document metadata will be resolved relative to this path. If None, file paths are treated as absolute paths.
+        :param detail: Optional detail level of the image (only supported by OpenAI). Can be "auto", "high", or "low".
+        :param size: If provided, resizes the image to fit within (width, height) while keeping aspect ratio.
+        :param raise_on_failure: If True, exceptions from the LLM are raised. If False, failed documents are returned.
+        :param max_workers: Maximum number of threads for parallel LLM calls.
+        """
+        self._chat_generator = chat_generator
+        self.prompt = prompt
+        self.file_path_meta_field = file_path_meta_field
+        self.root_path = root_path or ""
+        self.detail = detail
+        self.size = size
+        LLMDocumentContentExtractor._validate_prompt_no_variables(prompt)
+        self.raise_on_failure = raise_on_failure
+        self.max_workers = max_workers
+        self._document_to_image_content = DocumentToImageContent(
+            file_path_meta_field=file_path_meta_field, root_path=root_path, detail=detail, size=size
+        )
+
+    def warm_up(self) -> None:
+        """
+        Warm up the underlying chat generator.
+        """
+        if hasattr(self._chat_generator, "warm_up"):
+            self._chat_generator.warm_up()
+
+    async def warm_up_async(self) -> None:
+        """
+        Warm up the underlying chat generator on the serving event loop.
+        """
+        if hasattr(self._chat_generator, "warm_up_async"):
+            await self._chat_generator.warm_up_async()
+        elif hasattr(self._chat_generator, "warm_up"):
+            self._chat_generator.warm_up()
+
+    def close(self) -> None:
+        """
+        Release the underlying chat generator's resources.
+        """
+        if hasattr(self._chat_generator, "close"):
+            self._chat_generator.close()
+
+    async def close_async(self) -> None:
+        """
+        Release the underlying chat generator's async resources.
+        """
+        if hasattr(self._chat_generator, "close_async"):
+            await self._chat_generator.close_async()
+        elif hasattr(self._chat_generator, "close"):
+            self._chat_generator.close()
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(
+            self,
+            chat_generator=component_to_dict(obj=self._chat_generator, name="chat_generator"),
+            prompt=self.prompt,
+            file_path_meta_field=self.file_path_meta_field,
+            root_path=self.root_path,
+            detail=self.detail,
+            size=self.size,
+            raise_on_failure=self.raise_on_failure,
+            max_workers=self.max_workers,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "LLMDocumentContentExtractor":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary with serialized data.
+        :returns:
+            An instance of the component.
+        """
+        init_params = data.get("init_parameters", {})
+        deserialize_chatgenerator_inplace(init_params, key="chat_generator")
+
+        return default_from_dict(cls, data)
+
+    @staticmethod
+    def _validate_prompt_no_variables(prompt: str) -> None:
+        ast = SandboxedEnvironment().parse(prompt)
+        template_variables = meta.find_undeclared_variables(ast)
+        variables = list(template_variables)
+        if variables:
+            raise ValueError(
+                f"The prompt must not have any variables, only instructions on how to extract the content of the "
+                f"the image-based document. Found {','.join(variables)} in the prompt."
+            )
+
+    @staticmethod
+    def _process_response(response_text: str) -> tuple[str | None, dict[str, Any], str | None]:
+        """
+        Parse LLM response. Returns (content, meta_updates, error).
+
+        - Plain string (non-JSON): use entire response as document content;
+        - Valid JSON object: use key ``document_content`` for Document.content and all other keys for Document.metadata;
+        - Valid JSON but not an object (e.g. array or primitive), report an error;
+        """
+        try:
+            parsed = _parse_dict_from_json(response_text, raise_on_failure=True)
+        except json.JSONDecodeError:
+            return response_text, {}, None
+        except ValueError:
+            return None, {}, "Response must be a JSON object, not an array or primitive."
+
+        content = parsed.get(DOCUMENT_CONTENT_KEY)
+        meta_updates = {k: v for k, v in parsed.items() if k != DOCUMENT_CONTENT_KEY}
+        return content, meta_updates, None
+
+    def _run_on_thread(self, image_content: ImageContent | None) -> dict[str, Any]:
+        """
+        Execute the LLM inference in a separate thread for each document.
+
+        :param image_content: The image content for one document, or None if conversion failed.
+        :returns:
+            The LLM response if successful, or a dictionary with an "error" key on failure.
+        """
+        if image_content is None:
+            return {"error": "Document has no content, skipping LLM call."}
+
+        # the prompt is the same for all documents, so we can set it up once here for each document/thread
+        message = ChatMessage.from_user(content_parts=[TextContent(text=self.prompt), image_content])
+
+        try:
+            result = self._chat_generator.run(messages=[message])
+        except Exception as e:
+            if self.raise_on_failure:
+                raise e
+            logger.exception(
+                "LLM {class_name} execution failed. Skipping metadata extraction. Failed with exception '{error}'.",
+                class_name=self._chat_generator.__class__.__name__,
+                error=e,
+            )
+            result = {"error": "LLM failed with exception: " + str(e)}
+
+        return result
+
+    async def _run_async(self, image_content: ImageContent | None) -> dict[str, Any]:
+        """
+        Execute the LLM inference asynchronously for each document.
+
+        :param image_content: The image content for one document, or None if conversion failed.
+        :returns:
+            The LLM response if successful, or a dictionary with an "error" key on failure.
+        """
+        if image_content is None:
+            return {"error": "Document has no content, skipping LLM call."}
+
+        # the prompt is the same for all documents, so we can set it up once here for each document
+        message = ChatMessage.from_user(content_parts=[TextContent(text=self.prompt), image_content])
+
+        try:
+            result = await _execute_component_async(self._chat_generator, messages=[message])
+        except Exception as e:
+            if self.raise_on_failure:
+                raise e
+            logger.exception(
+                "LLM {class_name} execution failed. Skipping metadata extraction. Failed with exception '{error}'.",
+                class_name=self._chat_generator.__class__.__name__,
+                error=e,
+            )
+            result = {"error": "LLM failed with exception: " + str(e)}
+
+        return result
+
+    @staticmethod
+    def _process_llm_results(document: Document, result: dict[str, Any]) -> tuple[Document, bool]:
+        """
+        Process one document's LLM result using the unified response logic.
+
+        Returns (updated_document, True if success else False).
+        """
+        if "error" in result:
+            new_meta = {**document.meta, "extraction_error": result["error"]}
+            return replace(document, meta=new_meta), False
+
+        # remove potentially existing error metadata from previous runs
+        new_meta = {**document.meta}
+        new_meta.pop("extraction_error", None)
+
+        # process the LLM response considering the possible response formats
+        response_text = result["replies"][0].text
+        content, meta_updates, error = LLMDocumentContentExtractor._process_response(response_text)
+
+        if error:
+            new_meta["extraction_error"] = error
+            return replace(document, meta=new_meta), False
+
+        new_meta.update(meta_updates)
+        final_content = document.content if content is None else content
+        return replace(document, content=final_content, meta=new_meta), True
+
+    @component.output_types(documents=list[Document], failed_documents=list[Document])
+    def run(self, documents: list[Document]) -> dict[str, list[Document]]:
+        """
+        Run extraction on image-based documents. One LLM call per document.
+
+        :param documents: A list of image-based documents to process. Each must have a valid file path in its metadata.
+        :returns:
+            A dictionary with "documents" (successfully processed) and "failed_documents" (with failure metadata).
+        """
+        if not documents:
+            return {"documents": [], "failed_documents": []}
+
+        self.warm_up()
+
+        image_contents = self._document_to_image_content.run(documents=documents)["image_contents"]
+
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            results = executor.map(self._run_on_thread, image_contents)
+
+        successful_documents = []
+        failed_documents = []
+        for document, result in zip(documents, results, strict=True):
+            doc, success = self._process_llm_results(document, result)
+            if success:
+                successful_documents.append(doc)
+            else:
+                failed_documents.append(doc)
+
+        return {"documents": successful_documents, "failed_documents": failed_documents}
+
+    @component.output_types(documents=list[Document], failed_documents=list[Document])
+    async def run_async(self, documents: list[Document]) -> dict[str, list[Document]]:
+        """
+        Asynchronously run extraction on image-based documents. One LLM call per document.
+
+        This is the asynchronous version of the `run` method. It has the same parameters and return values
+        but can be used with `await` in an async code. LLM calls are made concurrently, bounded by `max_workers`.
+        If the chat generator only implements a synchronous `run` method, it is executed in a thread to avoid
+        blocking the event loop.
+
+        :param documents: A list of image-based documents to process. Each must have a valid file path in its metadata.
+        :returns:
+            A dictionary with "documents" (successfully processed) and "failed_documents" (with failure metadata).
+        """
+        if not documents:
+            return {"documents": [], "failed_documents": []}
+
+        await self.warm_up_async()
+
+        image_contents = self._document_to_image_content.run(documents=documents)["image_contents"]
+
+        # Run the LLM on each image content, bounding concurrency per task so max_workers is enforced.
+        sem = asyncio.Semaphore(max(1, self.max_workers))
+
+        async def _bounded_run(image_content: ImageContent | None) -> dict[str, Any]:
+            async with sem:
+                return await self._run_async(image_content)
+
+        results = await asyncio.gather(*[_bounded_run(image_content) for image_content in image_contents])
+
+        successful_documents = []
+        failed_documents = []
+        for document, result in zip(documents, results, strict=True):
+            doc, success = self._process_llm_results(document, result)
+            if success:
+                successful_documents.append(doc)
+            else:
+                failed_documents.append(doc)
+
+        return {"documents": successful_documents, "failed_documents": failed_documents}

haystack/components/extractors/llm_metadata_extractor.py

@@ -0,0 +1,474 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import copy
+import json
+from asyncio import Semaphore, gather
+from collections.abc import Iterable
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import replace
+from typing import Any
+
+from jinja2 import meta
+from jinja2.sandbox import SandboxedEnvironment
+
+from haystack import Document, component, default_from_dict, default_to_dict, logging
+from haystack.components.builders import PromptBuilder
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.core.serialization import component_to_dict
+from haystack.dataclasses import ChatMessage
+from haystack.utils import deserialize_chatgenerator_inplace, expand_page_range
+from haystack.utils.async_utils import _execute_component_async
+from haystack.utils.misc import _parse_dict_from_json
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class LLMMetadataExtractor:
+    """
+    Extracts metadata from documents using a Large Language Model (LLM).
+
+    The metadata is extracted by providing a prompt to an LLM that generates the metadata.
+
+    This component expects as input a list of documents and a prompt. The prompt must have exactly one variable, called
+    `document`, that points to a single document in the list of documents. So to access the content of the document,
+    you can use `{{ document.content }}` in the prompt.
+
+    The component will run the LLM on each document in the list and extract metadata from the document. The metadata
+    will be added to the document's metadata field. If the LLM fails to extract metadata from a document, the document
+    will be added to the `failed_documents` list. The failed documents will have the keys `metadata_extraction_error` and
+    `metadata_extraction_response` in their metadata. These documents can be re-run with another extractor to
+    extract metadata by using the `metadata_extraction_response` and `metadata_extraction_error` in the prompt.
+
+    ```python
+    from haystack import Document
+    from haystack.components.extractors.llm_metadata_extractor import LLMMetadataExtractor
+    from haystack.components.generators.chat import OpenAIChatGenerator
+
+    NER_PROMPT = '''
+    -Goal-
+    Given text and a list of entity types, identify all entities of those types from the text.
+
+    -Steps-
+    1. Identify all entities. For each identified entity, extract the following information:
+    - entity: Name of the entity
+    - entity_type: One of the following types: [organization, product, service, industry]
+    Format each entity as a JSON like: {"entity": <entity_name>, "entity_type": <entity_type>}
+
+    2. Return output in a single list with all the entities identified in steps 1.
+
+    -Examples-
+    ######################
+    Example 1:
+    entity_types: [organization, person, partnership, financial metric, product, service, industry, investment strategy, market trend]
+    text: Another area of strength is our co-brand issuance. Visa is the primary network partner for eight of the top
+    10 co-brand partnerships in the US today and we are pleased that Visa has finalized a multi-year extension of
+    our successful credit co-branded partnership with Alaska Airlines, a portfolio that benefits from a loyal customer
+    base and high cross-border usage.
+    We have also had significant co-brand momentum in CEMEA. First, we launched a new co-brand card in partnership
+    with Qatar Airways, British Airways and the National Bank of Kuwait. Second, we expanded our strong global
+    Marriott relationship to launch Qatar's first hospitality co-branded card with Qatar Islamic Bank. Across the
+    United Arab Emirates, we now have exclusive agreements with all the leading airlines marked by a recent
+    agreement with Emirates Skywards.
+    And we also signed an inaugural Airline co-brand agreement in Morocco with Royal Air Maroc. Now newer digital
+    issuers are equally
+    ------------------------
+    output:
+    {"entities": [{"entity": "Visa", "entity_type": "company"}, {"entity": "Alaska Airlines", "entity_type": "company"}, {"entity": "Qatar Airways", "entity_type": "company"}, {"entity": "British Airways", "entity_type": "company"}, {"entity": "National Bank of Kuwait", "entity_type": "company"}, {"entity": "Marriott", "entity_type": "company"}, {"entity": "Qatar Islamic Bank", "entity_type": "company"}, {"entity": "Emirates Skywards", "entity_type": "company"}, {"entity": "Royal Air Maroc", "entity_type": "company"}]}
+    #############################
+    -Real Data-
+    ######################
+    entity_types: [company, organization, person, country, product, service]
+    text: {{ document.content }}
+    ######################
+    output:
+    '''
+
+    docs = [
+        Document(content="deepset was founded in 2018 in Berlin, and is known for its Haystack framework"),
+        Document(content="Hugging Face is a company that was founded in New York, USA and is known for its Transformers library")
+    ]
+
+    chat_generator = OpenAIChatGenerator(
+        generation_kwargs={
+            "max_completion_tokens": 500,
+            "temperature": 0.0,
+            "seed": 0,
+            "response_format": {
+                "type": "json_schema",
+                "json_schema": {
+                    "name": "entity_extraction",
+                    "schema": {
+                        "type": "object",
+                        "properties": {
+                            "entities": {
+                                "type": "array",
+                                "items": {
+                                    "type": "object",
+                                    "properties": {
+                                        "entity": {"type": "string"},
+                                        "entity_type": {"type": "string"}
+                                    },
+                                    "required": ["entity", "entity_type"],
+                                    "additionalProperties": False
+                                }
+                            }
+                        },
+                        "required": ["entities"],
+                        "additionalProperties": False
+                    }
+                }
+            },
+        },
+        max_retries=1,
+        timeout=60.0,
+    )
+
+    extractor = LLMMetadataExtractor(
+        prompt=NER_PROMPT,
+        chat_generator=chat_generator,
+        expected_keys=["entities"],
+        raise_on_failure=False,
+    )
+
+    extractor.run(documents=docs)
+    # >> {'documents': [
+    #     Document(id=.., content: 'deepset was founded in 2018 in Berlin, and is known for its Haystack framework',
+    #     meta: {'entities': [{'entity': 'deepset', 'entity_type': 'company'}, {'entity': 'Berlin', 'entity_type': 'city'},
+    #           {'entity': 'Haystack', 'entity_type': 'product'}]}),
+    #     Document(id=.., content: 'Hugging Face is a company that was founded in New York, USA and is known for its Transformers library',
+    #     meta: {'entities': [
+    #             {'entity': 'Hugging Face', 'entity_type': 'company'}, {'entity': 'New York', 'entity_type': 'city'},
+    #             {'entity': 'USA', 'entity_type': 'country'}, {'entity': 'Transformers', 'entity_type': 'product'}
+    #             ]})
+    #        ]
+    #     'failed_documents': []
+    #    }
+    # >>
+    ```
+    """  # noqa: E501
+
+    def __init__(
+        self,
+        prompt: str,
+        chat_generator: ChatGenerator,
+        expected_keys: list[str] | None = None,
+        page_range: list[str | int] | None = None,
+        raise_on_failure: bool = False,
+        max_workers: int = 3,
+    ) -> None:
+        """
+        Initializes the LLMMetadataExtractor.
+
+        :param prompt: The prompt to be used for the LLM. It must contain exactly one variable, called `document`,
+            which points to a single document in the list of documents. For example, to access the content of the
+            document, use `{{ document.content }}` in the prompt.
+        :param chat_generator: a ChatGenerator instance which represents the LLM. In order for the component to work,
+            the LLM should be configured to return a JSON object. For example, when using the OpenAIChatGenerator, you
+            should pass `{"response_format": {"type": "json_object"}}` in the `generation_kwargs`.
+        :param expected_keys: The keys expected in the JSON output from the LLM.
+        :param page_range: A range of pages to extract metadata from. For example, page_range=['1', '3'] will extract
+            metadata from the first and third pages of each document. It also accepts printable range strings, e.g.:
+            ['1-3', '5', '8', '10-12'] will extract metadata from pages 1, 2, 3, 5, 8, 10,11, 12.
+            If None, metadata will be extracted from the entire document for each document in the documents list.
+            This parameter is optional and can be overridden in the `run` method.
+        :param raise_on_failure: Whether to raise an error on failure during the execution of the Generator or
+            validation of the JSON output.
+        :param max_workers: The maximum number of workers to use in the thread pool executor.
+            This parameter is used limit the maximum number of requests that should be allowed to run concurrently
+            when using the `run_async` method.
+        """
+        self.prompt = prompt
+        ast = SandboxedEnvironment().parse(prompt)
+        template_variables = meta.find_undeclared_variables(ast)
+        variables = list(template_variables)
+        if variables != ["document"]:
+            raise ValueError(
+                f"Prompt must have exactly one variable called 'document'. "
+                f"Found {','.join(variables) or 'no variables'} in the prompt."
+            )
+        self.builder = PromptBuilder(prompt, required_variables=variables)
+        self.raise_on_failure = raise_on_failure
+        self.expected_keys = expected_keys or []
+        self.splitter = DocumentSplitter(split_by="page", split_length=1)
+        self.expanded_range = expand_page_range(page_range) if page_range else None
+        self.max_workers = max_workers
+        self._chat_generator = chat_generator
+
+    def warm_up(self) -> None:
+        """
+        Warm up the underlying chat generator and splitter.
+        """
+        for inner in (self._chat_generator, self.splitter):
+            if hasattr(inner, "warm_up"):
+                inner.warm_up()
+
+    async def warm_up_async(self) -> None:
+        """
+        Warm up the underlying chat generator and splitter on the serving event loop.
+        """
+        for inner in (self._chat_generator, self.splitter):
+            if hasattr(inner, "warm_up_async"):
+                await inner.warm_up_async()
+            elif hasattr(inner, "warm_up"):
+                inner.warm_up()
+
+    def close(self) -> None:
+        """
+        Release the underlying chat generator's and splitter's resources.
+        """
+        for inner in (self._chat_generator, self.splitter):
+            if hasattr(inner, "close"):
+                inner.close()
+
+    async def close_async(self) -> None:
+        """
+        Release the underlying chat generator's and splitter's async resources.
+        """
+        for inner in (self._chat_generator, self.splitter):
+            if hasattr(inner, "close_async"):
+                await inner.close_async()
+            elif hasattr(inner, "close"):
+                inner.close()
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+
+        return default_to_dict(
+            self,
+            prompt=self.prompt,
+            chat_generator=component_to_dict(obj=self._chat_generator, name="chat_generator"),
+            expected_keys=self.expected_keys,
+            page_range=self.expanded_range,
+            raise_on_failure=self.raise_on_failure,
+            max_workers=self.max_workers,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "LLMMetadataExtractor":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            Dictionary with serialized data.
+        :returns:
+            An instance of the component.
+        """
+
+        deserialize_chatgenerator_inplace(data["init_parameters"], key="chat_generator")
+        return default_from_dict(cls, data)
+
+    def _extract_metadata(self, llm_answer: str) -> dict[str, Any]:
+        try:
+            parsed_metadata = _parse_dict_from_json(llm_answer, expected_keys=self.expected_keys, raise_on_failure=True)
+        except (ValueError, json.JSONDecodeError) as e:
+            logger.warning(
+                "Response from the LLM is not valid JSON or missing expected keys. Received output: {response}",
+                response=llm_answer,
+            )
+            if self.raise_on_failure:
+                raise e
+            return {"error": "Response is not valid JSON or missing keys. Error: " + str(e)}
+
+        return parsed_metadata
+
+    def _prepare_prompts(
+        self, documents: list[Document], expanded_range: list[int] | None = None
+    ) -> list[ChatMessage | None]:
+        all_prompts: list[ChatMessage | None] = []
+        for document in documents:
+            if not document.content:
+                logger.warning("Document {doc_id} has no content. Skipping metadata extraction.", doc_id=document.id)
+                all_prompts.append(None)
+                continue
+
+            if expanded_range:
+                doc_copy = copy.deepcopy(document)
+                pages = self.splitter.run(documents=[doc_copy])
+                content = ""
+                for idx, page in enumerate(pages["documents"]):
+                    if idx + 1 in expanded_range and page.content is not None:
+                        content += page.content
+                doc_copy = replace(doc_copy, content=content)
+            else:
+                doc_copy = document
+
+            prompt_with_doc = self.builder.run(template=self.prompt, template_variables={"document": doc_copy})
+
+            # build a ChatMessage with the prompt
+            message = ChatMessage.from_user(prompt_with_doc["prompt"])
+            all_prompts.append(message)
+
+        return all_prompts
+
+    def _run_on_thread(self, prompt: ChatMessage | None) -> dict[str, Any]:
+        # If prompt is None, return an error dictionary
+        if prompt is None:
+            return {"error": "Document has no content, skipping LLM call."}
+
+        try:
+            result = self._chat_generator.run(messages=[prompt])
+        except Exception as e:
+            if self.raise_on_failure:
+                raise e
+            logger.exception(
+                "LLM {class_name} execution failed. Skipping metadata extraction. Failed with exception '{error}'.",
+                class_name=self._chat_generator.__class__.__name__,
+                error=e,
+            )
+            result = {"error": "LLM failed with exception: " + str(e)}
+        return result
+
+    async def _run_async(self, prompt: ChatMessage | None) -> dict[str, Any]:
+        # If prompt is None, return an error dictionary
+        if prompt is None:
+            return {"error": "Document has no content, skipping LLM call."}
+
+        try:
+            result = await _execute_component_async(self._chat_generator, messages=[prompt])
+        except Exception as e:
+            if self.raise_on_failure:
+                raise e
+            logger.exception(
+                "LLM {class_name} execution failed. Skipping metadata extraction. Failed with exception '{error}'.",
+                class_name=self._chat_generator.__class__.__name__,
+                error=e,
+            )
+            result = {"error": "LLM failed with exception: " + str(e)}
+        return result
+
+    def _process_results(
+        self, documents: list[Document], results: Iterable[dict[str, Any]]
+    ) -> tuple[list[Document], list[Document]]:
+        successful_documents = []
+        failed_documents = []
+        for document, result in zip(documents, results, strict=True):
+            new_meta = {**document.meta}
+            if "error" in result:
+                new_meta["metadata_extraction_error"] = result["error"]
+                new_meta["metadata_extraction_response"] = None
+                failed_documents.append(replace(document, meta=new_meta))
+                continue
+
+            parsed_metadata = self._extract_metadata(result["replies"][0].text)
+            if "error" in parsed_metadata:
+                new_meta["metadata_extraction_error"] = parsed_metadata["error"]
+                new_meta["metadata_extraction_response"] = result["replies"][0]
+                failed_documents.append(replace(document, meta=new_meta))
+                continue
+
+            for key in parsed_metadata:
+                new_meta[key] = parsed_metadata[key]
+                # Remove metadata_extraction_error and metadata_extraction_response if present from previous runs
+                new_meta.pop("metadata_extraction_error", None)
+                new_meta.pop("metadata_extraction_response", None)
+            successful_documents.append(replace(document, meta=new_meta))
+        return successful_documents, failed_documents
+
+    @component.output_types(documents=list[Document], failed_documents=list[Document])
+    def run(self, documents: list[Document], page_range: list[str | int] | None = None) -> dict[str, Any]:
+        """
+        Extract metadata from documents using a Large Language Model.
+
+        If `page_range` is provided, the metadata will be extracted from the specified range of pages. This component
+        will split the documents into pages and extract metadata from the specified range of pages. The metadata will be
+        extracted from the entire document if `page_range` is not provided.
+
+        The original documents will be returned  updated with the extracted metadata.
+
+        :param documents: List of documents to extract metadata from.
+        :param page_range: A range of pages to extract metadata from. For example, page_range=['1', '3'] will extract
+                           metadata from the first and third pages of each document. It also accepts printable range
+                           strings, e.g.: ['1-3', '5', '8', '10-12'] will extract metadata from pages 1, 2, 3, 5, 8, 10,
+                           11, 12.
+                           If None, metadata will be extracted from the entire document for each document in the
+                           documents list.
+        :returns:
+            A dictionary with the keys:
+            - "documents": A list of documents that were successfully updated with the extracted metadata.
+            - "failed_documents": A list of documents that failed to extract metadata. These documents will have
+            "metadata_extraction_error" and "metadata_extraction_response" in their metadata. These documents can be
+            re-run with the extractor to extract metadata.
+        """
+        if len(documents) == 0:
+            logger.warning("No documents provided. Skipping metadata extraction.")
+            return {"documents": [], "failed_documents": []}
+
+        self.warm_up()
+
+        expanded_range = self.expanded_range
+        if page_range:
+            expanded_range = expand_page_range(page_range)
+
+        # Create ChatMessage prompts for each document
+        all_prompts = self._prepare_prompts(documents=documents, expanded_range=expanded_range)
+
+        # Run the LLM on each prompt
+        with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
+            results = executor.map(self._run_on_thread, all_prompts)
+
+        successful_documents, failed_documents = self._process_results(documents, results)
+
+        return {"documents": successful_documents, "failed_documents": failed_documents}
+
+    @component.output_types(documents=list[Document], failed_documents=list[Document])
+    async def run_async(self, documents: list[Document], page_range: list[str | int] | None = None) -> dict[str, Any]:
+        """
+        Asynchronously extract metadata from documents using a Large Language Model.
+
+        If `page_range` is provided, the metadata will be extracted from the specified range of pages. This component
+        will split the documents into pages and extract metadata from the specified range of pages. The metadata will be
+        extracted from the entire document if `page_range` is not provided.
+
+        The original documents will be returned  updated with the extracted metadata.
+
+        This is the asynchronous version of the `run` method. It has the same parameters
+        and return values but can be used with `await` in an async code.
+
+        :param documents: List of documents to extract metadata from.
+        :param page_range: A range of pages to extract metadata from. For example, page_range=['1', '3'] will extract
+                           metadata from the first and third pages of each document. It also accepts printable range
+                           strings, e.g.: ['1-3', '5', '8', '10-12'] will extract metadata from pages 1, 2, 3, 5, 8, 10,
+                           11, 12.
+                           If None, metadata will be extracted from the entire document for each document in the
+                           documents list.
+        :returns:
+            A dictionary with the keys:
+            - "documents": A list of documents that were successfully updated with the extracted metadata.
+            - "failed_documents": A list of documents that failed to extract metadata. These documents will have
+            "metadata_extraction_error" and "metadata_extraction_response" in their metadata. These documents can be
+            re-run with the extractor to extract metadata.
+        """
+        if len(documents) == 0:
+            logger.warning("No documents provided. Skipping metadata extraction.")
+            return {"documents": [], "failed_documents": []}
+
+        await self.warm_up_async()
+
+        expanded_range = self.expanded_range
+        if page_range:
+            expanded_range = expand_page_range(page_range)
+
+        # Create ChatMessage prompts for each document
+        all_prompts = self._prepare_prompts(documents=documents, expanded_range=expanded_range)
+
+        # Run the LLM on each prompt, bounding concurrency per task so max_workers is enforced.
+        sem = Semaphore(max(1, self.max_workers))
+
+        async def _bounded_run(prompt: ChatMessage | None) -> dict[str, Any]:
+            async with sem:
+                return await self._run_async(prompt)
+
+        results = await gather(*[_bounded_run(prompt) for prompt in all_prompts])
+
+        successful_documents, failed_documents = self._process_results(documents, results)
+
+        return {"documents": successful_documents, "failed_documents": failed_documents}

haystack/components/extractors/regex_text_extractor.py

@@ -0,0 +1,146 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import re
+from typing import Any
+
+from haystack import component, logging
+from haystack.core.serialization import default_from_dict, default_to_dict
+from haystack.dataclasses import ChatMessage
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class RegexTextExtractor:
+    """
+    Extracts text from chat message or string input using a regex pattern.
+
+    RegexTextExtractor parses input text or ChatMessages using a provided regular expression pattern.
+    It can be configured to search through all messages or only the last message in a list of ChatMessages.
+
+    ### Usage example
+
+    ```python
+    from haystack.components.extractors import RegexTextExtractor
+    from haystack.dataclasses import ChatMessage
+
+    # Using with a string
+    parser = RegexTextExtractor(regex_pattern='<issue url=\"(.+)\">')
+    result = parser.run(text_or_messages='<issue url="github.com/hahahaha">hahahah</issue>')
+    # result: {"captured_text": "github.com/hahahaha"}
+
+    # Using with ChatMessages
+    messages = [ChatMessage.from_user('<issue url="github.com/hahahaha">hahahah</issue>')]
+    result = parser.run(text_or_messages=messages)
+    # result: {"captured_text": "github.com/hahahaha"}
+    ```
+    """
+
+    def __init__(self, regex_pattern: str) -> None:
+        """
+        Creates an instance of the RegexTextExtractor component.
+
+        :param regex_pattern:
+            The regular expression pattern used to extract text.
+            The pattern should include a capture group to extract the desired text.
+            Example: `'<issue url="(.+)">'` captures `'github.com/hahahaha'` from `'<issue url="github.com/hahahaha">'`.
+        """
+        self.regex_pattern = regex_pattern
+
+        # Check if the pattern has at least one capture group
+        num_groups = re.compile(regex_pattern).groups
+        if num_groups < 1:
+            logger.warning(
+                "The provided regex pattern {regex_pattern} doesn't contain any capture groups. "
+                "The entire match will be returned instead.",
+                regex_pattern=regex_pattern,
+            )
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the component to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        return default_to_dict(self, regex_pattern=self.regex_pattern)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "RegexTextExtractor":
+        """
+        Deserializes the component from a dictionary.
+
+        :param data:
+            The dictionary to deserialize from.
+        :returns:
+            The deserialized component.
+        """
+        # return_empty_on_no_match is an old parameter. We'd like to avoid that pipelines break if it's still present.
+        if "return_empty_on_no_match" in data["init_parameters"]:
+            logger.warning("The `return_empty_on_no_match` init parameter has been removed and will be ignored.")
+            data["init_parameters"].pop("return_empty_on_no_match")
+
+        return default_from_dict(cls, data)
+
+    @component.output_types(captured_text=str)
+    def run(self, text_or_messages: str | list[ChatMessage]) -> dict[str, str]:
+        """
+        Extracts text from input using the configured regex pattern.
+
+        :param text_or_messages:
+            Either a string or a list of ChatMessage objects to search through.
+
+        :returns:
+          - `{"captured_text": "matched text"}` if a match is found
+          - `{"captured_text": ""}` if no match is found
+
+        :raises TypeError: if receiving a list the last element is not a ChatMessage instance.
+        """
+        if isinstance(text_or_messages, str):
+            return self._build_result(self._extract_from_text(text_or_messages))
+        if not text_or_messages:
+            logger.warning("Received empty list of messages")
+            return {"captured_text": ""}
+        return self._process_last_message(text_or_messages)
+
+    def _build_result(self, result: str | list[str]) -> dict:
+        """Helper method to build the return dictionary based on configuration."""
+        if (isinstance(result, str) and result == "") or (isinstance(result, list) and not result):
+            return {"captured_text": ""}
+        return {"captured_text": result}
+
+    def _process_last_message(self, messages: list[ChatMessage]) -> dict:
+        """
+        Process only the last message and build the result.
+
+        :raises TypeError: If the last element of the list is not a ChatMessage instance.
+        """
+        last_message = messages[-1]
+        if not isinstance(last_message, ChatMessage):
+            raise TypeError(f"Expected ChatMessage object, got {type(last_message)}")
+        if last_message.text is None:
+            logger.warning("Last message has no text content")
+            return {"captured_text": ""}
+        result = self._extract_from_text(last_message.text)
+        return self._build_result(result)
+
+    def _extract_from_text(self, text: str) -> str | list[str]:
+        """
+        Extract text using the regex pattern.
+
+        :param text:
+            The text to search through.
+
+        :returns:
+            The text captured by the first capturing group in the regex pattern.
+            If the pattern has no capture groups, returns the entire match.
+            If no match is found, returns an empty string.
+        """
+        match = re.search(self.regex_pattern, text)
+        if not match:
+            return ""
+        if match.groups():
+            return match.group(1)
+        return match.group(0)