chore: import upstream snapshot with attribution

unstructured/__init__.py

@@ -0,0 +1,8 @@
+from .partition.utils.config import env_config
+from .telemetry import init_telemetry
+
+# init env_config
+env_config
+
+# Explicit startup boundary for telemetry (opt-in, best-effort)
+init_telemetry()

unstructured/__main__.py

@@ -0,0 +1,6 @@
+"""Allow ``python -m unstructured``."""
+
+from unstructured.cli import main
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

unstructured/__version__.py

@@ -0,0 +1 @@
+__version__ = "0.24.2"  # pragma: no cover

unstructured/chunking/__init__.py

@@ -0,0 +1,22 @@
+"""Chunking module initializer.
+
+Publishes the public aspects of the chunking sub-package interface.
+"""
+
+from __future__ import annotations
+
+from unstructured.chunking.base import CHUNK_MAX_CHARS_DEFAULT, CHUNK_MULTI_PAGE_DEFAULT
+from unstructured.chunking.dispatch import (
+    Chunker,
+    add_chunking_strategy,
+    register_chunking_strategy,
+)
+
+__all__ = [
+    "CHUNK_MAX_CHARS_DEFAULT",
+    "CHUNK_MULTI_PAGE_DEFAULT",
+    "add_chunking_strategy",
+    # -- these must be published to allow pluggable chunkers in other code-bases --
+    "Chunker",
+    "register_chunking_strategy",
+]

unstructured/chunking/basic.py

@@ -0,0 +1,124 @@
+"""Implementation of baseline chunking.
+
+This is the "plain-vanilla" chunking strategy. All the fundamental chunking behaviors are present in
+this strategy and also in all other strategies. Those are:
+
+- Maximally fill each chunk with sequential elements.
+- Isolate oversized elements and divide (only) those chunks by text-splitting.
+- Overlap when requested.
+
+"Fancier" strategies add higher-level semantic-unit boundaries to be respected. For example, in the
+by-title strategy, section boundaries are respected, meaning a chunk never contains text from two
+different sections. When a new section is detected the current chunk is closed and a new one
+started.
+"""
+
+from __future__ import annotations
+
+from typing import Iterable, Optional
+
+from unstructured.chunking.base import ChunkingOptions, PreChunker
+from unstructured.documents.elements import Element
+
+
+def chunk_elements(
+    elements: Iterable[Element],
+    *,
+    include_orig_elements: Optional[bool] = None,
+    max_characters: Optional[int] = None,
+    max_tokens: Optional[int] = None,
+    new_after_n_chars: Optional[int] = None,
+    new_after_n_tokens: Optional[int] = None,
+    overlap: Optional[int] = None,
+    overlap_all: Optional[bool] = None,
+    tokenizer: Optional[str] = None,
+    repeat_table_headers: Optional[bool] = None,
+    skip_table_chunking: Optional[bool] = None,
+    isolate_table: Optional[bool] = None,
+) -> list[Element]:
+    """Combine sequential `elements` into chunks, respecting specified text-length limits.
+
+    Produces a sequence of `CompositeElement`, `Table`, and `TableChunk` elements (chunks).
+
+    Parameters
+    ----------
+    elements
+        A list of unstructured elements. Usually the output of a partition function.
+    include_orig_elements
+        When `True` (default), add elements from pre-chunk to the `.metadata.orig_elements` field
+        of the chunk(s) formed from that pre-chunk. Among other things, this allows access to
+        original-element metadata that cannot be consolidated and is dropped in the course of
+        chunking.
+    max_characters
+        Hard maximum chunk length. No chunk will exceed this length. A single element that exceeds
+        this length will be divided into two or more chunks using text-splitting. Mutually
+        exclusive with `max_tokens`.
+    max_tokens
+        Hard maximum chunk token count. No chunk will exceed this token count. Requires `tokenizer`
+        to be specified. Mutually exclusive with `max_characters`.
+    new_after_n_chars
+        A chunk that of this length or greater is not extended to include the next element, even if
+        that element would fit without exceeding `max_characters`. A "soft max" length that can be
+        used in conjunction with `max_characters` to limit most chunks to a preferred length while
+        still allowing larger elements to be included in a single chunk without resorting to
+        text-splitting. Defaults to `max_characters` when not specified, which effectively disables
+        any soft window. Specifying 0 for this argument causes each element to appear in a chunk by
+        itself (although an element with text longer than `max_characters` will be still be split
+        into two or more chunks).
+    new_after_n_tokens
+        Token-based equivalent of `new_after_n_chars`. A chunk with this token count or greater is
+        not extended. Requires `max_tokens` and `tokenizer` to be specified.
+    overlap
+        Specifies the length of a string ("tail") to be drawn from each chunk and prefixed to the
+        next chunk as a context-preserving mechanism. By default, this only applies to split-chunks
+        where an oversized element is divided into multiple chunks by text-splitting.
+    overlap_all
+        Default: `False`. When `True`, apply overlap between "normal" chunks formed from whole
+        elements and not subject to text-splitting. Use this with caution as it produces a certain
+        level of "pollution" of otherwise clean semantic chunk boundaries.
+    tokenizer
+        The tokenizer to use for token-based chunking. Can be either an encoding name (e.g.,
+        "cl100k_base") or a model name (e.g., "gpt-4"). Required when using `max_tokens`.
+    repeat_table_headers
+        Default: `True`. When `True`, repeated table-header behavior is enabled for chunked table
+        continuations. Specify `False` to opt out and preserve legacy table-chunk behavior.
+    skip_table_chunking
+        Default: `False`. When `True`, `Table` elements are passed through unchanged without
+        being split into `TableChunk` elements, regardless of their size.
+    isolate_table
+        Default: `True`. When `True`, `Table` and `TableChunk` elements are always staged in
+        their own pre-chunk and never combined with adjacent non-table elements. Specify
+        `False` to allow tables to share pre-chunks with adjacent elements (the pre-#4307
+        behavior).
+    """
+    # -- raises ValueError on invalid parameters --
+    opts = _BasicChunkingOptions.new(
+        include_orig_elements=include_orig_elements,
+        max_characters=max_characters,
+        max_tokens=max_tokens,
+        new_after_n_chars=new_after_n_chars,
+        new_after_n_tokens=new_after_n_tokens,
+        overlap=overlap,
+        overlap_all=overlap_all,
+        tokenizer=tokenizer,
+        repeat_table_headers=repeat_table_headers,
+        skip_table_chunking=skip_table_chunking,
+        isolate_table=isolate_table,
+    )
+
+    return _chunk_elements(elements, opts)
+
+
+def _chunk_elements(elements: Iterable[Element], opts: _BasicChunkingOptions) -> list[Element]:
+    """Implementation of actual basic chunking."""
+    # -- Note(scanny): it might seem like over-abstraction for this to be a separate function but
+    # -- it eases overriding or adding individual chunking options when customizing a stock chunker.
+    return [
+        chunk
+        for pre_chunk in PreChunker.iter_pre_chunks(elements, opts)
+        for chunk in pre_chunk.iter_chunks()
+    ]
+
+
+class _BasicChunkingOptions(ChunkingOptions):
+    """Options for `basic` chunking."""

unstructured/chunking/dispatch.py

@@ -0,0 +1,325 @@
+"""Handles dispatch of elements to a chunking-strategy by name.
+
+Also provides the `@add_chunking_strategy` decorator which is the chief current user of "by-name"
+chunking dispatch.
+"""
+
+from __future__ import annotations
+
+import copy
+import dataclasses as dc
+import functools
+import inspect
+from functools import cached_property
+from typing import Any, Callable, Iterable, Optional, Protocol
+
+from lxml.etree import ParserError, tostring
+from lxml.html import fragment_fromstring
+from typing_extensions import ParamSpec
+
+from unstructured.chunking.basic import chunk_elements
+from unstructured.chunking.title import chunk_by_title
+from unstructured.documents.elements import Element, Table, TableChunk
+from unstructured.utils import get_call_args_applying_defaults
+
+_P = ParamSpec("_P")
+
+
+class Chunker(Protocol):
+    """Abstract interface for chunking functions."""
+
+    def __call__(
+        self, elements: Iterable[Element], *, max_characters: Optional[int]
+    ) -> list[Element]:
+        """A chunking function must have this signature.
+
+        In particular it must minimally have an `elements` parameter and all chunkers will have a
+        `max_characters` parameter (doesn't need to follow `elements` directly). All others can
+        vary by chunker.
+        """
+        ...
+
+
+def add_chunking_strategy(func: Callable[_P, list[Element]]) -> Callable[_P, list[Element]]:
+    """Decorator for chunking text.
+
+    Chunks the element sequence produced by the partitioner it decorates when a `chunking_strategy`
+    argument is present in the partitioner call and it names an available chunking strategy.
+    """
+    # -- Patch the docstring of the decorated function to add chunking strategy and
+    # -- chunking-related argument documentation. This only applies when `chunking_strategy`
+    # -- is an explicit argument of the decorated function and "chunking_strategy" is not
+    # -- already mentioned in the docstring.
+    if func.__doc__ and (
+        "chunking_strategy" in func.__code__.co_varnames and "chunking_strategy" not in func.__doc__
+    ):
+        func.__doc__ += (
+            "\nchunking_strategy"
+            + "\n\tStrategy used for chunking text into larger or smaller elements."
+            + "\n\tDefaults to `None` with optional arg of 'basic' or 'by_title'."
+            + "\n\tAdditional Parameters:"
+            + "\n\t\tmultipage_sections"
+            + "\n\t\t\tIf True, sections can span multiple pages. Defaults to True."
+            + "\n\t\tcombine_text_under_n_chars"
+            + "\n\t\t\tCombines elements (for example a series of titles) until a section"
+            + "\n\t\t\treaches a length of n characters. Only applies to 'by_title' strategy."
+            + "\n\t\tnew_after_n_chars"
+            + "\n\t\t\tCuts off chunks once they reach a length of n characters; a soft max."
+            + "\n\t\tmax_characters"
+            + "\n\t\t\tChunks elements text and text_as_html (if present) into chunks"
+            + "\n\t\t\tof length n characters, a hard max."
+            + "\n\t\trepeat_table_headers"
+            + "\n\t\t\tDefault: True. Repeat detected table headers on continuation"
+            + "\n\t\t\ttable chunks. Set to False to opt out."
+            + "\n\t\tskip_table_chunking"
+            + "\n\t\t\tDefault: False. When True, Table elements are passed through"
+            + "\n\t\t\tunchanged without being split into TableChunk elements."
+            + "\n\t\tisolate_table"
+            + "\n\t\t\tDefault: True. When True, Table/TableChunk elements are always"
+            + "\n\t\t\tstaged in their own pre-chunk. Set to False to allow tables to"
+            + "\n\t\t\tshare pre-chunks with adjacent non-table elements."
+        )
+
+    @functools.wraps(func)
+    def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> list[Element]:
+        """The decorated function is replaced with this one."""
+
+        # -- call the partitioning function to get the elements --
+        elements = func(*args, **kwargs)
+
+        # -- look for a chunking-strategy argument --
+        call_args = get_call_args_applying_defaults(func, *args, **kwargs)
+        chunking_strategy = call_args.pop("chunking_strategy", None)
+
+        # -- no chunking-strategy means no chunking --
+        if chunking_strategy is None:
+            return elements
+
+        # -- otherwise, chunk away :) --
+        return chunk(elements, chunking_strategy, **call_args)
+
+    return wrapper
+
+
+def chunk(elements: Iterable[Element], chunking_strategy: str, **kwargs: Any) -> list[Element]:
+    """Dispatch chunking of `elements` to the chunking function for `chunking_strategy`."""
+    chunker_spec = _chunker_registry.get(chunking_strategy)
+
+    if chunker_spec is None:
+        raise ValueError(f"unrecognized chunking strategy {repr(chunking_strategy)}")
+
+    # -- `kwargs` will in general be an omnibus dict of all keyword arguments to the partitioner;
+    # -- pick out and use only those supported by this chunker.
+    chunking_kwargs = {k: v for k, v in kwargs.items() if k in chunker_spec.kw_arg_names}
+
+    return chunker_spec.chunker(elements, **chunking_kwargs)
+
+
+def register_chunking_strategy(name: str, chunker: Chunker) -> None:
+    """Make chunker available by using `name` as `chunking_strategy` arg in partitioner call."""
+    _chunker_registry[name] = _ChunkerSpec(chunker)
+
+
+@dc.dataclass(frozen=True)
+class _ChunkerSpec:
+    """A registry entry for a chunker."""
+
+    chunker: Chunker
+    """The "chunk_by_{x}() function that implements this chunking strategy."""
+
+    @cached_property
+    def kw_arg_names(self) -> tuple[str, ...]:
+        """Keyword arguments supported by this chunker.
+
+        These are all arguments other than the required `elements: list[Element]` first parameter.
+        """
+        sig = inspect.signature(self.chunker)
+        return tuple(key for key in sig.parameters if key != "elements")
+
+
+_chunker_registry: dict[str, _ChunkerSpec] = {
+    "basic": _ChunkerSpec(chunk_elements),
+    "by_title": _ChunkerSpec(chunk_by_title),
+}
+
+
+def reconstruct_table_from_chunks(elements: Iterable[Element]) -> list[Table]:
+    """Reconstruct original tables from a mixed list of chunked elements.
+
+    Filters `TableChunk` elements, groups them by `table_id`, orders by `chunk_index`, and
+    merges each group into a single `Table` with combined text and HTML. Non-`TableChunk`
+    elements are ignored. Returns reconstructed tables in reading order (order of first chunk
+    appearance).
+    """
+    # -- filter to only TableChunk instances, preserving input order --
+    table_chunks = [e for e in elements if isinstance(e, TableChunk)]
+    if not table_chunks:
+        return []
+
+    # -- group by table_id, preserving first-seen order --
+    groups: dict[str, list[TableChunk]] = {}
+    for chunk in table_chunks:
+        tid = chunk.metadata.table_id
+        if tid is None:
+            continue
+        if tid not in groups:
+            groups[tid] = []
+        groups[tid].append(chunk)
+
+    # -- sort each group by chunk_index and merge --
+    tables: list[Table] = []
+
+    def _chunk_sort_key(chunk: TableChunk) -> tuple[bool, int]:
+        chunk_index = chunk.metadata.chunk_index
+        return (chunk_index is None, 0 if chunk_index is None else chunk_index)
+
+    for group in groups.values():
+        group.sort(key=_chunk_sort_key)
+        tables.append(_merge_table_chunks(group))
+
+    return tables
+
+
+def _merge_table_chunks(chunks: list[TableChunk]) -> Table:
+    """Merge an ordered list of TableChunks from the same table into a single Table."""
+    # -- combine text --
+    text = " ".join(
+        chunk_text for chunk in chunks if (chunk_text := _strip_carried_over_header_text(chunk))
+    )
+
+    # -- build metadata from first chunk --
+    metadata = copy.deepcopy(chunks[0].metadata)
+    metadata.is_continuation = None
+    metadata.table_id = None
+    metadata.chunk_index = None
+    metadata.num_carried_over_header_rows = None
+
+    # -- combine HTML if all chunks have it --
+    if all(c.metadata.text_as_html for c in chunks):
+        combined = fragment_fromstring("<table></table>")
+        canonical_header_row_count, canonical_header_rows = _first_carried_header_rows(chunks)
+        if canonical_header_rows:
+            thead = fragment_fromstring("<thead></thead>")
+            for row in canonical_header_rows:
+                thead.append(row)
+            combined.append(thead)
+
+        for c in chunks:
+            parsed = fragment_fromstring(c.metadata.text_as_html)
+            carried_over_header_rows = _num_carried_over_header_rows(c)
+            rows = parsed.xpath("./tr | ./thead/tr | ./tbody/tr | ./tfoot/tr")
+            skip_count = carried_over_header_rows
+            if c is chunks[0] and canonical_header_row_count:
+                skip_count = canonical_header_row_count
+            for row in rows[skip_count:]:
+                combined.append(row)
+        metadata.text_as_html = tostring(combined, encoding=str)
+    else:
+        metadata.text_as_html = None
+
+    return Table(text=text, metadata=metadata)
+
+
+def _num_carried_over_header_rows(chunk: TableChunk) -> int:
+    """Header rows prepended synthetically to this chunk.
+
+    Reconstruction can be called on user-provided/deserialized chunks, so treat missing values as
+    "no carried header rows."
+    """
+    value = chunk.metadata.num_carried_over_header_rows
+    return value or 0
+
+
+def _first_carried_header_rows(chunks: list[TableChunk]) -> tuple[int, list[Any]]:
+    """Header rows from first continuation chunk carrying repeated headers, if any."""
+    first_chunk_rows = _top_level_table_rows(chunks[0].metadata.text_as_html)
+    if first_chunk_rows is None:
+        return 0, []
+
+    for chunk in chunks:
+        carried_row_count = _num_carried_over_header_rows(chunk)
+        if carried_row_count <= 0:
+            continue
+
+        rows = _top_level_table_rows(chunk.metadata.text_as_html)
+        if rows is None:
+            continue
+
+        if carried_row_count > len(rows):
+            continue
+
+        carried_rows = rows[:carried_row_count]
+        if not _leading_row_texts_match(first_chunk_rows, carried_rows):
+            continue
+
+        return carried_row_count, [copy.deepcopy(row) for row in rows[:carried_row_count]]
+
+    return 0, []
+
+
+def _top_level_table_rows(text_as_html: str | None) -> list[Any] | None:
+    """Top-level rows from a table fragment, preserving section ordering."""
+    if not text_as_html:
+        return None
+
+    try:
+        parsed = fragment_fromstring(text_as_html)
+    except (ParserError, ValueError):
+        return None
+
+    return parsed.xpath("./tr | ./thead/tr | ./tbody/tr | ./tfoot/tr")
+
+
+def _leading_row_texts_match(first_chunk_rows: list[Any], carried_rows: list[Any]) -> bool:
+    """True when carried rows match first chunk's leading rows by normalized cell text."""
+    if len(first_chunk_rows) < len(carried_rows):
+        return False
+
+    for first_row, carried_row in zip(first_chunk_rows, carried_rows):
+        if _row_text_signature(first_row) != _row_text_signature(carried_row):
+            return False
+
+    return True
+
+
+def _row_text_signature(row: Any) -> tuple[str, ...]:
+    """Normalized cell text tuple for a row."""
+    return tuple(" ".join(cell.text_content().split()) for cell in row.iter("td", "th"))
+
+
+def _strip_carried_over_header_text(chunk: TableChunk) -> str:
+    """Strip synthetic carried-over header text from continuation chunk text."""
+    carried_row_count = _num_carried_over_header_rows(chunk)
+    if carried_row_count == 0:
+        return chunk.text
+
+    text_as_html = chunk.metadata.text_as_html
+    if not text_as_html:
+        return chunk.text
+
+    try:
+        parsed = fragment_fromstring(text_as_html)
+    except (ParserError, ValueError):
+        return chunk.text
+
+    rows = parsed.xpath("./tr | ./thead/tr | ./tbody/tr | ./tfoot/tr")
+    if carried_row_count > len(rows):
+        return chunk.text
+
+    carried_header_text = " ".join(
+        text
+        for row in rows[:carried_row_count]
+        for text in (" ".join(cell.text_content().split()) for cell in row.iter("td", "th"))
+        if text
+    )
+    if not carried_header_text:
+        return chunk.text
+
+    chunk_text = chunk.text.lstrip()
+    if chunk_text == carried_header_text:
+        return ""
+    if chunk_text.startswith(f"{carried_header_text} "):
+        return chunk_text[len(carried_header_text) + 1 :]
+    if chunk_text.startswith(carried_header_text):
+        return chunk_text[len(carried_header_text) :].lstrip()
+    return chunk.text

unstructured/chunking/title.py

@@ -0,0 +1,199 @@
+"""Implementation of chunking by title.
+
+Main entry point is the `@add_chunking_strategy()` decorator.
+"""
+
+from __future__ import annotations
+
+from functools import cached_property
+from typing import Iterable, Iterator, Optional
+
+from unstructured.chunking.base import (
+    CHUNK_MULTI_PAGE_DEFAULT,
+    BoundaryPredicate,
+    ChunkingOptions,
+    PreChunkCombiner,
+    PreChunker,
+    is_on_next_page,
+    is_title,
+)
+from unstructured.documents.elements import Element
+
+
+def chunk_by_title(
+    elements: Iterable[Element],
+    *,
+    combine_text_under_n_chars: Optional[int] = None,
+    include_orig_elements: Optional[bool] = None,
+    max_characters: Optional[int] = None,
+    max_tokens: Optional[int] = None,
+    multipage_sections: Optional[bool] = None,
+    new_after_n_chars: Optional[int] = None,
+    new_after_n_tokens: Optional[int] = None,
+    overlap: Optional[int] = None,
+    overlap_all: Optional[bool] = None,
+    tokenizer: Optional[str] = None,
+    repeat_table_headers: Optional[bool] = None,
+    skip_table_chunking: Optional[bool] = None,
+    isolate_table: Optional[bool] = None,
+) -> list[Element]:
+    """Uses title elements to identify sections within the document for chunking.
+
+    Splits off into a new CompositeElement when a title is detected or if metadata changes, which
+    happens when page numbers or sections change. Cuts off sections once they have exceeded a
+    character length of max_characters (or token count of max_tokens).
+
+    Parameters
+    ----------
+    elements
+        A list of unstructured elements. Usually the output of a partition function.
+    combine_text_under_n_chars
+        Combines elements (for example a series of titles) until a section reaches a length of
+        n characters. Defaults to `max_characters` which combines chunks whenever space allows.
+        Specifying 0 for this argument suppresses combining of small chunks. Note this value is
+        "capped" at the `new_after_n_chars` value since a value higher than that would not change
+        this parameter's effect.
+    include_orig_elements
+        When `True` (default), add elements from pre-chunk to the `.metadata.orig_elements` field
+        of the chunk(s) formed from that pre-chunk. Among other things, this allows access to
+        original-element metadata that cannot be consolidated and is dropped in the course of
+        chunking.
+    max_characters
+        Chunks elements text and text_as_html (if present) into chunks of length
+        n characters (hard max). Mutually exclusive with `max_tokens`.
+    max_tokens
+        Chunks elements into chunks of n tokens (hard max). Requires `tokenizer` to be specified.
+        Mutually exclusive with `max_characters`.
+    multipage_sections
+        If True, sections can span multiple pages. Defaults to True.
+    new_after_n_chars
+        Cuts off new sections once they reach a length of n characters (soft max). Defaults to
+        `max_characters` when not specified, which effectively disables any soft window.
+        Specifying 0 for this argument causes each element to appear in a chunk by itself (although
+        an element with text longer than `max_characters` will be still be split into two or more
+        chunks).
+    new_after_n_tokens
+        Token-based equivalent of `new_after_n_chars`. Cuts off new sections once they reach
+        n tokens (soft max). Requires `max_tokens` and `tokenizer` to be specified.
+    overlap
+        Specifies the length of a string ("tail") to be drawn from each chunk and prefixed to the
+        next chunk as a context-preserving mechanism. By default, this only applies to split-chunks
+        where an oversized element is divided into multiple chunks by text-splitting.
+    overlap_all
+        Default: `False`. When `True`, apply overlap between "normal" chunks formed from whole
+        elements and not subject to text-splitting. Use this with caution as it entails a certain
+        level of "pollution" of otherwise clean semantic chunk boundaries.
+    tokenizer
+        The tokenizer to use for token-based chunking. Can be either an encoding name (e.g.,
+        "cl100k_base") or a model name (e.g., "gpt-4"). Required when using `max_tokens`.
+    repeat_table_headers
+        Default: `True`. When `True`, repeated table-header behavior is enabled for chunked table
+        continuations. Specify `False` to opt out and preserve legacy table-chunk behavior.
+    skip_table_chunking
+        Default: `False`. When `True`, `Table` elements are passed through unchanged without
+        being split into `TableChunk` elements, regardless of their size.
+    isolate_table
+        Default: `True`. When `True`, `Table` and `TableChunk` elements are always staged in
+        their own pre-chunk and never combined with adjacent non-table elements. Specify
+        `False` to allow tables to share pre-chunks with adjacent elements (the pre-#4307
+        behavior).
+    """
+    opts = _ByTitleChunkingOptions.new(
+        combine_text_under_n_chars=combine_text_under_n_chars,
+        include_orig_elements=include_orig_elements,
+        max_characters=max_characters,
+        max_tokens=max_tokens,
+        multipage_sections=multipage_sections,
+        new_after_n_chars=new_after_n_chars,
+        new_after_n_tokens=new_after_n_tokens,
+        overlap=overlap,
+        overlap_all=overlap_all,
+        tokenizer=tokenizer,
+        repeat_table_headers=repeat_table_headers,
+        skip_table_chunking=skip_table_chunking,
+        isolate_table=isolate_table,
+    )
+    return _chunk_by_title(elements, opts)
+
+
+def _chunk_by_title(elements: Iterable[Element], opts: _ByTitleChunkingOptions) -> list[Element]:
+    """Implementation of actual "by-title" chunking."""
+    # -- Note(scanny): it might seem like over-abstraction for this to be a separate function but
+    # -- it eases overriding or adding individual chunking options when customizing a stock chunker.
+    pre_chunks = PreChunkCombiner(
+        PreChunker.iter_pre_chunks(elements, opts), opts=opts
+    ).iter_combined_pre_chunks()
+
+    return [chunk for pre_chunk in pre_chunks for chunk in pre_chunk.iter_chunks()]
+
+
+class _ByTitleChunkingOptions(ChunkingOptions):
+    """Adds the by-title-specific chunking options to the base case.
+
+    `by_title`-specific options:
+
+    combine_text_under_n_chars
+        A remedy to over-chunking caused by elements mis-identified as Title elements.
+        Every Title element would start a new chunk and this setting mitigates that, at the
+        expense of sometimes violating legitimate semantic boundaries.
+    multipage_sections
+        Indicates that page-boundaries should not be respected while chunking, i.e. elements
+        appearing on two different pages can appear in the same chunk.
+    """
+
+    @cached_property
+    def boundary_predicates(self) -> tuple[BoundaryPredicate, ...]:
+        """The semantic-boundary detectors to be applied to break pre-chunks.
+
+        For the `by_title` strategy these are sections indicated by a title (section-heading), an
+        explicit section metadata item (only present for certain document types), and optionally
+        page boundaries.
+        """
+
+        def iter_boundary_predicates() -> Iterator[BoundaryPredicate]:
+            yield is_title
+            if not self.multipage_sections:
+                yield is_on_next_page()
+
+        return tuple(iter_boundary_predicates())
+
+    @cached_property
+    def combine_text_under_n_chars(self) -> int:
+        """Combine consecutive text pre-chunks if former is smaller than this and both will fit.
+
+        - Does not combine text chunks if together they would exceed the chunking window.
+        - Defaults to `max_characters` when not specified.
+        - Is reduced to `new_after_n_chars` when it exceeds that value.
+        """
+        # -- `combine_text_under_n_chars` defaults to `max_characters` when not specified --
+        arg_value = self._kwargs.get("combine_text_under_n_chars")
+        return self.hard_max if arg_value is None else arg_value
+
+    @cached_property
+    def multipage_sections(self) -> bool:
+        """When False, break pre-chunks on page-boundaries."""
+        arg_value = self._kwargs.get("multipage_sections")
+        return CHUNK_MULTI_PAGE_DEFAULT if arg_value is None else bool(arg_value)
+
+    def _validate(self) -> None:
+        """Raise ValueError if request option-set is invalid."""
+        # -- start with base-class validations --
+        super()._validate()
+
+        # -- `combine_text_under_n_chars == 0` is valid (suppresses chunk combination)
+        # -- but a negative value is not
+        if self.combine_text_under_n_chars < 0:
+            raise ValueError(
+                f"'combine_text_under_n_chars' argument must be >= 0,"
+                f" got {self.combine_text_under_n_chars}"
+            )
+
+        # -- `combine_text_under_n_chars` > `max_characters` can produce behavior confusing to
+        # -- users. The chunking behavior would be no different than when
+        # -- `combine_text_under_n_chars == max_characters`, but if `max_characters` is left to
+        # -- default (500) then it can look like chunk-combining isn't working.
+        if self.combine_text_under_n_chars > self.hard_max:
+            raise ValueError(
+                f"'combine_text_under_n_chars' argument must not exceed `max_characters`"
+                f" value, got {self.combine_text_under_n_chars} > {self.hard_max}"
+            )

unstructured/cleaners/core.py

@@ -0,0 +1,490 @@
+from __future__ import annotations
+
+import quopri
+import re
+import sys
+import unicodedata
+from typing import Optional, Tuple
+
+import numpy as np
+
+from unstructured.file_utils.encoding import (
+    format_encoding_str,
+)
+from unstructured.nlp.patterns import (
+    DOUBLE_PARAGRAPH_PATTERN_RE,
+    E_BULLET_PATTERN,
+    LINE_BREAK_RE,
+    PARAGRAPH_PATTERN,
+    PARAGRAPH_PATTERN_RE,
+    UNICODE_BULLETS_RE,
+    UNICODE_BULLETS_RE_0W,
+)
+
+
+def clean_non_ascii_chars(text) -> str:
+    """Cleans non-ascii characters from unicode string.
+
+    Example
+    -------
+    \x88This text contains non-ascii characters!\x88
+        -> This text contains non-ascii characters!
+    """
+    en = text.encode("ascii", "ignore")
+    return en.decode()
+
+
+def clean_bullets(text: str) -> str:
+    """Cleans unicode bullets from a section of text.
+
+    Example
+    -------
+    ●  This is an excellent point! -> This is an excellent point!
+    """
+    search = UNICODE_BULLETS_RE.match(text)
+    if search is None:
+        return text
+
+    cleaned_text = UNICODE_BULLETS_RE.sub("", text, 1)
+    return cleaned_text.strip()
+
+
+def clean_ordered_bullets(text) -> str:
+    """Cleans the start of bulleted text sections up to three “sub-section”
+    bullets accounting numeric and alphanumeric types.
+
+    Example
+    -------
+    1.1 This is a very important point -> This is a very important point
+    a.b This is a very important point -> This is a very important point
+    """
+    text_sp = text.split()
+    text_cl = " ".join(text_sp[1:])
+    if any(["." not in text_sp[0], ".." in text_sp[0]]):
+        return text
+
+    bullet = re.split(pattern=r"[\.]", string=text_sp[0])
+    if not bullet[-1]:
+        del bullet[-1]
+
+    if len(bullet[0]) > 2:
+        return text
+
+    return text_cl
+
+
+def clean_ligatures(text) -> str:
+    """Replaces ligatures with their most likely equivalent characters.
+
+    Example
+    -------
+    The benefits -> The benefits
+    High quality financial -> High quality financial
+    """
+    ligatures_map = {
+        "æ": "ae",
+        "Æ": "AE",
+        "ff": "ff",
+        "fi": "fi",
+        "fl": "fl",
+        "ffi": "ffi",
+        "ffl": "ffl",
+        "ſt": "ft",
+        "ʪ": "ls",
+        "œ": "oe",
+        "Œ": "OE",
+        "ȹ": "qp",
+        "st": "st",
+        "ʦ": "ts",
+    }
+    cleaned_text: str = text
+    for k, v in ligatures_map.items():
+        cleaned_text = cleaned_text.replace(k, v)
+
+    return cleaned_text
+
+
+def group_bullet_paragraph(paragraph: str) -> list:
+    """Groups paragraphs with bullets that have line breaks for visual/formatting purposes.
+    For example:
+
+    '''○ The big red fox
+    is walking down the lane.
+
+    ○ At the end of the lane
+    the fox met a friendly bear.'''
+
+    Gets converted to
+
+    '''○ The big red fox is walking down the lane.
+    ○ At the end of the land the fox met a bear.'''
+    """
+    paragraph_pattern_re = re.compile(PARAGRAPH_PATTERN)
+
+    # pytesseract converts some bullet points to standalone "e" characters.
+    # Substitute "e" with bullets since they are later used in partition_text
+    # to determine list element type.
+    paragraph = E_BULLET_PATTERN.sub("·", paragraph).strip()
+
+    bullet_paras = UNICODE_BULLETS_RE_0W.split(paragraph)
+    clean_paragraphs = []
+    for bullet in bullet_paras:
+        if bullet:
+            clean_paragraphs.append(paragraph_pattern_re.sub(" ", bullet))
+    return clean_paragraphs
+
+
+def group_broken_paragraphs(
+    text: str,
+    line_split: re.Pattern[str] = PARAGRAPH_PATTERN_RE,
+    paragraph_split: re.Pattern[str] = DOUBLE_PARAGRAPH_PATTERN_RE,
+) -> str:
+    """Groups paragraphs that have line breaks for visual/formatting purposes.
+    For example:
+
+    '''The big red fox
+    is walking down the lane.
+
+    At the end of the lane
+    the fox met a bear.'''
+
+    Gets converted to
+
+    '''The big red fox is walking down the lane.
+    At the end of the land the fox met a bear.'''
+    """
+    paragraph_pattern_re = (
+        PARAGRAPH_PATTERN
+        if isinstance(PARAGRAPH_PATTERN, re.Pattern)
+        else re.compile(PARAGRAPH_PATTERN)
+    )
+
+    paragraphs = paragraph_split.split(text)
+    clean_paragraphs = []
+    for paragraph in paragraphs:
+        stripped_par = paragraph.strip()
+        if not stripped_par:
+            continue
+
+        if UNICODE_BULLETS_RE.match(stripped_par) or E_BULLET_PATTERN.match(stripped_par):
+            clean_paragraphs.extend(group_bullet_paragraph(paragraph))
+            continue
+        # NOTE(robinson) - This block is to account for lines like the following that shouldn't be
+        # grouped together, but aren't separated by a double line break.
+        #     Apache License
+        #     Version 2.0, January 2004
+        #     http://www.apache.org/licenses/
+        para_split = line_split.split(paragraph)
+        all_lines_short = all(len(line.strip().split(" ")) < 5 for line in para_split)
+        if all_lines_short:
+            clean_paragraphs.extend(line for line in para_split if line.strip())
+        else:
+            clean_paragraphs.append(paragraph_pattern_re.sub(" ", paragraph))
+
+    return "\n\n".join(clean_paragraphs)
+
+
+def new_line_grouper(
+    text: str,
+    paragraph_split: re.Pattern[str] = LINE_BREAK_RE,
+) -> str:
+    """
+    Concatenates text document that has one-line paragraph break pattern
+
+    For example,
+
+    Iwan Roberts
+    Roberts celebrating after scoring a goal for Norwich City
+    in 2004
+
+    Will be returned as:
+
+    Iwan Roberts\n\nRoberts celebrating after scoring a goal for Norwich City\n\nin 2004
+    """
+    paragraphs = paragraph_split.split(text)
+    clean_paragraphs = []
+    for paragraph in paragraphs:
+        if not paragraph.strip():
+            continue
+        clean_paragraphs.append(paragraph)
+    return "\n\n".join(clean_paragraphs)
+
+
+def blank_line_grouper(
+    text: str,
+    paragraph_split: re.Pattern = DOUBLE_PARAGRAPH_PATTERN_RE,
+) -> str:
+    """
+    Concatenates text document that has blank-line paragraph break pattern
+
+    For example,
+
+    Vestibulum auctor dapibus neque.
+
+    Nunc dignissim risus id metus.
+
+    Will be returned as:
+
+    Vestibulum auctor dapibus neque.\n\nNunc dignissim risus id metus.\n\n
+
+    """
+    return group_broken_paragraphs(text)
+
+
+def auto_paragraph_grouper(
+    text: str,
+    line_split: re.Pattern[str] = LINE_BREAK_RE,
+    max_line_count: int = 2000,
+    threshold: float = 0.1,
+) -> str:
+    """
+    Checks the ratio of new line (\n) over the total max_line_count
+
+    If the ratio of new line is less than the threshold,
+    the document is considered a new-line grouping type
+    and return the original text
+
+    If the ratio of new line is greater than or equal to the threshold,
+    the document is considered a blank-line grouping type
+    and passed on to blank_line_grouper function
+    """
+    lines = line_split.split(text)
+    max_line_count = min(len(lines), max_line_count)
+    line_count, empty_line_count = 0, 0
+    for line in lines[:max_line_count]:
+        line_count += 1
+        if not line.strip():
+            empty_line_count += 1
+    ratio = empty_line_count / line_count
+
+    # NOTE(klaijan) - for ratio < threshold, we pass to new-line grouper,
+    # otherwise to blank-line grouper
+    if ratio < threshold:
+        return new_line_grouper(text)
+    else:
+        return blank_line_grouper(text)
+
+
+# TODO(robinson) - There's likely a cleaner was to accomplish this and get all of the
+# unicode characters instead of just the quotes. Doing this for now since quotes are
+# an issue that are popping up in the SEC filings tests
+def replace_unicode_quotes(text: str) -> str:
+    """Replaces unicode bullets in text with the expected character
+
+    Example
+    -------
+    \x93What a lovely quote!\x94 -> “What a lovely quote!”
+    """
+    # NOTE(robinson) - We should probably make this something more sane like a regex
+    # instead of a whole big series of replaces
+    text = text.replace("\x91", "‘")
+    text = text.replace("\x92", "’")
+    text = text.replace("\x93", "“")
+    text = text.replace("\x94", "”")
+    text = text.replace("&apos;", "'")
+    text = text.replace("â\x80\x99", "'")
+    text = text.replace("â\x80“", "—")
+    text = text.replace("â\x80”", "–")
+    text = text.replace("â\x80˜", "‘")
+    text = text.replace("â\x80¦", "…")
+    text = text.replace("â\x80™", "’")
+    text = text.replace("â\x80œ", "“")
+    text = text.replace("â\x80?", "”")
+    text = text.replace("â\x80ť", "”")
+    text = text.replace("â\x80ś", "“")
+    text = text.replace("â\x80¨", "—")
+    text = text.replace("â\x80ł", "″")
+    text = text.replace("â\x80Ž", "")
+    text = text.replace("â\x80‚", "")
+    text = text.replace("â\x80‰", "")
+    text = text.replace("â\x80‹", "")
+    text = text.replace("â\x80", "")
+    text = text.replace("â\x80s'", "")
+    return text
+
+
+tbl = dict.fromkeys(
+    i for i in range(sys.maxunicode) if unicodedata.category(chr(i)).startswith("P")
+)
+
+
+def remove_punctuation(s: str) -> str:
+    """Removes punctuation from a given string."""
+    return s.translate(tbl)
+
+
+def remove_sentence_punctuation(s: str, exclude_punctuation: Optional[list]) -> str:
+    tbl_new = tbl.copy()
+    if exclude_punctuation:
+        for punct in exclude_punctuation:
+            del tbl_new[ord(punct)]
+    s = s.translate(tbl_new)
+    return s
+
+
+def clean_extra_whitespace(text: str) -> str:
+    """Cleans extra whitespace characters that appear between words.
+
+    Example
+    -------
+    ITEM 1.     BUSINESS -> ITEM 1. BUSINESS
+    """
+    cleaned_text = re.sub(r"[\xa0\n]", " ", text)
+    cleaned_text = re.sub(r"([ ]{2,})", " ", cleaned_text)
+    return cleaned_text.strip()
+
+
+def clean_dashes(text: str) -> str:
+    """Cleans dash characters in text.
+
+    Example
+    -------
+    ITEM 1. -BUSINESS -> ITEM 1.  BUSINESS
+    """
+    # NOTE(Yuming): '\u2013' is the unicode string of 'EN DASH', a variation of "-"
+    return re.sub(r"[-\u2013]", " ", text).strip()
+
+
+def clean_trailing_punctuation(text: str) -> str:
+    """Clean all trailing punctuation in text
+
+    Example
+    -------
+    ITEM 1.     BUSINESS. -> ITEM 1.     BUSINESS
+    """
+    return text.strip().rstrip(".,:;")
+
+
+def replace_mime_encodings(text: str, encoding: str = "utf-8") -> str:
+    """Replaces MIME encodings with their equivalent characters in the specified encoding.
+
+    Example
+    -------
+    5 w=E2=80-99s -> 5 w’s
+    """
+    formatted_encoding = format_encoding_str(encoding)
+    return quopri.decodestring(text.encode(formatted_encoding)).decode(formatted_encoding)
+
+
+def clean_prefix(text: str, pattern: str, ignore_case: bool = False, strip: bool = True) -> str:
+    """Removes prefixes from a string according to the specified pattern. Strips leading
+    whitespace if the strip parameter is set to True.
+
+    Input
+    -----
+    text: The text to clean
+    pattern: The pattern for the prefix. Can be a simple string or a regex pattern
+    ignore_case: If True, ignores case in the pattern
+    strip: If True, removes leading whitespace from the cleaned string.
+    """
+    flags = re.IGNORECASE if ignore_case else 0
+    clean_text = re.sub(rf"^{pattern}", "", text, flags=flags)
+    clean_text = clean_text.lstrip() if strip else clean_text
+    return clean_text
+
+
+def clean_postfix(text: str, pattern: str, ignore_case: bool = False, strip: bool = True) -> str:
+    """Removes postfixes from a string according to the specified pattern. Strips trailing
+    whitespace if the strip parameters is set to True.
+
+    Input
+    -----
+    text: The text to clean
+    pattern: The pattern for the postfix. Can be a simple string or a regex pattern
+    ignore_case: If True, ignores case in the pattern
+    strip: If True, removes trailing whitespace from the cleaned string.
+    """
+    flags = re.IGNORECASE if ignore_case else 0
+    clean_text = re.sub(rf"{pattern}$", "", text, flags=flags)
+    clean_text = clean_text.rstrip() if strip else clean_text
+    return clean_text
+
+
+def clean(
+    text: str,
+    extra_whitespace: bool = False,
+    dashes: bool = False,
+    bullets: bool = False,
+    trailing_punctuation: bool = False,
+    lowercase: bool = False,
+) -> str:
+    """Cleans text.
+
+    Input
+    -----
+    extra_whitespace: Whether to clean extra whitespace characters in text.
+    dashes: Whether to clean dash characters in text.
+    bullets: Whether to clean unicode bullets from a section of text.
+    trailing_punctuation: Whether to clean all trailing punctuation in text.
+    lowercase: Whether to return lowercase text.
+    """
+
+    cleaned_text = text.lower() if lowercase else text
+    cleaned_text = (
+        clean_trailing_punctuation(cleaned_text) if trailing_punctuation else cleaned_text
+    )
+    cleaned_text = clean_dashes(cleaned_text) if dashes else cleaned_text
+    cleaned_text = clean_extra_whitespace(cleaned_text) if extra_whitespace else cleaned_text
+    cleaned_text = clean_bullets(cleaned_text) if bullets else cleaned_text
+    return cleaned_text.strip()
+
+
+def bytes_string_to_string(text: str, encoding: str = "utf-8"):
+    """Converts a string representation of a byte string to a regular string using the
+    specified encoding."""
+    text_bytes = bytes([ord(char) for char in text])
+    formatted_encoding = format_encoding_str(encoding)
+    return text_bytes.decode(formatted_encoding)
+
+
+def clean_extra_whitespace_with_index_run(text: str) -> Tuple[str, np.ndarray]:
+    """Cleans extra whitespace characters that appear between words.
+    Calculate distance between characters of original text and cleaned text.
+
+    Returns cleaned text along with array of indices it has moved from original.
+
+    Example
+    -------
+    ITEM 1.     BUSINESS -> ITEM 1. BUSINESS
+    array([0., 0., 0., 0., 0., 0., 0., 0., 4., 4., 4., 4., 4., 4., 4., 4., 4., 4., 4., 4.]))
+    """
+
+    # Replace non-breaking space and newlines with a space (using translation table for speed)
+    translate_table = {ord("\xa0"): ord(" "), ord("\n"): ord(" ")}
+    cleaned_text = text.translate(translate_table)
+    # Collapse multiple spaces into one (keeps only single runs)
+    cleaned_text = re.sub(r"([ ]{2,})", " ", cleaned_text)
+
+    cleaned_text = cleaned_text.strip()
+
+    moved_indices = np.zeros(len(text))
+
+    cleaned_len = len(cleaned_text)
+
+    ws_chars = {"\xa0", "\n"}  # For a quick lookup
+
+    distance = 0
+    original_index = 0
+    cleaned_index = 0
+
+    while cleaned_index < cleaned_len:
+        c_orig = text[original_index]
+        c_clean = cleaned_text[cleaned_index]
+
+        if c_orig == c_clean or (c_orig in ws_chars and c_clean == " "):
+            moved_indices[cleaned_index] = distance
+            original_index += 1
+            cleaned_index += 1
+            continue
+
+        distance += 1
+        moved_indices[cleaned_index] = distance
+        original_index += 1
+
+    moved_indices[cleaned_index:] = distance
+
+    return cleaned_text, moved_indices
+
+
+def index_adjustment_after_clean_extra_whitespace(index, moved_indices) -> int:
+    return int(index - moved_indices[index])

unstructured/cleaners/extract.py

@@ -0,0 +1,143 @@
+import datetime
+import re
+from typing import List, Optional
+
+from unstructured.nlp.patterns import (
+    EMAIL_ADDRESS_PATTERN,
+    EMAIL_DATETIMETZ_PATTERN,
+    IMAGE_URL_PATTERN,
+    IP_ADDRESS_NAME_PATTERN,
+    IP_ADDRESS_PATTERN_RE,
+    MAPI_ID_PATTERN,
+    US_PHONE_NUMBERS_RE,
+)
+
+
+def _get_indexed_match(text: str, pattern: str, index: int = 0) -> re.Match:
+    if not isinstance(index, int) or index < 0:
+        raise ValueError(f"The index is {index}. Index must be a non-negative integer.")
+
+    regex_match = None
+    for i, result in enumerate(re.finditer(pattern, text)):
+        if i == index:
+            regex_match = result
+
+    if regex_match is None:
+        raise ValueError(f"Result with index {index} was not found. The largest index was {i}.")
+
+    return regex_match
+
+
+def extract_text_before(text: str, pattern: str, index: int = 0, strip: bool = True) -> str:
+    """Extracts texts that occurs before the specified pattern. By default, it will use
+    the first occurrence of the pattern (index 0). Use the index kwarg to choose a different
+    index.
+
+    Input
+    -----
+    strip: If True, removes trailing whitespace from the extracted string
+    """
+    regex_match = _get_indexed_match(text, pattern, index)
+    start, _ = regex_match.span()
+    before_text = text[:start]
+    return before_text.rstrip() if strip else before_text
+
+
+def extract_text_after(text: str, pattern: str, index: int = 0, strip: bool = True) -> str:
+    """Extracts texts that occurs before the specified pattern. By default, it will use
+    the first occurrence of the pattern (index 0). Use the index kwarg to choose a different
+    index.
+
+    Input
+    -----
+    strip: If True, removes leading whitespace from the extracted string
+    """
+    regex_match = _get_indexed_match(text, pattern, index)
+    _, end = regex_match.span()
+    before_text = text[end:]
+    return before_text.lstrip() if strip else before_text
+
+
+def extract_email_address(text: str) -> List[str]:
+    return re.findall(EMAIL_ADDRESS_PATTERN, text.lower())
+
+
+def extract_ip_address(text: str) -> List[str]:
+    return re.findall(IP_ADDRESS_PATTERN_RE, text)
+
+
+def extract_ip_address_name(text: str) -> List[str]:
+    return re.findall(IP_ADDRESS_NAME_PATTERN, text)
+
+
+def extract_mapi_id(text: str) -> List[str]:
+    mapi_ids = re.findall(MAPI_ID_PATTERN, text)
+    mapi_ids = [mid.replace(";", "") for mid in mapi_ids]
+    return mapi_ids
+
+
+def extract_datetimetz(text: str) -> Optional[datetime.datetime]:
+    date_extractions = re.findall(EMAIL_DATETIMETZ_PATTERN, text)
+    if len(date_extractions) > 0:
+        return datetime.datetime.strptime(date_extractions[0], "%a, %d %b %Y %H:%M:%S %z")
+    else:
+        return None
+
+
+def extract_us_phone_number(text: str):
+    """Extracts a US phone number from a section of text that includes a phone number. If there
+    is no phone number present, the result will be an empty string.
+
+    Example
+    -------
+    extract_phone_number("Phone Number: 215-867-5309") -> "215-867-5309"
+    """
+    regex_match = US_PHONE_NUMBERS_RE.search(text)
+    if regex_match is None:
+        return ""
+
+    start, end = regex_match.span()
+    phone_number = text[start:end]
+    return phone_number.strip()
+
+
+def extract_ordered_bullets(text) -> tuple:
+    """Extracts the start of bulleted text sections bullets
+    accounting numeric and alphanumeric types.
+
+    Output
+    -----
+    tuple(section, sub_section, sub_sub_section): Each bullet partition
+    is a string or None if not present.
+
+    Example
+    -------
+    This is a very important point -> (None, None, None)
+    1.1 This is a very important point -> ("1", "1", None)
+    a.1 This is a very important point -> ("a", "1", None)
+    """
+    a, b, c, temp = None, None, None, None
+    text_sp = text.split()
+    if any(["." not in text_sp[0], ".." in text_sp[0]]):
+        return a, b, c
+
+    bullet = re.split(pattern=r"[\.]", string=text_sp[0])
+    if not bullet[-1]:
+        del bullet[-1]
+
+    if len(bullet[0]) > 2:
+        return a, b, c
+
+    a, *temp = bullet
+    if temp:
+        try:
+            b, c, *_ = temp
+        except ValueError:
+            b = temp
+        b = "".join(b)
+        c = "".join(c) if c else None
+    return a, b, c
+
+
+def extract_image_urls_from_html(text: str) -> List[str]:
+    return re.findall(IMAGE_URL_PATTERN, text)

unstructured/cleaners/translate.py

@@ -0,0 +1,87 @@
+import warnings
+from typing import List, Optional
+
+import langdetect
+from transformers import MarianMTModel, MarianTokenizer
+
+from unstructured.nlp.tokenize import sent_tokenize
+from unstructured.staging.huggingface import chunk_by_attention_window
+
+
+def _get_opus_mt_model_name(source_lang: str, target_lang: str):
+    """Constructs the name of the MarianMT machine translation model based on the
+    source and target language."""
+    return f"Helsinki-NLP/opus-mt-{source_lang}-{target_lang}"
+
+
+def _validate_language_code(language_code: str):
+    if not isinstance(language_code, str) or len(language_code) != 2:
+        raise ValueError(
+            f"Invalid language code: {language_code}. Language codes must be two letter strings.",
+        )
+
+
+def translate_text(text: str, source_lang: Optional[str] = None, target_lang: str = "en") -> str:
+    """Translates the foreign language text. If the source language is not specified, the
+    function will attempt to detect it using langdetect.
+
+    Parameters
+    ----------
+    text: str
+        The text to translate
+    target_lang: str
+        The two letter language code for the target langague. Defaults to "en".
+    source_lang: Optional[str]
+        The two letter language code for the language of the input text. If source_lang is
+        not provided, the function will try to detect it.
+    """
+    if text.strip() == "":
+        return text
+
+    _source_lang: str = source_lang if source_lang is not None else langdetect.detect(text)
+    # NOTE(robinson) - Chinese gets detected with codes zh-cn, zh-tw, zh-hk for various
+    # Chinese variants. We normalizes these because there is a single model for Chinese
+    # machine translation
+    if _source_lang.startswith("zh"):
+        _source_lang = "zh"
+
+    _validate_language_code(target_lang)
+    _validate_language_code(_source_lang)
+
+    if target_lang == _source_lang:
+        return text
+
+    model_name = _get_opus_mt_model_name(_source_lang, target_lang)
+    print(f"Using model: {model_name}")
+
+    try:
+        tokenizer = MarianTokenizer.from_pretrained(model_name)
+        model = MarianMTModel.from_pretrained(model_name)
+    except OSError:
+        raise ValueError(
+            f"Transformers could not find the translation model {model_name}. "
+            "The requested source/target language combo is not supported.",
+        )
+
+    chunks: List[str] = chunk_by_attention_window(text, tokenizer, split_function=sent_tokenize)
+
+    translated_chunks: List[str] = []
+    for chunk in chunks:
+        translated_chunks.append(_translate_text(text, model, tokenizer))
+
+    return " ".join(translated_chunks)
+
+
+def _translate_text(text, model, tokenizer):
+    """Translates text using the specified model and tokenizer."""
+    # NOTE(robinson) - Suppresses the HuggingFace UserWarning resulting from the "max_length"
+    # key in the MarianMT config. The warning states that "max_length" will be deprecated
+    # in transformers v5
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        translated = model.generate(
+            **tokenizer([text], return_tensors="pt", padding=True, truncation=True),
+        )
+    return [tokenizer.decode(t, max_new_tokens=512, skip_special_tokens=True) for t in translated][
+        0
+    ]

unstructured/cli.py

@@ -0,0 +1,87 @@
+"""Command-line interface for :mod:`unstructured`."""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+
+def _cmd_doctor(argv: list[str] | None = None) -> int:
+    from unstructured.doctor import build_report, evaluate_specifier, file_path_to_capability
+
+    parser = argparse.ArgumentParser(
+        prog="unstructured doctor",
+        description="Print dependency and capability diagnostics for partitioning.",
+    )
+    parser.add_argument(
+        "--for",
+        dest="for_cap",
+        metavar="TYPE",
+        help="Check readiness for a file type or family (e.g. pdf, docx, image, audio).",
+    )
+    parser.add_argument(
+        "--file",
+        dest="file_path",
+        metavar="PATH",
+        help="Infer file type from PATH and check readiness for that type.",
+    )
+    ns = parser.parse_args(argv)
+
+    if ns.for_cap and ns.file_path:
+        print("Use either --for or --file, not both.", file=sys.stderr)
+        return 2
+
+    if ns.for_cap:
+        try:
+            result = evaluate_specifier(ns.for_cap)
+        except ValueError as e:
+            print(str(e), file=sys.stderr)
+            return 2
+        if result.messages:
+            print("\n".join(result.messages))
+        if not result.ready:
+            return 1
+        return 0
+
+    if ns.file_path:
+        result = file_path_to_capability(ns.file_path)
+        if result.messages:
+            print("\n".join(result.messages))
+        if not result.ready:
+            return 1
+        return 0
+
+    print(build_report(), end="")
+    return 0
+
+
+def main(argv: list[str] | None = None) -> int:
+    """Entry point for the ``unstructured`` console script."""
+    argv = list(sys.argv[1:] if argv is None else argv)
+
+    if not argv or argv[0] in ("-h", "--help"):
+        parser = argparse.ArgumentParser(
+            prog="unstructured",
+            description="Unstructured document processing utilities.",
+        )
+        parser.add_argument(
+            "command",
+            nargs="?",
+            choices=["doctor"],
+            help="Subcommand to run.",
+        )
+        parser.print_help()
+        return 0
+
+    if argv[0] == "doctor":
+        return _cmd_doctor(argv[1:])
+
+    print(
+        f"unstructured: unknown command {argv[0]!r}. Try `unstructured --help`.",
+        file=sys.stderr,
+    )
+    return 2
+
+
+if __name__ == "__main__":  # pragma: no cover
+    raise SystemExit(main())

unstructured/common/html_table.py

@@ -0,0 +1,207 @@
+"""Provides operations related to the HTML table stored in `.metadata.text_as_html`.
+
+Used during partitioning as well as chunking.
+"""
+
+from __future__ import annotations
+
+import html
+from functools import cached_property
+from typing import TYPE_CHECKING, Iterator, Sequence, cast
+
+from lxml import etree
+from lxml.html import fragment_fromstring
+
+if TYPE_CHECKING:
+    from lxml.html import HtmlElement
+
+
+def htmlify_matrix_of_cell_texts(matrix: Sequence[Sequence[str]]) -> str:
+    """Form an HTML table from "rows" and "columns" of `matrix`.
+
+    Character overhead is minimized:
+    - No whitespace padding is added for human readability
+    - No newlines ("\n") are added
+    - No `<thead>`, `<tbody>`, or `<tfoot>` elements are used; we can't tell where those might be
+      semantically appropriate anyway so at best they would consume unnecessary space and at worst
+      would be misleading.
+    """
+
+    def iter_trs(rows_of_cell_strs: Sequence[Sequence[str]]) -> Iterator[str]:
+        for row_cell_strs in rows_of_cell_strs:
+            # -- suppress emission of rows with no cells --
+            if not row_cell_strs:
+                continue
+            yield f"<tr>{''.join(iter_tds(row_cell_strs))}</tr>"
+
+    def iter_tds(row_cell_strs: Sequence[str]) -> Iterator[str]:
+        for s in row_cell_strs:
+            # -- take care of things like '<' and '>' in the text --
+            s = html.escape(s)
+            # -- substitute <br/> elements for line-feeds in the text --
+            s = "<br/>".join(s.split("\n"))
+            # -- normalize whitespace in cell --
+            cell_text = " ".join(s.split())
+            # -- emit void `<td/>` when cell text is empty string --
+            yield f"<td>{cell_text}</td>" if cell_text else "<td/>"
+
+    return f"<table>{''.join(iter_trs(matrix))}</table>" if matrix else ""
+
+
+class HtmlTable:
+    """A `<table>` element."""
+
+    def __init__(
+        self,
+        table: HtmlElement,
+        header_row_idxs: set[int] | None = None,
+        source_row_htmls: Sequence[str] | None = None,
+    ):
+        self._table = table
+        self._header_row_idxs = header_row_idxs or set()
+        self._source_row_htmls = tuple(source_row_htmls or ())
+
+    @classmethod
+    def from_html_text(cls, html_text: str) -> HtmlTable:
+        # -- root is always a `<table>` element so far but let's be robust --
+        root = fragment_fromstring(html_text)
+        tables = root.xpath("//table")
+        if not tables:
+            raise ValueError("`html_text` contains no `<table>` element")
+        table = tables[0]
+
+        # -- capture header semantics and source row HTML before compactification strips details --
+        rows = cast("list[HtmlElement]", table.xpath("./tr | ./thead/tr | ./tbody/tr | ./tfoot/tr"))
+        source_row_htmls = tuple(etree.tostring(tr, encoding=str) for tr in rows)
+        header_row_idxs = {
+            idx
+            for idx, tr in enumerate(rows)
+            if tr.getparent().tag == "thead" or bool(tr.xpath("./th"))
+        }
+
+        # -- remove `<thead>`, `<tbody>`, and `<tfoot>` noise elements when present --
+        noise_elements = table.xpath(".//thead | .//tbody | .//tfoot")
+        for e in noise_elements:
+            e.drop_tag()
+
+        # -- normalize and compactify the HTML --
+        for e in table.iter():
+            # -- Strip cosmetic attributes like border="1", class="dataframe" added
+            # -- by pandas.DataFrame.to_html(), style="text-align: right;", etc.
+            # -- Preserve colspan/rowspan: they are structural, not cosmetic, and are
+            # -- required to reconstruct merged-cell layout in chunk HTML.
+            preserved = {k: e.attrib[k] for k in ("colspan", "rowspan") if k in e.attrib}
+            e.attrib.clear()
+            for k, v in preserved.items():
+                e.attrib[k] = v
+
+            # -- change any `<th>` elements to `<td>` so all cells have the same tag --
+            if e.tag == "th":
+                e.tag = "td"
+
+            # -- normalize whitespace in element text; this removes indent whitespace before nested
+            # -- elements and reduces whitespace between words to a single space.
+            if e.text:
+                e.text = " ".join(e.text.split())
+
+            # -- normalize tails. A tail is the text between an element's closing tag and the
+            # -- start of the next sibling. Pure-whitespace tails are pretty-printing noise and
+            # -- can be dropped, but tails can also carry real content (e.g. mixed inline
+            # -- markup like `<b>foo</b> bar <b>baz</b>` or text between `<br/>` tags), which
+            # -- must be preserved.
+            if e.tail:
+                parts = e.tail.split()
+                if not parts:
+                    e.tail = None
+                else:
+                    prefix = " " if e.tail[0].isspace() else ""
+                    suffix = " " if e.tail[-1].isspace() else ""
+                    e.tail = prefix + " ".join(parts) + suffix
+
+        return cls(table, header_row_idxs=header_row_idxs, source_row_htmls=source_row_htmls)
+
+    @cached_property
+    def html(self) -> str:
+        """The HTML-fragment for this `<table>` element, all on one line.
+
+        Like: `<table><tr><td>foo</td></tr><tr><td>bar</td></tr></table>`
+
+        The HTML contains no human-readability whitespace, attributes, or `<thead>`, `<tbody>`, or
+        `<tfoot>` tags. It is made as compact as possible to maximize the semantic content in a
+        given space. This is particularly important for chunking.
+        """
+        return etree.tostring(self._table, encoding=str)
+
+    def iter_rows(self) -> Iterator[HtmlRow]:
+        rows = cast("list[HtmlElement]", self._table.xpath("./tr"))
+        for idx, tr in enumerate(rows):
+            source_html = self._source_row_htmls[idx] if idx < len(self._source_row_htmls) else None
+            yield HtmlRow(tr, is_header=(idx in self._header_row_idxs), source_html=source_html)
+
+    @cached_property
+    def text(self) -> str:
+        """The clean, concatenated, text for this table."""
+        table_text = " ".join(self._table.itertext())
+        # -- blank cells will introduce extra whitespace, so normalize after accumulating --
+        return " ".join(table_text.split())
+
+
+class HtmlRow:
+    """A `<tr>` element."""
+
+    def __init__(self, tr: HtmlElement, is_header: bool = False, source_html: str | None = None):
+        self._tr = tr
+        self._is_header = is_header
+        self._source_html = source_html
+
+    @cached_property
+    def html(self) -> str:
+        """Like  "<tr><td>foo</td><td>bar</td></tr>"."""
+        return etree.tostring(self._tr, encoding=str)
+
+    def iter_cells(self) -> Iterator[HtmlCell]:
+        for td in self._tr:
+            yield HtmlCell(td)
+
+    @property
+    def is_header(self) -> bool:
+        """True when this row originated from `<thead>` or contains `<th>` cells."""
+        return self._is_header
+
+    @property
+    def source_html(self) -> str | None:
+        """Original source `<tr>` HTML captured before compactification, when available."""
+        return self._source_html
+
+    def iter_cell_texts(self) -> Iterator[str]:
+        """Generate contents of each cell of this row as a separate string.
+
+        A cell that is empty or contains only whitespace does not generate a string.
+        """
+        for td in self._tr:
+            text = " ".join(td.text_content().split())
+            if not text:
+                continue
+            yield text
+
+    @cached_property
+    def text_len(self) -> int:
+        """Length of the normalized text, as it would appear in `element.text`."""
+        return len(" ".join(self.iter_cell_texts()))
+
+
+class HtmlCell:
+    """A `<td>` element."""
+
+    def __init__(self, td: HtmlElement):
+        self._td = td
+
+    @cached_property
+    def html(self) -> str:
+        """Like  "<td>foo bar baz</td>"."""
+        return etree.tostring(self._td, encoding=str) if self.text else "<td/>"
+
+    @cached_property
+    def text(self) -> str:
+        """Text inside `<td>` element, empty string when no text."""
+        return " ".join(self._td.text_content().split())

unstructured/doctor.py

@@ -0,0 +1,296 @@
+"""Environment and capability diagnostics for optional dependencies and system tools."""
+
+from __future__ import annotations
+
+import importlib.util
+import platform
+import shutil
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Literal
+
+from unstructured.__version__ import __version__
+from unstructured.file_utils.model import FileType
+from unstructured.utils import dependency_exists
+
+Status = Literal["ok", "missing", "warn"]
+
+# File types that rely on LibreOffice for conversion (see partition/common/common.py).
+_FILE_TYPES_NEEDING_SOFFICE: frozenset[FileType] = frozenset({FileType.DOC, FileType.PPT})
+
+
+@dataclass(frozen=True)
+class CapabilityResult:
+    """Result of checking whether partitioning is viable for a target file type."""
+
+    ready: bool
+    """True when Python deps and required external tools for this type are satisfied."""
+    messages: tuple[str, ...]
+    """Human-readable issues (blocking or informational)."""
+
+
+def _pip_hint_for_file_type(file_type: FileType) -> str:
+    extra = file_type.extra_name
+    if extra:
+        return f'pip install "unstructured[{extra}]"'
+    return "pip install unstructured"
+
+
+def _needs_pandoc_runtime(file_type: FileType) -> bool:
+    return "pypandoc" in file_type.importable_package_dependencies
+
+
+def _audio_extra_import_ok() -> bool:
+    return importlib.util.find_spec("whisper") is not None
+
+
+def _libmagic_status() -> tuple[Status, str]:
+    try:
+        import magic
+
+        _ = magic.from_buffer(b"%PDF-1.4\n", mime=True)
+    except ImportError:
+        return "warn", "python `magic` module not available (filetype fallback still works)"
+    except Exception as exc:  # noqa: BLE001 — surface libmagic load/binary issues
+        return "warn", f"libmagic not usable: {exc!s}"[:200]
+    else:
+        return "ok", "libmagic (python-magic) OK"
+
+
+def _tool_status(name: str, on_path: bool) -> tuple[Status, str]:
+    if on_path:
+        return "ok", f"{name} found on PATH"
+    return "missing", f"{name} not found on PATH"
+
+
+def python_import_deps_ok(file_type: FileType) -> tuple[bool, list[str]]:
+    """Return whether declared importable dependencies for `file_type` are importable."""
+    missing: list[str] = []
+    for mod in file_type.importable_package_dependencies:
+        if not dependency_exists(mod):
+            missing.append(mod)
+    return len(missing) == 0, missing
+
+
+def evaluate_file_type_capability(file_type: FileType) -> CapabilityResult:
+    """Check partitioning readiness for a single :class:`FileType`."""
+    if not file_type.is_partitionable:
+        return CapabilityResult(
+            ready=False,
+            messages=(f"{file_type.name} is not partitionable.",),
+        )
+
+    ok, missing = python_import_deps_ok(file_type)
+    messages: list[str] = []
+    if not ok:
+        messages.append(
+            f"Missing Python module(s): {', '.join(missing)}. "
+            f"Install with: {_pip_hint_for_file_type(file_type)}",
+        )
+
+    if _needs_pandoc_runtime(file_type) and ok and shutil.which("pandoc") is None:
+        ok = False
+        messages.append(
+            "Pandoc executable not found on PATH (required by pypandoc). "
+            "Install pandoc from https://pandoc.org/installing.html",
+        )
+
+    if file_type in _FILE_TYPES_NEEDING_SOFFICE and ok and _libreoffice_on_path() is None:
+        ok = False
+        messages.append(
+            "soffice (LibreOffice CLI) not found on PATH. "
+            "Required to convert legacy .doc / .ppt to Open XML formats.",
+        )
+
+    # Audio partitioner uses the `audio` extra (Whisper); FileType members only list empty deps.
+    if file_type.extra_name == "audio":
+        if not _audio_extra_import_ok():
+            ok = False
+            messages.append(
+                "Missing audio extra (e.g. Whisper). "
+                'Install with: pip install "unstructured[audio]"',
+            )
+        if shutil.which("ffmpeg") is None:
+            ok = False
+            messages.append(
+                "ffmpeg not on PATH - required for Whisper audio decoding "
+                "(https://ffmpeg.org/download.html).",
+            )
+
+    return CapabilityResult(ready=ok, messages=tuple(messages))
+
+
+def _libreoffice_on_path() -> str | None:
+    """Return ``soffice`` on PATH; matches :func:`convert_office_doc` in ``partition.common``."""
+    return shutil.which("soffice")
+
+
+def resolve_specifier(spec: str) -> list[FileType]:
+    """Map a user string (e.g. ``pdf``, ``png``, ``image``) to :class:`FileType` members."""
+    raw = spec.strip()
+    if not raw:
+        raise ValueError("Empty specifier")
+    lower = raw.lower()
+
+    if lower == "image":
+        return [ft for ft in FileType if ft.is_partitionable and ft.extra_name == "image"]
+    if lower == "audio":
+        return [ft for ft in FileType if ft.is_partitionable and ft.extra_name == "audio"]
+
+    matches: list[FileType] = []
+    for ft in FileType:
+        if not ft.is_partitionable:
+            continue
+        if ft.name.lower() == lower or ft.value == lower:
+            matches.append(ft)
+            continue
+        if ft.partitioner_shortname and ft.partitioner_shortname.lower() == lower:
+            matches.append(ft)
+
+    if not matches:
+        valid = sorted(
+            {ft.value for ft in FileType if ft.is_partitionable}
+            | {ft.name.lower() for ft in FileType if ft.is_partitionable}
+            | {"image", "audio"},
+        )
+        sample = ", ".join(valid[:20])
+        raise ValueError(f"Unknown file type or alias {spec!r}. Examples: {sample}...")
+
+    # Prefer exact value/name matches over partitioner_shortname duplicates.
+    exact = [ft for ft in matches if ft.value == lower or ft.name.lower() == lower]
+    return exact or matches
+
+
+def evaluate_specifier(spec: str) -> CapabilityResult:
+    """Combine evaluation for all file types matched by :func:`resolve_specifier`."""
+    targets = resolve_specifier(spec)
+    if (
+        len(targets) > 1
+        and targets[0].extra_name in ("image", "audio")
+        and all(t.extra_name == targets[0].extra_name for t in targets)
+    ):
+        targets = [targets[0]]
+    combined_ready = True
+    all_messages: list[str] = []
+    for ft in targets:
+        result = evaluate_file_type_capability(ft)
+        if not result.ready:
+            combined_ready = False
+        for m in result.messages:
+            all_messages.append(f"[{ft.name}] {m}")
+    # De-duplicate messages while preserving order
+    seen: set[str] = set()
+    deduped: list[str] = []
+    for m in all_messages:
+        if m not in seen:
+            seen.add(m)
+            deduped.append(m)
+    return CapabilityResult(ready=combined_ready, messages=tuple(deduped))
+
+
+def environment_rows() -> list[tuple[str, str, str]]:
+    """Rows for the environment section: (name, status, detail)."""
+    return [
+        ("Python", "ok", platform.python_version()),
+        ("Platform", "ok", platform.platform()),
+        ("unstructured", "ok", __version__),
+    ]
+
+
+def system_tool_rows() -> list[tuple[str, Status, str]]:
+    """Rows for optional system tools."""
+    rows: list[tuple[str, Status, str]] = []
+    st, detail = _libmagic_status()
+    rows.append(("libmagic (MIME detection)", st, detail))
+
+    for label, cmd in (
+        ("tesseract (OCR)", "tesseract"),
+        ("pandoc", "pandoc"),
+        ("ffmpeg (audio/codecs)", "ffmpeg"),
+    ):
+        status, msg = _tool_status(label, shutil.which(cmd) is not None)
+        rows.append((label, status, msg))
+    lo = _libreoffice_on_path()
+    rows.append(
+        (
+            "soffice (LibreOffice CLI)",
+            "ok" if lo else "missing",
+            f"found: {lo}" if lo else "not found on PATH",
+        ),
+    )
+    return rows
+
+
+def partitionable_file_type_rows() -> list[tuple[str, str, str, str]]:
+    """One row per partitionable file type: type, deps OK, extra, notes."""
+    out: list[tuple[str, str, str, str]] = []
+    for ft in sorted(
+        (m for m in FileType if m.is_partitionable),
+        key=lambda x: x.name,
+    ):
+        py_ok, _missing = python_import_deps_ok(ft)
+        deps_cell = "yes" if py_ok else "no"
+        extra = ft.extra_name or "-"
+        cap = evaluate_file_type_capability(ft)
+        notes = " | ".join(cap.messages) if cap.messages else "-"
+        out.append((ft.name, deps_cell, extra, notes))
+    return out
+
+
+def format_table(headers: tuple[str, ...], rows: list[tuple[str, ...]]) -> str:
+    """Render a simple fixed-width table (no third-party deps)."""
+    if not rows:
+        return " | ".join(headers) + "\n(no rows)\n"
+    col_widths = [len(h) for h in headers]
+    str_rows: list[list[str]] = []
+    for row in rows:
+        cells = [str(c) for c in row]
+        str_rows.append(cells)
+        for i, cell in enumerate(cells):
+            col_widths[i] = max(col_widths[i], len(cell))
+    sep = "-+-".join("-" * w for w in col_widths)
+    lines = [
+        " | ".join(headers[i].ljust(col_widths[i]) for i in range(len(headers))),
+        sep,
+    ]
+    for cells in str_rows:
+        lines.append(" | ".join(cells[i].ljust(col_widths[i]) for i in range(len(headers))))
+    return "\n".join(lines) + "\n"
+
+
+def build_report() -> str:
+    """Full diagnostic report for :command:`unstructured doctor` with no filter."""
+    parts: list[str] = []
+    parts.append("Environment\n")
+    parts.append(format_table(("Component", "Status", "Detail"), environment_rows()))
+
+    parts.append("System tools (optional but commonly needed)\n")
+    sys_rows = system_tool_rows()
+    parts.append(format_table(("Tool", "Status", "Detail"), sys_rows))
+
+    parts.append("Partitionable file types (Python extras)\n")
+    p_rows = partitionable_file_type_rows()
+    parts.append(
+        format_table(
+            ("File type", "Python deps OK", "pip extra", "Notes"),
+            [tuple(r) for r in p_rows],
+        ),
+    )
+    return "".join(parts)
+
+
+def file_path_to_capability(path: str | Path) -> CapabilityResult:
+    """Infer file type from ``path`` and evaluate capability."""
+    p = Path(path)
+    if not p.is_file():
+        return CapabilityResult(ready=False, messages=(f"Not a file or not found: {p}",))
+
+    from unstructured.file_utils.filetype import detect_filetype
+
+    ft = detect_filetype(file_path=str(p.resolve()))
+    if not ft.is_partitionable:
+        return CapabilityResult(
+            ready=False,
+            messages=(f"Detected type {ft.name} is not partitionable.",),
+        )
+    return evaluate_file_type_capability(ft)

unstructured/documents/coordinates.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Dict, Sequence, Tuple, Union
+
+
+class Orientation(Enum):
+    SCREEN = (1, -1)  # Origin in top left, y increases in the down direction
+    CARTESIAN = (1, 1)  # Origin in bottom left, y increases in upward direction
+
+
+def convert_coordinate(old_t, old_t_max, new_t_max, t_orientation):
+    """Convert a coordinate into another system along an axis using a linear transformation"""
+    return (
+        (1 - old_t / old_t_max) * (1 - t_orientation) / 2
+        + old_t / old_t_max * (1 + t_orientation) / 2
+    ) * new_t_max
+
+
+class CoordinateSystem:
+    """A finite coordinate plane with given width and height."""
+
+    orientation: Orientation
+
+    def __init__(self, width: Union[int, float], height: Union[int, float]):
+        self.width = width
+        self.height = height
+
+    def __eq__(self, other: object):
+        if not isinstance(other, CoordinateSystem):
+            return False
+        return (
+            str(self.__class__.__name__) == str(other.__class__.__name__)
+            and self.width == other.width
+            and self.height == other.height
+            and self.orientation == other.orientation
+        )
+
+    def convert_from_relative(
+        self,
+        x: Union[float, int],
+        y: Union[float, int],
+    ) -> Tuple[Union[float, int], Union[float, int]]:
+        """Convert to this coordinate system from a relative coordinate system."""
+        x_orientation, y_orientation = self.orientation.value
+        new_x = convert_coordinate(x, 1, self.width, x_orientation)
+        new_y = convert_coordinate(y, 1, self.height, y_orientation)
+        return new_x, new_y
+
+    def convert_to_relative(
+        self,
+        x: Union[float, int],
+        y: Union[float, int],
+    ) -> Tuple[Union[float, int], Union[float, int]]:
+        """Convert from this coordinate system to a relative coordinate system."""
+        x_orientation, y_orientation = self.orientation.value
+        new_x = convert_coordinate(x, self.width, 1, x_orientation)
+        new_y = convert_coordinate(y, self.height, 1, y_orientation)
+        return new_x, new_y
+
+    def convert_coordinates_to_new_system(
+        self,
+        new_system: CoordinateSystem,
+        x: Union[float, int],
+        y: Union[float, int],
+    ) -> Tuple[Union[float, int], Union[float, int]]:
+        """Convert from this coordinate system to another given coordinate system."""
+        rel_x, rel_y = self.convert_to_relative(x, y)
+        return new_system.convert_from_relative(rel_x, rel_y)
+
+    def convert_multiple_coordinates_to_new_system(
+        self,
+        new_system: CoordinateSystem,
+        coordinates: Sequence[Tuple[Union[float, int], Union[float, int]]],
+    ) -> Tuple[Tuple[Union[float, int], Union[float, int]], ...]:
+        """Convert (x, y) coordinates from current system to another coordinate system."""
+        new_system_coordinates = []
+        for x, y in coordinates:
+            new_system_coordinates.append(
+                self.convert_coordinates_to_new_system(new_system=new_system, x=x, y=y),
+            )
+        return tuple(new_system_coordinates)
+
+
+class RelativeCoordinateSystem(CoordinateSystem):
+    """Relative coordinate system where x and y are on a scale from 0 to 1."""
+
+    orientation = Orientation.CARTESIAN
+
+    def __init__(self):
+        self.width = 1
+        self.height = 1
+
+
+class PixelSpace(CoordinateSystem):
+    """Coordinate system representing a pixel space, such as an image. The origin is at the top
+    left."""
+
+    orientation = Orientation.SCREEN
+
+
+class PointSpace(CoordinateSystem):
+    """Coordinate system representing a point space, such as a pdf. The origin is at the bottom
+    left."""
+
+    orientation = Orientation.CARTESIAN
+
+
+TYPE_TO_COORDINATE_SYSTEM_MAP: Dict[str, Any] = {
+    "PixelSpace": PixelSpace,
+    "PointSpace": PointSpace,
+    "CoordinateSystem": CoordinateSystem,
+}

unstructured/documents/html_sanitization.py

@@ -0,0 +1,427 @@
+"""Central HTML output-sanitization policy for the ontology (v2) HTML path.
+
+`unstructured` renders untrusted document content into HTML in two places:
+
+* ``OntologyElement.to_html`` (``documents/ontology.py``), which fills
+  ``ElementMetadata.text_as_html``. Some callers return this value to clients
+  verbatim, so it must be safe on its own.
+* ``elements_to_html`` (``partition/html/convert.py``), which assembles a full
+  HTML document from a list of elements.
+
+Both used to interpolate attacker-controlled text, attribute names, attribute
+values, and URL schemes with no output encoding, allowing stored XSS
+(GHSA-v5mq-3xhg-98m9). This module is the single source of truth for the
+sanitization policy shared by both paths:
+
+* an allowlist of HTML tags we ever legitimately emit,
+* an allowlist of attribute names (event-handler ``on*`` attributes are never
+  allowed, killing ``onerror``/``onload``/``onmouseover``),
+* a URL-scheme allowlist for URL-bearing attributes (``href``/``src``/...),
+  which drops ``javascript:`` / ``vbscript:`` and permits ``data:`` only for
+  raster image MIME types on ``img[src]``.
+
+The emitter (``ontology.py``) uses the lightweight filters here plus
+``html.escape`` to make ``text_as_html`` safe on its own; ``elements_to_html``
+additionally runs the assembled document through :func:`sanitize_html_fragment`
+(``nh3``) as defense-in-depth that also covers attributes it injects itself
+(e.g. ``href`` from ``metadata.url``).
+"""
+
+from __future__ import annotations
+
+import re
+
+import nh3
+
+# -- Tags the ontology / convert paths legitimately emit. Anything outside this
+# -- set (``<script>``, ``<iframe>``, ...) is dropped/neutralized. --
+ALLOWED_TAGS: frozenset[str] = frozenset(
+    {
+        # layout / structural
+        "body",
+        "div",
+        "section",
+        "header",
+        "footer",
+        "aside",
+        "nav",
+        "figure",
+        "figcaption",
+        "hr",
+        "br",
+        # text
+        "span",
+        "p",
+        "blockquote",
+        "pre",
+        "address",
+        "time",
+        "mark",
+        "ins",
+        "del",
+        "cite",
+        "sub",
+        "sup",
+        "b",
+        "i",
+        "s",
+        "code",
+        # headings
+        "h1",
+        "h2",
+        "h3",
+        "h4",
+        "h5",
+        "h6",
+        # lists
+        "ul",
+        "ol",
+        "li",
+        "dl",
+        # tables
+        "table",
+        "thead",
+        "tbody",
+        "tr",
+        "td",
+        "th",
+        # links / media
+        "a",
+        "img",
+        "svg",
+        "audio",
+        "video",
+        # forms
+        "form",
+        "input",
+        "label",
+        "button",
+        # misc content
+        "math",
+        "meta",
+    }
+)
+
+# -- Attribute names carrying a URL; their values are scheme-filtered. Keep this
+# -- list broader than the attributes we currently allow so newly-allowed URL
+# -- attributes are scheme-checked by default. --
+URL_ATTRIBUTES: frozenset[str] = frozenset(
+    {
+        "action",
+        "cite",
+        "data-src",
+        "formaction",
+        "href",
+        "poster",
+        "src",
+        "srcset",
+        "xlink:href",
+    }
+)
+
+# -- Data URLs are only needed for embedded image bytes. SVG is intentionally
+# -- excluded because SVG documents can carry active content in some render
+# -- contexts. --
+ALLOWED_DATA_IMAGE_MIME_TYPES: frozenset[str] = frozenset(
+    {
+        "image/avif",
+        "image/bmp",
+        "image/gif",
+        "image/jpeg",
+        "image/jpg",
+        "image/png",
+        "image/webp",
+        "image/x-icon",
+    }
+)
+
+# -- Inline CSS is untrusted too. Keep only inert presentation properties that
+# -- preserve existing table/background formatting and reject layout/overlay
+# -- controls like position/inset/z-index.
+ALLOWED_CSS_PROPERTIES: frozenset[str] = frozenset(
+    {
+        "background-color",
+        "border",
+        "border-bottom",
+        "border-collapse",
+        "border-color",
+        "border-left",
+        "border-right",
+        "border-style",
+        "border-top",
+        "border-width",
+        "color",
+        "font-style",
+        "font-weight",
+        "text-align",
+        "text-decoration",
+        "vertical-align",
+        "white-space",
+    }
+)
+
+# -- Attributes allowed on every tag. --
+_GLOBAL_ATTRIBUTES: frozenset[str] = frozenset(
+    {"class", "id", "style", "title", "dir", "lang", "role", "name", "align"}
+)
+
+# -- Per-tag attributes in addition to the global set. --
+_TAG_ATTRIBUTES: dict[str, frozenset[str]] = {
+    "a": frozenset({"href", "target", "rel"}),
+    "img": frozenset({"src", "alt", "width", "height"}),
+    "svg": frozenset({"src", "alt", "width", "height", "xlink:href"}),
+    "audio": frozenset({"src", "controls"}),
+    "video": frozenset({"src", "controls", "poster", "width", "height"}),
+    "input": frozenset({"type", "checked", "value", "placeholder"}),
+    "td": frozenset({"colspan", "rowspan", "headers", "scope"}),
+    "th": frozenset({"colspan", "rowspan", "headers", "scope"}),
+    "ol": frozenset({"start", "type"}),
+    "label": frozenset({"for"}),
+    "meta": frozenset({"charset", "content"}),
+    "time": frozenset({"datetime"}),
+}
+
+# -- Attribute-name prefixes allowed on any tag (data-page-number, aria-*, ...). --
+_GENERIC_ATTRIBUTE_PREFIXES: frozenset[str] = frozenset({"data-", "aria-"})
+
+# -- URL schemes permitted on URL-bearing attributes. ``data`` is permitted here
+# -- but further restricted to raster image MIME types on ``img[src]`` by
+# -- :func:`is_safe_url` / the nh3 attribute filter; relative URLs (no scheme)
+# -- are always allowed. --
+ALLOWED_URL_SCHEMES: frozenset[str] = frozenset({"http", "https", "mailto", "tel", "data"})
+
+# -- Valid HTML/XML attribute name (prevents attribute-name breakout on emit). --
+_ATTRIBUTE_NAME_RE = re.compile(r"^[a-zA-Z_:][-a-zA-Z0-9_:.]*$")
+
+# -- Matches a leading ``scheme:`` ignoring surrounding whitespace and embedded
+# -- control chars that browsers strip (e.g. ``java\tscript:``). --
+_SCHEME_RE = re.compile(r"^([a-zA-Z][a-zA-Z0-9+.\-]*):")
+
+_REQUIRED_BLANK_TARGET_REL_VALUES: frozenset[str] = frozenset({"noopener", "noreferrer"})
+
+# -- Match an anchor start tag and detect `target="_blank"` / an existing `rel`
+# -- so we can add reverse-tabnabbing protection to new-tab links only. nh3's
+# -- output is well-formed (attribute values are escaped, so `>` never appears
+# -- inside one), which makes matching whole start tags with a regex safe. --
+_ANCHOR_START_TAG_RE = re.compile(r"<a\b[^>]*>", flags=re.IGNORECASE)
+_BLANK_TARGET_RE = re.compile(r"""\btarget\s*=\s*["']?_blank\b""", flags=re.IGNORECASE)
+_REL_ATTRIBUTE_RE = re.compile(r"""\brel\s*=""", flags=re.IGNORECASE)
+
+_CSS_COMMENT_RE = re.compile(r"/\*.*?\*/", flags=re.DOTALL)
+_CSS_UNSAFE_VALUE_RE = re.compile(
+    r"(@import|expression\s*\(|url\s*\(|javascript:|vbscript:|data:|-moz-binding|behavior\s*:)",
+    flags=re.IGNORECASE,
+)
+
+
+def _normalize_url(value: str) -> str:
+    """Lower-case and strip whitespace/control chars a browser would ignore."""
+    return re.sub(r"[\x00-\x20]+", "", value).lower()
+
+
+def _normalized_tag_and_attribute(
+    tag_name: str | None = None,
+    attribute_name: str | None = None,
+) -> tuple[str | None, str | None]:
+    tag = tag_name.strip().lower() if tag_name else None
+    attribute = attribute_name.strip().lower() if attribute_name else None
+    return tag, attribute
+
+
+def _is_allowed_data_image_url(
+    normalized_value: str,
+    *,
+    tag_name: str | None = None,
+    attribute_name: str | None = None,
+) -> bool:
+    tag, attribute = _normalized_tag_and_attribute(tag_name, attribute_name)
+    if tag != "img" or attribute != "src":
+        return False
+    if not normalized_value.startswith("data:") or "," not in normalized_value:
+        return False
+    media_type = normalized_value[5:].split(",", 1)[0].split(";", 1)[0]
+    return media_type in ALLOWED_DATA_IMAGE_MIME_TYPES
+
+
+def is_safe_url(
+    value: str,
+    *,
+    tag_name: str | None = None,
+    attribute_name: str | None = None,
+) -> bool:
+    """True if ``value`` is safe to keep in a URL-bearing attribute.
+
+    Relative URLs (no scheme) are allowed. Absolute URLs are allowed only for
+    :data:`ALLOWED_URL_SCHEMES`, and ``data:`` is further narrowed to raster
+    image MIME types on ``img[src]`` so embedded base64 images survive while
+    SVG / HTML / JavaScript data documents are rejected.
+    """
+    normalized = _normalize_url(value)
+    match = _SCHEME_RE.match(normalized)
+    if match is None:
+        # -- no scheme -> relative URL (or a fragment/anchor); safe --
+        return True
+    scheme = match.group(1)
+    if scheme not in ALLOWED_URL_SCHEMES:
+        return False
+    if scheme == "data":
+        return _is_allowed_data_image_url(
+            normalized,
+            tag_name=tag_name,
+            attribute_name=attribute_name,
+        )
+    return True
+
+
+def is_event_handler_attribute(name: str) -> bool:
+    """True for ``on*`` event-handler attribute names (onerror, onload, ...)."""
+    return name.strip().lower().startswith("on")
+
+
+def _is_allowed_attribute_for_tag(name: str, tag_name: str | None) -> bool:
+    lowered = name.lower()
+    if any(lowered.startswith(prefix) for prefix in _GENERIC_ATTRIBUTE_PREFIXES):
+        return True
+
+    allowed_attributes = set(_GLOBAL_ATTRIBUTES)
+    if tag_name:
+        allowed_attributes.update(_TAG_ATTRIBUTES.get(tag_name.strip().lower(), frozenset()))
+    return lowered in allowed_attributes
+
+
+def _link_rel_with_required_values(value: object | None) -> str:
+    existing_values = str(value or "").split()
+    existing_lowered = {rel.lower() for rel in existing_values}
+    missing_values = [
+        rel for rel in sorted(_REQUIRED_BLANK_TARGET_REL_VALUES) if rel not in existing_lowered
+    ]
+    return " ".join([*existing_values, *missing_values]).strip()
+
+
+def sanitize_style_attribute(value: object) -> str:
+    """Return a style attribute containing only safe presentation declarations."""
+    style = _CSS_COMMENT_RE.sub("", str(value))
+    safe_declarations: list[str] = []
+    for declaration in style.split(";"):
+        if ":" not in declaration:
+            continue
+        raw_property, raw_value = declaration.split(":", 1)
+        property_name = raw_property.strip().lower()
+        property_value = " ".join(raw_value.strip().split())
+        if property_name not in ALLOWED_CSS_PROPERTIES:
+            continue
+        if not property_value or _CSS_UNSAFE_VALUE_RE.search(property_value):
+            continue
+        safe_declarations.append(f"{property_name}: {property_value}")
+    return "; ".join(safe_declarations)
+
+
+def sanitize_attributes(
+    attributes: dict[str, object],
+    tag_name: str | None = None,
+) -> dict[str, object]:
+    """Filter an attribute mapping for safe emission (does NOT html-escape).
+
+    Drops event-handler (``on*``) attributes, attribute names that aren't valid
+    HTML attribute names, attributes not allowed for ``tag_name``, and URL-bearing
+    attributes whose value uses an unsafe scheme. Values are returned unchanged;
+    the emitter is responsible for ``html.escape``-ing them so escaping happens
+    exactly once.
+    """
+    safe: dict[str, object] = {}
+    for key, value in attributes.items():
+        name = str(key).strip()
+        lowered = name.lower()
+        if is_event_handler_attribute(lowered):
+            continue
+        if not _ATTRIBUTE_NAME_RE.match(name):
+            continue
+        if not _is_allowed_attribute_for_tag(name, tag_name):
+            continue
+        if lowered == "style":
+            sanitized_style = sanitize_style_attribute(value)
+            if not sanitized_style:
+                continue
+            safe[name] = sanitized_style
+            continue
+        if lowered in URL_ATTRIBUTES and value is not None:
+            candidate = value[0] if isinstance(value, list) and value else value
+            if isinstance(candidate, str) and not is_safe_url(
+                candidate,
+                tag_name=tag_name,
+                attribute_name=lowered,
+            ):
+                continue
+        safe[name] = value
+
+    if (tag_name or "").strip().lower() == "a" and str(safe.get("target", "")).lower() == "_blank":
+        safe["rel"] = _link_rel_with_required_values(safe.get("rel"))
+
+    return safe
+
+
+def is_safe_tag(tag_name: str | None) -> bool:
+    """True if ``tag_name`` is in the emit allowlist."""
+    return bool(tag_name) and tag_name.strip().lower() in ALLOWED_TAGS
+
+
+def _nh3_attribute_filter(tag: str, attribute: str, value: str) -> str | None:
+    """nh3 per-attribute hook: drop event handlers and unsafe URL values."""
+    if is_event_handler_attribute(attribute):
+        return None
+    if attribute.lower() == "style":
+        return sanitize_style_attribute(value) or None
+    if attribute.lower() in URL_ATTRIBUTES and not is_safe_url(
+        value,
+        tag_name=tag,
+        attribute_name=attribute,
+    ):
+        return None
+    return value
+
+
+def _add_rel_to_blank_target_links(html_fragment: str) -> str:
+    """Add reverse-tabnabbing ``rel`` tokens to ``target="_blank"`` anchors only.
+
+    nh3's ``link_rel`` would stamp ``rel`` onto *every* link, stripping the
+    ``Referer`` header from ordinary same-tab navigation. Reverse tabnabbing is
+    only a concern for new-tab links, so we scope the tokens to anchors that
+    actually open a new browsing context and leave same-tab links untouched.
+    """
+
+    def add_rel(match: re.Match[str]) -> str:
+        tag = match.group(0)
+        if not _BLANK_TARGET_RE.search(tag) or _REL_ATTRIBUTE_RE.search(tag):
+            return tag
+        rel = " ".join(sorted(_REQUIRED_BLANK_TARGET_REL_VALUES))
+        return f'{tag[:-1].rstrip()} rel="{rel}">'
+
+    return _ANCHOR_START_TAG_RE.sub(add_rel, html_fragment)
+
+
+def sanitize_html_fragment(html_fragment: str) -> str:
+    """Sanitize an assembled HTML fragment with ``nh3`` (defense-in-depth).
+
+    Applies the shared tag/attribute/URL-scheme allowlists. Used on the final
+    output of ``elements_to_html`` so that attributes injected outside the
+    ontology emitter (e.g. an ``href`` built from ``metadata.url``) are also
+    neutralized.
+    """
+    attributes: dict[str, set[str]] = {"*": set(_GLOBAL_ATTRIBUTES)}
+    for tag, attrs in _TAG_ATTRIBUTES.items():
+        attributes[tag] = set(attrs)
+    cleaned = nh3.clean(
+        html_fragment,
+        tags=set(ALLOWED_TAGS),
+        attributes=attributes,
+        url_schemes=set(ALLOWED_URL_SCHEMES),
+        generic_attribute_prefixes=set(_GENERIC_ATTRIBUTE_PREFIXES),
+        filter_style_properties=set(ALLOWED_CSS_PROPERTIES),
+        attribute_filter=_nh3_attribute_filter,
+        # -- Scope reverse-tabnabbing `rel` tokens to `_blank` anchors ourselves
+        # -- rather than letting nh3 add them to every link (which would strip
+        # -- the Referer header from same-tab navigation). --
+        link_rel=None,
+        strip_comments=True,
+    )
+    return _add_rel_to_blank_target_links(cleaned)

unstructured/documents/mappings.py

@@ -0,0 +1,195 @@
+"""
+This module contains mapping between:
+HTML Tags <-> Elements Ontology <-> Unstructured Element classes
+They are used to simplify transformations between different representations
+of parsed documents
+"""
+
+from typing import Dict, Type
+
+from unstructured.documents import elements, ontology
+from unstructured.documents.elements import Element
+
+
+def get_all_subclasses(cls: type) -> list[type]:
+    """
+    Recursively find all subclasses of a given class.
+
+    Parameters:
+    cls (type): The class for which to find all subclasses.
+
+    Returns:
+    list[type]: A list of all subclasses of the given class.
+    """
+    subclasses = cls.__subclasses__()
+    all_subclasses = subclasses.copy()
+
+    for subclass in subclasses:
+        all_subclasses.extend(get_all_subclasses(subclass))
+
+    return all_subclasses
+
+
+def get_ontology_to_unstructured_type_mapping() -> dict[
+    Type[ontology.OntologyElement], Type[Element]
+]:
+    """
+    Get a mapping of ontology element to unstructured type.
+
+    The dictionary here was created base on ontology mapping json
+    Can be generated via the following code:
+    ```
+        ontology_elements_list = json.loads(
+            Path("unstructured_element_ontology.json").read_text()
+        )
+        ontology_to_unstructured_class_mapping = {
+            ontology_element["name"]: ontology_element["ontologyV1Mapping"]
+            for ontology_element in ontology_elements_list
+    }
+    ```
+
+    Returns:
+    dict: A dictionary where keys are ontology element classes
+          and values are unstructured types.
+    """
+    ontology_to_unstructured_class_mapping: Dict[Type[ontology.OntologyElement], Type[Element]] = {
+        ontology.Document: elements.Text,
+        ontology.Section: elements.Text,
+        ontology.Page: elements.Text,
+        ontology.Column: elements.Text,
+        ontology.Paragraph: elements.NarrativeText,
+        ontology.Header: elements.Header,
+        ontology.Footer: elements.Footer,
+        ontology.Sidebar: elements.Text,
+        ontology.PageBreak: elements.PageBreak,
+        ontology.Title: elements.Title,
+        ontology.Subtitle: elements.Title,
+        ontology.Heading: elements.Title,
+        ontology.NarrativeText: elements.NarrativeText,
+        ontology.Quote: elements.NarrativeText,
+        ontology.Footnote: elements.Text,
+        ontology.Caption: elements.FigureCaption,
+        ontology.PageNumber: elements.PageNumber,
+        ontology.UncategorizedText: elements.Text,
+        ontology.OrderedList: elements.Text,
+        ontology.UnorderedList: elements.Text,
+        ontology.DefinitionList: elements.Text,
+        ontology.ListItem: elements.ListItem,
+        ontology.Table: elements.Table,
+        ontology.TableRow: elements.Table,
+        ontology.TableCell: elements.Table,
+        ontology.TableCellHeader: elements.Table,
+        ontology.TableBody: elements.Table,
+        ontology.TableHeader: elements.Table,
+        ontology.Image: elements.Image,
+        ontology.Figure: elements.Image,
+        ontology.Video: elements.Text,
+        ontology.Audio: elements.Text,
+        ontology.Barcode: elements.Image,
+        ontology.QRCode: elements.Image,
+        ontology.Logo: elements.Image,
+        ontology.CodeBlock: elements.CodeSnippet,
+        ontology.InlineCode: elements.CodeSnippet,
+        ontology.Formula: elements.Formula,
+        ontology.Equation: elements.Formula,
+        ontology.FootnoteReference: elements.Text,
+        ontology.Citation: elements.Text,
+        ontology.Bibliography: elements.Text,
+        ontology.Glossary: elements.Text,
+        ontology.Author: elements.Text,
+        ontology.MetaDate: elements.Text,
+        ontology.Keywords: elements.Text,
+        ontology.Abstract: elements.NarrativeText,
+        ontology.Hyperlink: elements.Text,
+        ontology.TableOfContents: elements.Table,
+        ontology.Index: elements.Text,
+        ontology.Form: elements.Text,
+        ontology.FormField: elements.Text,
+        ontology.FormFieldValue: elements.Text,
+        ontology.Checkbox: elements.Text,
+        ontology.RadioButton: elements.Text,
+        ontology.Button: elements.Text,
+        ontology.Comment: elements.Text,
+        ontology.Highlight: elements.Text,
+        ontology.RevisionInsertion: elements.Text,
+        ontology.RevisionDeletion: elements.Text,
+        ontology.Address: elements.Address,
+        ontology.EmailAddress: elements.EmailAddress,
+        ontology.PhoneNumber: elements.Text,
+        ontology.CalendarDate: elements.Text,
+        ontology.Time: elements.Text,
+        ontology.Currency: elements.Text,
+        ontology.Measurement: elements.Text,
+        ontology.Letterhead: elements.Header,
+        ontology.Signature: elements.Text,
+        ontology.Watermark: elements.Text,
+        ontology.Stamp: elements.Text,
+    }
+
+    return ontology_to_unstructured_class_mapping
+
+
+ALL_ONTOLOGY_ELEMENT_TYPES = get_all_subclasses(ontology.OntologyElement)
+HTML_TAG_AND_CSS_NAME_TO_ELEMENT_TYPE_MAP: Dict[tuple[str, str], Type[ontology.OntologyElement]] = {
+    (tag, element_type().css_class_name): element_type
+    for element_type in ALL_ONTOLOGY_ELEMENT_TYPES
+    for tag in element_type().allowed_tags
+}
+CSS_CLASS_TO_ELEMENT_TYPE_MAP: Dict[str, Type[ontology.OntologyElement]] = {
+    element_type().css_class_name: element_type for element_type in ALL_ONTOLOGY_ELEMENT_TYPES
+}
+
+HTML_TAG_TO_DEFAULT_ELEMENT_TYPE_MAP: Dict[str, Type[ontology.OntologyElement]] = {
+    "a": ontology.Hyperlink,
+    "address": ontology.Address,
+    "aside": ontology.Sidebar,
+    "audio": ontology.Audio,
+    "blockquote": ontology.Quote,
+    "body": ontology.Document,
+    "button": ontology.Button,
+    "cite": ontology.Citation,
+    "code": ontology.CodeBlock,
+    "del": ontology.RevisionDeletion,
+    "div": ontology.UncategorizedText,
+    "dl": ontology.DefinitionList,
+    "figcaption": ontology.Caption,
+    "figure": ontology.Figure,
+    "footer": ontology.Footer,
+    "form": ontology.Form,
+    "h1": ontology.Title,
+    "h2": ontology.Subtitle,
+    "h3": ontology.Heading,
+    "h4": ontology.Heading,
+    "h5": ontology.Heading,
+    "h6": ontology.Heading,
+    "header": ontology.Header,
+    "hr": ontology.PageBreak,
+    "img": ontology.Image,
+    "input": ontology.Checkbox,
+    "ins": ontology.RevisionInsertion,
+    "label": ontology.FormField,
+    "li": ontology.ListItem,
+    "mark": ontology.Highlight,
+    "math": ontology.Equation,
+    "meta": ontology.Keywords,
+    "nav": ontology.Index,
+    "ol": ontology.OrderedList,
+    "p": ontology.Paragraph,
+    "pre": ontology.CodeBlock,
+    "section": ontology.Section,
+    "span": ontology.UncategorizedText,
+    "sub": ontology.FootnoteReference,
+    "svg": ontology.Signature,
+    "table": ontology.Table,
+    "tbody": ontology.TableBody,
+    "td": ontology.TableCell,
+    "th": ontology.TableCellHeader,
+    "thead": ontology.TableHeader,
+    "time": ontology.Time,
+    "tr": ontology.TableRow,
+    "ul": ontology.UnorderedList,
+    "video": ontology.Video,
+}
+
+
+ONTOLOGY_CLASS_TO_UNSTRUCTURED_ELEMENT_TYPE = get_ontology_to_unstructured_type_mapping()

unstructured/documents/ontology.py

@@ -0,0 +1,647 @@
+"""
+This file contains all classes allowed in the ontology V2.
+This Type is used as intermediate representation between HTML
+and Unstructured Elements.
+All the processing could be done without the intermediate representation,
+but it simplifies the process.
+It needs to be decide whether we keep it or not.
+
+The classes are represented as pydantic models to mimic Unstructured Elements V1 solutions.
+However it results in lots of code that could be strongly simplified.
+
+TODO (Pluto): OntologyElement is the only needed class. It could contains data about
+ allowed html tags, css classes and descriptions as metadata.
+"""
+
+from __future__ import annotations
+
+import html
+import uuid
+from copy import copy
+from enum import Enum
+from typing import Any, List, Optional
+
+from bs4 import BeautifulSoup
+from pydantic import BaseModel, Field
+
+from unstructured.documents.html_sanitization import is_safe_tag, sanitize_attributes
+
+
+class ElementTypeEnum(str, Enum):
+    layout = "Layout"
+    text = "Text"
+    list = "List"
+    table = "Table"
+    media = "Media"
+    code = "Code"
+    mathematical = "Mathematical"
+    reference = "Reference"
+    metadata = "Metadata"
+    navigation = "Navigation"
+    form = "Form"
+    annotation = "Annotation"
+    specialized_text = "Specialized Text"
+    document_specific = "Document-Specific"
+
+
+class OntologyElement(BaseModel):
+    text: Optional[str] = Field("", description="Text content of the element")
+    css_class_name: Optional[str] = Field(
+        default_factory=lambda: "", description="CSS class associated with the element"
+    )
+    html_tag_name: Optional[str] = Field(
+        default_factory=lambda: "", description="HTML Tag name associated with the element"
+    )
+    elementType: ElementTypeEnum = Field(description="Type of the element")
+    children: list["OntologyElement"] = Field(
+        default_factory=list, description="List of child elements"
+    )
+
+    description: str = Field(description="Description of the element")
+    allowed_tags: list[str] = Field(description="HTML tags associated with the element")
+
+    additional_attributes: dict[str, Any] = Field(
+        default_factory=dict, description="Optional HTML attributes or CSS properties"
+    )
+
+    def __init__(self, **kwargs: dict[str, Any]):
+        super().__init__(**kwargs)
+        if self.css_class_name == "":  # if None, then do not set
+            self.css_class_name = self.__class__.__name__
+        if self.html_tag_name == "":
+            self.html_tag_name = self.allowed_tags[0]
+        if "id" not in self.additional_attributes:
+            self.additional_attributes["id"] = self.generate_unique_id()
+
+    @staticmethod
+    def generate_unique_id() -> str:
+        return str(uuid.uuid4()).replace("-", "")
+
+    def to_html(self, add_children: bool = True) -> str:
+        additional_attrs = copy(self.additional_attributes)
+        additional_attrs.pop("class", None)
+        additional_attrs.pop("id", None)
+
+        tag_name = self.html_tag_name if is_safe_tag(self.html_tag_name) else "span"
+        attr_str = self._construct_attribute_string(additional_attrs, tag_name)
+        class_attr = (
+            f'class="{html.escape(self.css_class_name, quote=True)}"' if self.css_class_name else ""
+        )
+
+        combined_attr_str = f"{class_attr} {attr_str}".strip()
+
+        children_html = self._generate_children_html(add_children)
+
+        result_html = self._generate_final_html(combined_attr_str, children_html, tag_name)
+
+        return result_html
+
+    def to_text(self, add_children: bool = True, add_img_alt_text: bool = True) -> str:
+        """
+        Returns the text representation of the element.
+
+        Args:
+            add_children: If True, the text of the children will be included.
+                            Otherwise, element is represented as single self-closing tag.
+            add_img_alt_text: If True, the alt text of the image will be included.
+        """
+        if self.children and add_children:
+            children_text = " ".join(
+                child.to_text(add_children, add_img_alt_text).strip() for child in self.children
+            )
+            return children_text
+
+        # -- Derive the plain text from the raw `text`. When it holds markup
+        # -- (e.g. the collapsed subtree stored when the recursion limit is hit)
+        # -- strip the tags with BeautifulSoup; otherwise use it as-is (which also
+        # -- avoids BeautifulSoup's MarkupResemblesLocatorWarning on plain strings).
+        # -- This is deliberately NOT taken from `to_html()`, whose output is now
+        # -- HTML-escaped for safety -- reparsing that escaped output would surface
+        # -- the markup as literal text instead of stripping it. --
+        raw_text = self.text or ""
+        if "<" in raw_text:
+            text = BeautifulSoup(raw_text, "html.parser").get_text().strip()
+        else:
+            text = raw_text.strip()
+
+        if add_img_alt_text and self.html_tag_name == "img" and "alt" in self.additional_attributes:
+            text += f" {self.additional_attributes.get('alt', '')}"
+
+        return text.strip()
+
+    def _construct_attribute_string(self, attributes: dict[str, str], tag_name: str) -> str:
+        # -- Drop event-handler (on*) and unsafe-scheme attributes, then HTML-escape
+        # -- each value (quote=True) so an attacker cannot break out of the quoted
+        # -- value with `">`. Escaping happens here, at emit time, so callers must
+        # -- pass raw (un-escaped) values -- see html_sanitization.sanitize_attributes. --
+        safe_attributes = sanitize_attributes(attributes, tag_name=tag_name)
+        return " ".join(
+            f'{key}="{html.escape(str(value), quote=True)}"' if value else f"{key}"
+            for key, value in safe_attributes.items()
+        )
+
+    def _generate_children_html(self, add_children: bool) -> str:
+        if not add_children or not self.children:
+            return ""
+        return "".join(child.to_html() for child in self.children)
+
+    def _generate_final_html(self, attr_str: str, children_html: str, tag_name: str) -> str:
+        # -- Escape element text; `children_html` is already-sanitized HTML from
+        # -- recursive `to_html` calls, so it must NOT be re-escaped. --
+        text = html.escape(self.text) if self.text else ""
+
+        if text or children_html:
+            inside_tag_text = f"{text} {children_html}".strip()
+            return f"<{tag_name} {attr_str}>{inside_tag_text}</{tag_name}>"
+        else:
+            return f"<{tag_name} {attr_str} />"
+
+    @property
+    def id(self) -> str | None:
+        return self.additional_attributes.get("id", None)
+
+    @property
+    def page_number(self) -> int | None:
+        if "data-page-number" in self.additional_attributes:
+            try:
+                page_attr = self.additional_attributes.get("data-page-number")
+                if page_attr is not None:
+                    return int(page_attr)
+            except ValueError:
+                return None
+        return None
+
+
+def remove_ids_and_class_from_table(
+    soup: BeautifulSoup, class_attr_to_keep: list[str] = ["img", "input"]
+) -> BeautifulSoup:
+    """
+    Remove id and class attributes from tags inside tables,
+    except preserve class attributes for selected tags.
+
+    Args:
+        soup: BeautifulSoup object containing the HTML
+        class_attr_to_keep: a list of tag names whose class attr will be kept
+
+    Returns:
+        BeautifulSoup: Modified soup with attributes removed
+    """
+    for tag in soup.find_all(True):
+        if tag.name.lower() == "table":  # type: ignore
+            continue  # We keep table tag
+        tag.attrs.pop("id", None)  # type: ignore
+        if tag.name.lower() not in class_attr_to_keep:  # type: ignore
+            tag.attrs.pop("class", None)  # type: ignore
+    return soup
+
+
+# Define specific elements
+class Document(OntologyElement):
+    description: str = Field("Root element of the document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.layout, frozen=True)
+    allowed_tags: List[str] = Field(["body"], frozen=True)
+
+
+class Section(OntologyElement):
+    description: str = Field("A distinct part or subdivision of a document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.layout, frozen=True)
+    allowed_tags: List[str] = Field(["section"], frozen=True)
+
+
+class Page(OntologyElement):
+    description: str = Field("A single side of a paper in a document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.layout, frozen=True)
+    allowed_tags: List[str] = Field(["div"], frozen=True)
+
+
+class Column(OntologyElement):
+    description: str = Field("A vertical section of a page", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.layout, frozen=True)
+    allowed_tags: List[str] = Field(["div"], frozen=True)
+
+
+class Paragraph(OntologyElement):
+    description: str = Field("A self-contained unit of discourse in writing", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["p"], frozen=True)
+
+
+class Header(OntologyElement):
+    description: str = Field("The top section of a page", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["header"], frozen=True)
+
+
+class Footer(OntologyElement):
+    description: str = Field("The bottom section of a page", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["footer"], frozen=True)
+
+
+class Sidebar(OntologyElement):
+    description: str = Field("A side section of a page", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.layout, frozen=True)
+    allowed_tags: List[str] = Field(["aside"], frozen=True)
+
+
+class PageBreak(OntologyElement):
+    description: str = Field("A break between pages", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.layout, frozen=True)
+    allowed_tags: List[str] = Field(["hr"], frozen=True)
+
+
+class Title(OntologyElement):
+    description: str = Field("Main heading of a document or section", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["h1"], frozen=True)
+
+
+class Subtitle(OntologyElement):
+    description: str = Field("Secondary title of a document or section", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["h2"], frozen=True)
+
+
+class Heading(OntologyElement):
+    description: str = Field("Section headings (levels 1-6)", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["h1", "h2", "h3", "h4", "h5", "h6"], frozen=True)
+
+
+class NarrativeText(OntologyElement):
+    description: str = Field("Main content text", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["p"], frozen=True)
+
+
+class Quote(OntologyElement):
+    description: str = Field("A repetition of someone else's statement", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["blockquote"], frozen=True)
+
+
+class Footnote(OntologyElement):
+    description: str = Field("A note at the bottom of a page", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["div"], frozen=True)
+
+
+class Caption(OntologyElement):
+    description: str = Field("Text describing a figure or image", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["figcaption"], frozen=True)
+
+
+class PageNumber(OntologyElement):
+    description: str = Field("The number of a page", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["span"], frozen=True)
+
+
+class UncategorizedText(OntologyElement):
+    description: str = Field("Miscellaneous text", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.text, frozen=True)
+    allowed_tags: List[str] = Field(["span"], frozen=True)
+
+
+class OrderedList(OntologyElement):
+    description: str = Field("A list with a specific sequence", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.list, frozen=True)
+    allowed_tags: List[str] = Field(["ol"], frozen=True)
+
+
+class UnorderedList(OntologyElement):
+    description: str = Field("A list without a specific sequence", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.list, frozen=True)
+    allowed_tags: List[str] = Field(["ul"], frozen=True)
+
+
+class DefinitionList(OntologyElement):
+    description: str = Field("A list of terms and their definitions", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.list, frozen=True)
+    allowed_tags: List[str] = Field(["dl"], frozen=True)
+
+
+class ListItem(OntologyElement):
+    description: str = Field("An item in a list", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.list, frozen=True)
+    allowed_tags: List[str] = Field(["li"], frozen=True)
+
+
+class Table(OntologyElement):
+    description: str = Field("A structured set of data", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.table, frozen=True)
+    allowed_tags: List[str] = Field(["table"], frozen=True)
+
+    def to_html(self, add_children: bool = True) -> str:
+        soup = BeautifulSoup(super().to_html(add_children), "html.parser")
+        soup = remove_ids_and_class_from_table(soup)
+        return str(soup)
+
+
+class TableBody(OntologyElement):
+    description: str = Field("A body of the table", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.table, frozen=True)
+    allowed_tags: List[str] = Field(["tbody"], frozen=True)
+
+
+class TableHeader(OntologyElement):
+    description: str = Field("A header of the table", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.table, frozen=True)
+    allowed_tags: List[str] = Field(["thead"], frozen=True)
+
+
+class TableRow(OntologyElement):
+    description: str = Field("A row in a table", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.table, frozen=True)
+    allowed_tags: List[str] = Field(["tr"], frozen=True)
+
+
+class TableCell(OntologyElement):
+    description: str = Field("A cell in a table", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.table, frozen=True)
+    allowed_tags: List[str] = Field(["td"], frozen=True)
+
+
+# Note(Pluto): Renamed from TableCellHeader to TableHeaderCell to be consistent with TableCell
+class TableCellHeader(OntologyElement):
+    description: str = Field("A header cell in a table", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.table, frozen=True)
+    allowed_tags: List[str] = Field(["th"], frozen=True)
+
+
+class Image(OntologyElement):
+    description: str = Field("A visual representation", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.media, frozen=True)
+    allowed_tags: List[str] = Field(["img"], frozen=True)
+
+
+class Figure(OntologyElement):
+    description: str = Field("An illustration or diagram in a document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.media, frozen=True)
+    allowed_tags: List[str] = Field(["figure"], frozen=True)
+
+
+class Video(OntologyElement):
+    description: str = Field("A moving visual media element", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.media, frozen=True)
+    allowed_tags: List[str] = Field(["video"], frozen=True)
+
+
+class Audio(OntologyElement):
+    description: str = Field("A sound or music element", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.media, frozen=True)
+    allowed_tags: List[str] = Field(["audio"], frozen=True)
+
+
+class Barcode(OntologyElement):
+    description: str = Field("A machine-readable representation of data", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.media, frozen=True)
+    allowed_tags: List[str] = Field(["img"], frozen=True)
+
+
+class QRCode(OntologyElement):
+    description: str = Field("A two-dimensional barcode", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.media, frozen=True)
+    allowed_tags: List[str] = Field(["img"], frozen=True)
+
+
+class Logo(OntologyElement):
+    description: str = Field("A graphical representation of a company or brand", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.media, frozen=True)
+    allowed_tags: List[str] = Field(["img"], frozen=True)
+
+
+class CodeBlock(OntologyElement):
+    description: str = Field("A block of programming code", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.code, frozen=True)
+    allowed_tags: List[str] = Field(["pre", "code"], frozen=True)
+
+
+class InlineCode(OntologyElement):
+    description: str = Field("Code within a line of text", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.code, frozen=True)
+    allowed_tags: List[str] = Field(["code"], frozen=True)
+
+
+class Formula(OntologyElement):
+    description: str = Field("A mathematical formula", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.mathematical, frozen=True)
+    allowed_tags: List[str] = Field(["math"], frozen=True)
+
+
+class Equation(OntologyElement):
+    description: str = Field("A mathematical equation", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.mathematical, frozen=True)
+    allowed_tags: List[str] = Field(["math"], frozen=True)
+
+
+class FootnoteReference(OntologyElement):
+    description: str = Field(
+        "A subscripted reference to a note at the bottom of a page", frozen=True
+    )
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.reference, frozen=True)
+    allowed_tags: List[str] = Field(["sub"], frozen=True)
+
+
+class Citation(OntologyElement):
+    description: str = Field("A reference to a source", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.reference, frozen=True)
+    allowed_tags: List[str] = Field(["cite"], frozen=True)
+
+
+class Bibliography(OntologyElement):
+    description: str = Field("A list of sources", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.reference, frozen=True)
+    allowed_tags: List[str] = Field(["ul"], frozen=True)
+
+
+class Glossary(OntologyElement):
+    description: str = Field("A list of terms and their definitions", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.reference, frozen=True)
+    allowed_tags: List[str] = Field(["dl"], frozen=True)
+
+
+class Author(OntologyElement):
+    description: str = Field("The creator of the document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.metadata, frozen=True)
+    allowed_tags: List[str] = Field(["meta"], frozen=True)
+
+
+class MetaDate(OntologyElement):
+    description: str = Field("The date associated with the document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.metadata, frozen=True)
+    allowed_tags: List[str] = Field(["meta"], frozen=True)
+
+
+class Keywords(OntologyElement):
+    description: str = Field("Key terms associated with the document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.metadata, frozen=True)
+    allowed_tags: List[str] = Field(["meta"], frozen=True)
+
+
+class Abstract(OntologyElement):
+    description: str = Field("A summary of the document", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.metadata, frozen=True)
+    allowed_tags: List[str] = Field(["section"], frozen=True)
+
+
+class Hyperlink(OntologyElement):
+    description: str = Field("A reference to data that can be directly followed", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.navigation, frozen=True)
+    allowed_tags: List[str] = Field(["a"], frozen=True)
+
+
+class TableOfContents(OntologyElement):
+    description: str = Field(
+        "A list of the document's contents. Total table columns will be "
+        "equal to the degree of hierarchy (n) plus 1 for the target value. "
+        "Header Row: L1,L2,...Ln,Value",
+        frozen=True,
+    )
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.table, frozen=True)
+    allowed_tags: List[str] = Field(["table"], frozen=True)
+
+    def to_html(self, add_children: bool = True) -> str:
+        soup = BeautifulSoup(super().to_html(add_children), "html.parser")
+        soup = remove_ids_and_class_from_table(soup)
+        return str(soup)
+
+
+class Index(OntologyElement):
+    description: str = Field("An alphabetical list of terms and their page numbers", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.navigation, frozen=True)
+    allowed_tags: List[str] = Field(["nav"], frozen=True)
+
+
+class Form(OntologyElement):
+    description: str = Field("A document section with interactive controls", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.form, frozen=True)
+    allowed_tags: List[str] = Field(["form"], frozen=True)
+
+
+class FormField(OntologyElement):
+    description: str = Field("A property value of a form", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.form, frozen=True)
+    allowed_tags: List[str] = Field(["label"], frozen=True)
+
+
+class FormFieldValue(OntologyElement):
+    description: str = Field("A field for user input", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.form, frozen=True)
+    allowed_tags: List[str] = Field(["input"], frozen=True)
+
+    def to_text(self, add_children: bool = True, add_img_alt_text: bool = True) -> str:
+        text = super().to_text(add_children, add_img_alt_text)
+        value = self.additional_attributes.get("value", "")
+        if not value:
+            return text
+        return f"{text} {value}".strip()
+
+
+class Checkbox(OntologyElement):
+    description: str = Field("A small box that can be checked or unchecked", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.form, frozen=True)
+    allowed_tags: List[str] = Field(["input"], frozen=True)
+
+
+class RadioButton(OntologyElement):
+    description: str = Field("A circular button that can be selected", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.form, frozen=True)
+    allowed_tags: List[str] = Field(["input"], frozen=True)
+
+
+class Button(OntologyElement):
+    description: str = Field("An interactive button element", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.form, frozen=True)
+    allowed_tags: List[str] = Field(["button"], frozen=True)
+
+
+class Comment(OntologyElement):
+    description: str = Field("A note or remark", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.annotation, frozen=True)
+    allowed_tags: List[str] = Field(["span"], frozen=True)
+
+
+class Highlight(OntologyElement):
+    description: str = Field("Emphasized text or section", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.annotation, frozen=True)
+    allowed_tags: List[str] = Field(["mark"], frozen=True)
+
+
+class RevisionInsertion(OntologyElement):
+    description: str = Field("A changed or edited element", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.annotation, frozen=True)
+    allowed_tags: List[str] = Field(["ins"], frozen=True)
+
+
+class RevisionDeletion(OntologyElement):
+    description: str = Field("A changed or edited element", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.annotation, frozen=True)
+    allowed_tags: List[str] = Field(["del"], frozen=True)
+
+
+class Address(OntologyElement):
+    description: str = Field("A physical location", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.specialized_text, frozen=True)
+    allowed_tags: List[str] = Field(["address"], frozen=True)
+
+
+class EmailAddress(OntologyElement):
+    description: str = Field("An email address", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.specialized_text, frozen=True)
+    allowed_tags: List[str] = Field(["a"], frozen=True)
+
+
+class PhoneNumber(OntologyElement):
+    description: str = Field("A telephone number", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.specialized_text, frozen=True)
+    allowed_tags: List[str] = Field(["span"], frozen=True)
+
+
+class CalendarDate(OntologyElement):
+    description: str = Field("A calendar date", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.specialized_text, frozen=True)
+    allowed_tags: List[str] = Field(["time"], frozen=True)
+
+
+class Time(OntologyElement):
+    description: str = Field("A specific time", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.specialized_text, frozen=True)
+    allowed_tags: List[str] = Field(["time"], frozen=True)
+
+
+class Currency(OntologyElement):
+    description: str = Field("A monetary value", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.specialized_text, frozen=True)
+    allowed_tags: List[str] = Field(["span"], frozen=True)
+
+
+class Measurement(OntologyElement):
+    description: str = Field("A quantitative value with units", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.specialized_text, frozen=True)
+    allowed_tags: List[str] = Field(["span"], frozen=True)
+
+
+class Letterhead(OntologyElement):
+    description: str = Field("The heading at the top of a letter", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.document_specific, frozen=True)
+    allowed_tags: List[str] = Field(["header"], frozen=True)
+
+
+class Signature(OntologyElement):
+    description: str = Field("A person's name written in a distinctive way", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.document_specific, frozen=True)
+    allowed_tags: List[str] = Field(["img", "svg"], frozen=True)
+
+
+class Watermark(OntologyElement):
+    description: str = Field("A faint design made in paper during manufacture", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.document_specific, frozen=True)
+    allowed_tags: List[str] = Field(["div"], frozen=True)
+
+
+class Stamp(OntologyElement):
+    description: str = Field("An official mark or seal", frozen=True)
+    elementType: ElementTypeEnum = Field(ElementTypeEnum.document_specific, frozen=True)
+    allowed_tags: List[str] = Field(["img", "svg"], frozen=True)

unstructured/embed/README.md

@@ -0,0 +1,6 @@
+# Embed
+![Project unmaintained](https://img.shields.io/badge/project-unmaintained-red.svg)
+
+Project has been moved to: [Unstructured Ingest](https://github.com/Unstructured-IO/unstructured-ingest)
+
+This python module will be removed from this repo in the near future.

unstructured/embed/__init__.py

@@ -0,0 +1,27 @@
+import warnings
+
+from unstructured.embed.bedrock import BedrockEmbeddingEncoder
+from unstructured.embed.huggingface import HuggingFaceEmbeddingEncoder
+from unstructured.embed.mixedbreadai import MixedbreadAIEmbeddingEncoder
+from unstructured.embed.octoai import OctoAIEmbeddingEncoder
+from unstructured.embed.openai import OpenAIEmbeddingEncoder
+from unstructured.embed.vertexai import VertexAIEmbeddingEncoder
+from unstructured.embed.voyageai import VoyageAIEmbeddingEncoder
+
+EMBEDDING_PROVIDER_TO_CLASS_MAP = {
+    "langchain-openai": OpenAIEmbeddingEncoder,
+    "langchain-huggingface": HuggingFaceEmbeddingEncoder,
+    "langchain-aws-bedrock": BedrockEmbeddingEncoder,
+    "langchain-vertexai": VertexAIEmbeddingEncoder,
+    "voyageai": VoyageAIEmbeddingEncoder,
+    "mixedbread-ai": MixedbreadAIEmbeddingEncoder,
+    "octoai": OctoAIEmbeddingEncoder,
+}
+
+
+warnings.warn(
+    "unstructured.ingest will be removed in a future version. "
+    "Functionality moved to the unstructured-ingest project.",
+    DeprecationWarning,
+    stacklevel=2,
+)

unstructured/embed/bedrock.py

@@ -0,0 +1,76 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List
+
+import numpy as np
+from pydantic import SecretStr
+
+from unstructured.documents.elements import (
+    Element,
+)
+from unstructured.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
+from unstructured.utils import requires_dependencies
+
+if TYPE_CHECKING:
+    from langchain_community.embeddings import BedrockEmbeddings
+
+
+class BedrockEmbeddingConfig(EmbeddingConfig):
+    aws_access_key_id: SecretStr
+    aws_secret_access_key: SecretStr
+    region_name: str = "us-west-2"
+
+    @requires_dependencies(
+        ["boto3", "numpy", "langchain_community"],
+        extras="bedrock",
+    )
+    def get_client(self) -> "BedrockEmbeddings":
+        # delay import only when needed
+        import boto3
+        from langchain_community.embeddings import BedrockEmbeddings
+
+        bedrock_runtime = boto3.client(
+            service_name="bedrock-runtime",
+            aws_access_key_id=self.aws_access_key_id.get_secret_value(),
+            aws_secret_access_key=self.aws_secret_access_key.get_secret_value(),
+            region_name=self.region_name,
+        )
+
+        bedrock_client = BedrockEmbeddings(client=bedrock_runtime)
+        return bedrock_client
+
+
+@dataclass
+class BedrockEmbeddingEncoder(BaseEmbeddingEncoder):
+    config: BedrockEmbeddingConfig
+
+    def get_exemplary_embedding(self) -> List[float]:
+        return self.embed_query(query="Q")
+
+    def __post_init__(self):
+        self.initialize()
+
+    def num_of_dimensions(self):
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.shape(exemplary_embedding)
+
+    def is_unit_vector(self):
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
+
+    def embed_query(self, query):
+        bedrock_client = self.config.get_client()
+        return np.array(bedrock_client.embed_query(query))
+
+    def embed_documents(self, elements: List[Element]) -> List[Element]:
+        bedrock_client = self.config.get_client()
+        embeddings = bedrock_client.embed_documents([str(e) for e in elements])
+        elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
+        return elements_with_embeddings
+
+    def _add_embeddings_to_elements(self, elements, embeddings) -> List[Element]:
+        assert len(elements) == len(embeddings)
+        elements_w_embedding = []
+        for i, element in enumerate(elements):
+            element.embeddings = embeddings[i]
+            elements_w_embedding.append(element)
+        return elements

unstructured/embed/huggingface.py

@@ -0,0 +1,67 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List, Optional
+
+import numpy as np
+from pydantic import Field
+
+from unstructured.documents.elements import (
+    Element,
+)
+from unstructured.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
+from unstructured.utils import requires_dependencies
+
+if TYPE_CHECKING:
+    from langchain_huggingface.embeddings import HuggingFaceEmbeddings
+
+
+class HuggingFaceEmbeddingConfig(EmbeddingConfig):
+    model_name: Optional[str] = Field(default="sentence-transformers/all-MiniLM-L6-v2")
+    model_kwargs: Optional[dict] = Field(default_factory=lambda: {"device": "cpu"})
+    encode_kwargs: Optional[dict] = Field(default_factory=lambda: {"normalize_embeddings": False})
+    cache_folder: Optional[dict] = Field(default=None)
+
+    @requires_dependencies(
+        ["langchain_huggingface"],
+        extras="embed-huggingface",
+    )
+    def get_client(self) -> "HuggingFaceEmbeddings":
+        """Creates a langchain Huggingface python client to embed elements."""
+        from langchain_huggingface.embeddings import HuggingFaceEmbeddings
+
+        client = HuggingFaceEmbeddings(**self.dict())
+        return client
+
+
+@dataclass
+class HuggingFaceEmbeddingEncoder(BaseEmbeddingEncoder):
+    config: HuggingFaceEmbeddingConfig
+
+    def get_exemplary_embedding(self) -> List[float]:
+        return self.embed_query(query="Q")
+
+    def num_of_dimensions(self):
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.shape(exemplary_embedding)
+
+    def is_unit_vector(self):
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
+
+    def embed_query(self, query):
+        client = self.config.get_client()
+        return client.embed_query(str(query))
+
+    def embed_documents(self, elements: List[Element]) -> List[Element]:
+        client = self.config.get_client()
+        embeddings = client.embed_documents([str(e) for e in elements])
+        elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
+        return elements_with_embeddings
+
+    def _add_embeddings_to_elements(self, elements, embeddings) -> List[Element]:
+        assert len(elements) == len(embeddings)
+        elements_w_embedding = []
+
+        for i, element in enumerate(elements):
+            element.embeddings = embeddings[i]
+            elements_w_embedding.append(element)
+        return elements

unstructured/embed/interfaces.py

@@ -0,0 +1,39 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import List, Tuple
+
+from pydantic import BaseModel
+
+from unstructured.documents.elements import Element
+
+
+class EmbeddingConfig(BaseModel):
+    pass
+
+
+@dataclass
+class BaseEmbeddingEncoder(ABC):
+    config: EmbeddingConfig
+
+    @abstractmethod
+    def initialize(self):
+        """Initializes the embedding encoder class. Should also validate the instance
+        is properly configured: e.g., embed a single a element"""
+
+    @property
+    @abstractmethod
+    def num_of_dimensions(self) -> Tuple[int]:
+        """Number of dimensions for the embedding vector."""
+
+    @property
+    @abstractmethod
+    def is_unit_vector(self) -> bool:
+        """Denotes if the embedding vector is a unit vector."""
+
+    @abstractmethod
+    def embed_documents(self, elements: List[Element]) -> List[Element]:
+        pass
+
+    @abstractmethod
+    def embed_query(self, query: str) -> List[float]:
+        pass

unstructured/embed/mixedbreadai.py

@@ -0,0 +1,178 @@
+import os
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, List, Optional
+
+import numpy as np
+from pydantic import Field, SecretStr
+
+from unstructured.documents.elements import Element
+from unstructured.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
+from unstructured.utils import requires_dependencies
+
+USER_AGENT = "@mixedbread-ai/unstructured"
+BATCH_SIZE = 128
+TIMEOUT = 60
+MAX_RETRIES = 3
+ENCODING_FORMAT = "float"
+TRUNCATION_STRATEGY = "end"
+
+
+if TYPE_CHECKING:
+    from mixedbread_ai.client import MixedbreadAI
+    from mixedbread_ai.core import RequestOptions
+
+
+class MixedbreadAIEmbeddingConfig(EmbeddingConfig):
+    """
+    Configuration class for Mixedbread AI Embedding Encoder.
+
+    Attributes:
+        api_key (str): API key for accessing Mixedbread AI..
+        model_name (str): Name of the model to use for embeddings.
+    """
+
+    api_key: SecretStr = Field(
+        default_factory=lambda: SecretStr(os.environ.get("MXBAI_API_KEY")),
+    )
+
+    model_name: str = Field(
+        default="mixedbread-ai/mxbai-embed-large-v1",
+    )
+
+    @requires_dependencies(
+        ["mixedbread_ai"],
+        extras="embed-mixedbreadai",
+    )
+    def get_client(self) -> "MixedbreadAI":
+        """
+        Create the Mixedbread AI client.
+
+        Returns:
+            MixedbreadAI: Initialized client.
+        """
+        from mixedbread_ai.client import MixedbreadAI
+
+        return MixedbreadAI(
+            api_key=self.api_key.get_secret_value(),
+        )
+
+
+@dataclass
+class MixedbreadAIEmbeddingEncoder(BaseEmbeddingEncoder):
+    """
+    Embedding encoder for Mixedbread AI.
+
+    Attributes:
+        config (MixedbreadAIEmbeddingConfig): Configuration for the embedding encoder.
+    """
+
+    config: MixedbreadAIEmbeddingConfig
+
+    _exemplary_embedding: Optional[List[float]] = field(init=False, default=None)
+    _request_options: Optional["RequestOptions"] = field(init=False, default=None)
+
+    def get_exemplary_embedding(self) -> List[float]:
+        """Get an exemplary embedding to determine dimensions and unit vector status."""
+        return self._embed(["Q"])[0]
+
+    def initialize(self):
+        if self.config.api_key is None:
+            raise ValueError(
+                "The Mixedbread AI API key must be specified."
+                + "You either pass it in the constructor using 'api_key'"
+                + "or via the 'MXBAI_API_KEY' environment variable."
+            )
+
+        from mixedbread_ai.core import RequestOptions
+
+        self._request_options = RequestOptions(
+            max_retries=MAX_RETRIES,
+            timeout_in_seconds=TIMEOUT,
+            additional_headers={"User-Agent": USER_AGENT},
+        )
+
+    @property
+    def num_of_dimensions(self):
+        """Get the number of dimensions for the embeddings."""
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.shape(exemplary_embedding)
+
+    @property
+    def is_unit_vector(self) -> bool:
+        """Check if the embedding is a unit vector."""
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
+
+    def _embed(self, texts: List[str]) -> List[List[float]]:
+        """
+        Embed a list of texts using the Mixedbread AI API.
+
+        Args:
+            texts (List[str]): List of texts to embed.
+
+        Returns:
+            List[List[float]]: List of embeddings.
+        """
+        batch_size = BATCH_SIZE
+        batch_itr = range(0, len(texts), batch_size)
+
+        responses = []
+        client = self.config.get_client()
+        for i in batch_itr:
+            batch = texts[i : i + batch_size]
+            response = client.embeddings(
+                model=self.config.model_name,
+                normalized=True,
+                encoding_format=ENCODING_FORMAT,
+                truncation_strategy=TRUNCATION_STRATEGY,
+                request_options=self._request_options,
+                input=batch,
+            )
+            responses.append(response)
+        return [item.embedding for response in responses for item in response.data]
+
+    @staticmethod
+    def _add_embeddings_to_elements(
+        elements: List[Element], embeddings: List[List[float]]
+    ) -> List[Element]:
+        """
+        Add embeddings to elements.
+
+        Args:
+            elements (List[Element]): List of elements.
+            embeddings (List[List[float]]): List of embeddings.
+
+        Returns:
+            List[Element]: Elements with embeddings added.
+        """
+        assert len(elements) == len(embeddings)
+        elements_w_embedding = []
+        for i, element in enumerate(elements):
+            element.embeddings = embeddings[i]
+            elements_w_embedding.append(element)
+        return elements
+
+    def embed_documents(self, elements: List[Element]) -> List[Element]:
+        """
+        Embed a list of document elements.
+
+        Args:
+            elements (List[Element]): List of document elements.
+
+        Returns:
+            List[Element]: Elements with embeddings.
+        """
+        embeddings = self._embed([str(e) for e in elements])
+        return self._add_embeddings_to_elements(elements, embeddings)
+
+    def embed_query(self, query: str) -> List[float]:
+        """
+        Embed a query string.
+
+        Args:
+            query (str): Query string to embed.
+
+        Returns:
+            List[float]: Embedding of the query.
+        """
+        return self._embed([query])[0]

unstructured/embed/octoai.py

@@ -0,0 +1,69 @@
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, List, Optional
+
+import numpy as np
+from pydantic import Field, SecretStr
+
+from unstructured.documents.elements import (
+    Element,
+)
+from unstructured.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
+from unstructured.utils import requires_dependencies
+
+if TYPE_CHECKING:
+    from openai import OpenAI
+
+
+class OctoAiEmbeddingConfig(EmbeddingConfig):
+    api_key: SecretStr
+    model_name: str = Field(default="thenlper/gte-large")
+    base_url: str = Field(default="https://text.octoai.run/v1")
+
+    @requires_dependencies(
+        ["openai", "tiktoken"],
+        extras="embed-octoai",
+    )
+    def get_client(self) -> "OpenAI":
+        """Creates an OpenAI python client to embed elements. Uses the OpenAI SDK."""
+        from openai import OpenAI
+
+        return OpenAI(api_key=self.api_key.get_secret_value(), base_url=self.base_url)
+
+
+@dataclass
+class OctoAIEmbeddingEncoder(BaseEmbeddingEncoder):
+    config: OctoAiEmbeddingConfig
+    # Uses the OpenAI SDK
+    _exemplary_embedding: Optional[List[float]] = field(init=False, default=None)
+
+    def get_exemplary_embedding(self) -> List[float]:
+        return self.embed_query("Q")
+
+    def initialize(self):
+        pass
+
+    def num_of_dimensions(self):
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.shape(exemplary_embedding)
+
+    def is_unit_vector(self):
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
+
+    def embed_query(self, query):
+        client = self.config.get_client()
+        response = client.embeddings.create(input=str(query), model=self.config.model_name)
+        return response.data[0].embedding
+
+    def embed_documents(self, elements: List[Element]) -> List[Element]:
+        embeddings = [self.embed_query(e) for e in elements]
+        elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
+        return elements_with_embeddings
+
+    def _add_embeddings_to_elements(self, elements, embeddings) -> List[Element]:
+        assert len(elements) == len(embeddings)
+        elements_w_embedding = []
+        for i, element in enumerate(elements):
+            element.embeddings = embeddings[i]
+            elements_w_embedding.append(element)
+        return elements

unstructured/embed/openai.py

@@ -0,0 +1,67 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List
+
+import numpy as np
+from pydantic import Field, SecretStr
+
+from unstructured.documents.elements import (
+    Element,
+)
+from unstructured.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
+from unstructured.utils import requires_dependencies
+
+if TYPE_CHECKING:
+    from langchain_openai.embeddings import OpenAIEmbeddings
+
+
+class OpenAIEmbeddingConfig(EmbeddingConfig):
+    api_key: SecretStr
+    model_name: str = Field(default="text-embedding-ada-002")
+
+    @requires_dependencies(["langchain_openai"], extras="openai")
+    def get_client(self) -> "OpenAIEmbeddings":
+        """Creates a langchain OpenAI python client to embed elements."""
+        from langchain_openai import OpenAIEmbeddings
+
+        openai_client = OpenAIEmbeddings(
+            openai_api_key=self.api_key.get_secret_value(),
+            model=self.model_name,  # type: ignore
+        )
+        return openai_client
+
+
+@dataclass
+class OpenAIEmbeddingEncoder(BaseEmbeddingEncoder):
+    config: OpenAIEmbeddingConfig
+
+    def get_exemplary_embedding(self) -> List[float]:
+        return self.embed_query(query="Q")
+
+    def initialize(self):
+        pass
+
+    def num_of_dimensions(self):
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.shape(exemplary_embedding)
+
+    def is_unit_vector(self):
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
+
+    def embed_query(self, query):
+        client = self.config.get_client()
+        return client.embed_query(str(query))
+
+    def embed_documents(self, elements: List[Element]) -> List[Element]:
+        client = self.config.get_client()
+        embeddings = client.embed_documents([str(e) for e in elements])
+        elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
+        return elements_with_embeddings
+
+    def _add_embeddings_to_elements(self, elements, embeddings) -> List[Element]:
+        assert len(elements) == len(embeddings)
+        elements_w_embedding = []
+        for i, element in enumerate(elements):
+            element.embeddings = embeddings[i]
+            elements_w_embedding.append(element)
+        return elements

unstructured/embed/vertexai.py

@@ -0,0 +1,76 @@
+# type: ignore
+import json
+import os
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List, Optional
+
+import numpy as np
+from pydantic import Field, SecretStr
+
+from unstructured.documents.elements import (
+    Element,
+)
+from unstructured.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
+from unstructured.utils import FileHandler, requires_dependencies
+
+if TYPE_CHECKING:
+    from langchain_google_vertexai import VertexAIEmbeddings
+
+
+class VertexAIEmbeddingConfig(EmbeddingConfig):
+    api_key: SecretStr
+    model_name: Optional[str] = Field(default="textembedding-gecko@001")
+
+    def register_application_credentials(self):
+        application_credentials_path = os.path.join("/tmp", "google-vertex-app-credentials.json")
+        credentials_file = FileHandler(application_credentials_path)
+        credentials_file.write_file(json.dumps(json.loads(self.api_key.get_secret_value())))
+        os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = application_credentials_path
+
+    @requires_dependencies(
+        ["langchain", "langchain_google_vertexai"],
+        extras="embed-vertexai",
+    )
+    def get_client(self) -> "VertexAIEmbeddings":
+        """Creates a Langchain VertexAI python client to embed elements."""
+        from langchain_google_vertexai import VertexAIEmbeddings
+
+        self.register_application_credentials()
+        vertexai_client = VertexAIEmbeddings(model_name=self.model_name)
+        return vertexai_client
+
+
+@dataclass
+class VertexAIEmbeddingEncoder(BaseEmbeddingEncoder):
+    config: VertexAIEmbeddingConfig
+
+    def get_exemplary_embedding(self) -> List[float]:
+        return self.embed_query(query="A sample query.")
+
+    def initialize(self):
+        pass
+
+    def num_of_dimensions(self):
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.shape(exemplary_embedding)
+
+    def is_unit_vector(self):
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
+
+    def embed_query(self, query):
+        client = self.config.get_client()
+        result = client.embed_query(str(query))
+        return result
+
+    def embed_documents(self, elements: List[Element]) -> List[Element]:
+        client = self.config.get_client()
+        embeddings = client.embed_documents([str(e) for e in elements])
+        elements_with_embeddings = self._add_embeddings_to_elements(elements, embeddings)
+        return elements_with_embeddings
+
+    def _add_embeddings_to_elements(self, elements, embeddings) -> List[Element]:
+        assert len(elements) == len(embeddings)
+        for element, embedding in zip(elements, embeddings):
+            element.embeddings = embedding
+        return elements

unstructured/embed/voyageai.py

@@ -0,0 +1,237 @@
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Iterable, List, Optional
+
+import numpy as np
+from pydantic import Field, SecretStr
+
+from unstructured.documents.elements import Element
+from unstructured.embed.interfaces import BaseEmbeddingEncoder, EmbeddingConfig
+from unstructured.utils import requires_dependencies
+
+if TYPE_CHECKING:
+    from voyageai import Client
+
+# Token limits for different VoyageAI models
+VOYAGE_TOTAL_TOKEN_LIMITS = {
+    "voyage-context-3": 32_000,
+    "voyage-3.5-lite": 1_000_000,
+    "voyage-3.5": 320_000,
+    "voyage-2": 320_000,
+    "voyage-02": 320_000,
+    "voyage-3-large": 120_000,
+    "voyage-code-3": 120_000,
+    "voyage-large-2-instruct": 120_000,
+    "voyage-finance-2": 120_000,
+    "voyage-multilingual-2": 120_000,
+    "voyage-law-2": 120_000,
+    "voyage-large-2": 120_000,
+    "voyage-3": 120_000,
+    "voyage-3-lite": 120_000,
+    "voyage-code-2": 120_000,
+    "voyage-3-m-exp": 120_000,
+    "voyage-multimodal-3": 120_000,
+}
+
+# Batch size for embedding requests (max documents per batch)
+MAX_BATCH_SIZE = 1000
+
+
+class VoyageAIEmbeddingConfig(EmbeddingConfig):
+    api_key: SecretStr
+    model_name: str
+    show_progress_bar: bool = False
+    batch_size: Optional[int] = Field(default=None)
+    truncation: Optional[bool] = Field(default=None)
+    output_dimension: Optional[int] = Field(default=None)
+
+    @requires_dependencies(
+        ["voyageai"],
+        extras="embed-voyageai",
+    )
+    def get_client(self) -> "Client":
+        """Creates a VoyageAI python client to embed elements."""
+        from voyageai import Client
+
+        return Client(
+            api_key=self.api_key.get_secret_value(),
+        )
+
+    def get_token_limit(self) -> int:
+        """Get the token limit for the current model."""
+        return VOYAGE_TOTAL_TOKEN_LIMITS.get(self.model_name, 120_000)
+
+
+@dataclass
+class VoyageAIEmbeddingEncoder(BaseEmbeddingEncoder):
+    config: VoyageAIEmbeddingConfig
+
+    def get_exemplary_embedding(self) -> List[float]:
+        return self.embed_query(query="A sample query.")
+
+    def initialize(self):
+        pass
+
+    @property
+    def num_of_dimensions(self) -> tuple[int, ...]:
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.shape(exemplary_embedding)
+
+    @property
+    def is_unit_vector(self) -> bool:
+        exemplary_embedding = self.get_exemplary_embedding()
+        return np.isclose(np.linalg.norm(exemplary_embedding), 1.0)
+
+    def _is_context_model(self) -> bool:
+        """Check if the model is a contextualized embedding model."""
+        return "context" in self.config.model_name
+
+    def _build_batches(self, texts: List[str], client: "Client") -> Iterable[List[str]]:
+        """
+        Generate batches of texts based on token limits.
+
+        Args:
+            texts: List of texts to batch.
+            client: VoyageAI client instance to use for tokenization.
+
+        Yields:
+            Batches of texts as lists.
+        """
+        if not texts:
+            return
+
+        max_tokens_per_batch = self.config.get_token_limit()
+        current_batch: List[str] = []
+        current_batch_tokens = 0
+
+        # Tokenize all texts in one API call
+        all_token_lists = client.tokenize(texts, model=self.config.model_name)
+        token_counts = [len(tokens) for tokens in all_token_lists]
+
+        for i, text in enumerate(texts):
+            n_tokens = token_counts[i]
+
+            # Check if adding this text would exceed limits
+            if current_batch and (
+                len(current_batch) >= MAX_BATCH_SIZE
+                or (current_batch_tokens + n_tokens > max_tokens_per_batch)
+            ):
+                # Yield the current batch and start a new one
+                yield current_batch
+                current_batch = []
+                current_batch_tokens = 0
+
+            current_batch.append(text)
+            current_batch_tokens += n_tokens
+
+        # Yield the last batch (always has at least one text)
+        if current_batch:
+            yield current_batch
+
+    def _embed_batch(
+        self, batch: List[str], client: "Client", input_type: str = "document"
+    ) -> List[List[float]]:
+        """
+        Embed a batch of texts using the appropriate method for the model.
+
+        Args:
+            batch: List of texts to embed.
+            client: VoyageAI client instance to use for embedding.
+            input_type: Type of input ("document" or "query").
+
+        Returns:
+            List of embedding vectors.
+        """
+        if self._is_context_model():
+            result = client.contextualized_embed(
+                inputs=[batch],
+                model=self.config.model_name,
+                input_type=input_type,
+                output_dimension=self.config.output_dimension,
+            )
+            return [list(emb) for emb in result.results[0].embeddings]
+        else:
+            result = client.embed(
+                texts=batch,
+                model=self.config.model_name,
+                input_type=input_type,
+                truncation=self.config.truncation,
+                output_dimension=self.config.output_dimension,
+            )
+            return [list(emb) for emb in result.embeddings]
+
+    def embed_documents(self, elements: List[Element]) -> List[Element]:
+        """
+        Embed documents with automatic batching based on token limits.
+
+        Args:
+            elements: List of elements to embed.
+
+        Returns:
+            List of elements with embeddings added.
+        """
+        if not elements:
+            return []
+
+        client = self.config.get_client()
+        texts = [str(e) for e in elements]
+        all_embeddings: List[List[float]] = []
+
+        # Process each batch
+        batches = list(self._build_batches(texts, client))
+
+        if self.config.show_progress_bar:
+            try:
+                from tqdm.auto import tqdm  # type: ignore
+
+                batches = tqdm(batches, desc="Embedding batches")
+            except ImportError as e:
+                raise ImportError(
+                    "Must have tqdm installed if `show_progress_bar` is set to True. "
+                    "Please install with `pip install tqdm`."
+                ) from e
+
+        for batch in batches:
+            batch_embeddings = self._embed_batch(batch, client, input_type="document")
+            all_embeddings.extend(batch_embeddings)
+
+        return self._add_embeddings_to_elements(elements, all_embeddings)
+
+    def embed_query(self, query: str) -> List[float]:
+        """
+        Embed a single query string.
+
+        Args:
+            query: Query string to embed.
+
+        Returns:
+            Embedding vector.
+        """
+        client = self.config.get_client()
+        batch_embeddings = self._embed_batch([query], client, input_type="query")
+        return batch_embeddings[0]
+
+    def count_tokens(self, texts: List[str]) -> List[int]:
+        """
+        Count tokens for the given texts.
+
+        Args:
+            texts: List of texts to count tokens for.
+
+        Returns:
+            List of token counts for each text.
+        """
+        if not texts:
+            return []
+
+        client = self.config.get_client()
+        token_lists = client.tokenize(texts, model=self.config.model_name)
+        return [len(token_list) for token_list in token_lists]
+
+    @staticmethod
+    def _add_embeddings_to_elements(elements, embeddings) -> List[Element]:
+        assert len(elements) == len(embeddings)
+        elements_w_embedding = []
+        for i, element in enumerate(elements):
+            element.embeddings = embeddings[i]
+            elements_w_embedding.append(element)
+        return elements

unstructured/errors.py

@@ -0,0 +1,27 @@
+class PageCountExceededError(ValueError):
+    """Error raised, when number of pages exceeds pdf_hi_res_max_pages limit."""
+
+    def __init__(self, document_pages: int, pdf_hi_res_max_pages: int):
+        self.document_pages = document_pages
+        self.pdf_hi_res_max_pages = pdf_hi_res_max_pages
+        self.message = (
+            f"Maximum number of PDF file pages exceeded - "
+            f"pages={document_pages}, maximum={pdf_hi_res_max_pages}."
+        )
+        super().__init__(self.message)
+
+
+class UnprocessableEntityError(Exception):
+    """Error raised when a file is not valid."""
+
+
+class DecompressedSizeExceededError(ValueError):
+    """Error raised when decompressed data exceeds the maximum size limit."""
+
+    def __init__(self, max_size: int):
+        self.max_size = max_size
+        self.message = (
+            f"Decompressed data exceeds maximum allowed size of {max_size} bytes "
+            f"({max_size / (1024 * 1024):.1f} MB)."
+        )
+        super().__init__(self.message)

unstructured/file_utils/encoding.py

@@ -0,0 +1,150 @@
+from typing import IO, Optional, Tuple, Union
+
+from charset_normalizer import detect
+
+from unstructured.errors import UnprocessableEntityError
+from unstructured.partition.common.common import convert_to_bytes
+
+ENCODE_REC_THRESHOLD = 0.8
+
+# popular encodings from https://en.wikipedia.org/wiki/Popularity_of_text_encodings
+COMMON_ENCODINGS = [
+    "utf_8",
+    "iso_8859_1",
+    "iso_8859_6",
+    "iso_8859_8",
+    "ascii",
+    "big5",
+    "utf_16",
+    "utf_16_be",
+    "utf_16_le",
+    "utf_32",
+    "utf_32_be",
+    "utf_32_le",
+    "euc_jis_2004",
+    "euc_jisx0213",
+    "euc_jp",
+    "euc_kr",
+    "gb18030",
+    "shift_jis",
+    "shift_jis_2004",
+    "shift_jisx0213",
+]
+
+
+def format_encoding_str(encoding: str) -> str:
+    """Format input encoding string (e.g., `utf-8`, `iso-8859-1`, etc).
+    Parameters
+    ----------
+    encoding
+        The encoding string to be formatted (e.g., `UTF-8`, `utf_8`, `ISO-8859-1`, `iso_8859_1`,
+        etc).
+    """
+    formatted_encoding = encoding.lower().replace("_", "-")
+
+    # Special case for Arabic and Hebrew charsets with directional annotations
+    annotated_encodings = ["iso-8859-6-i", "iso-8859-6-e", "iso-8859-8-i", "iso-8859-8-e"]
+    if formatted_encoding in annotated_encodings:
+        formatted_encoding = formatted_encoding[:-2]  # remove the annotation
+
+    return formatted_encoding
+
+
+def validate_encoding(encoding: str) -> bool:
+    """Checks if an encoding string is valid. Helps to avoid errors in cases where
+    invalid encodings are extracted from malformed documents."""
+    for common_encoding in COMMON_ENCODINGS:
+        if format_encoding_str(common_encoding) == format_encoding_str(encoding):
+            return True
+    return False
+
+
+def detect_file_encoding(
+    filename: str = "",
+    file: Optional[Union[bytes, IO[bytes]]] = None,
+) -> Tuple[str, str]:
+    if filename:
+        with open(filename, "rb") as f:
+            byte_data = f.read()
+    elif file:
+        byte_data = convert_to_bytes(file)
+    else:
+        raise FileNotFoundError("No filename nor file were specified")
+
+    result = detect(byte_data)
+    encoding = result["encoding"]
+    confidence = result["confidence"]
+
+    if encoding is None or confidence is None or confidence < ENCODE_REC_THRESHOLD:
+        # Encoding detection failed, fallback to predefined encodings
+        for enc in COMMON_ENCODINGS:
+            try:
+                if filename:
+                    with open(filename, encoding=enc) as f:
+                        file_text = f.read()
+                else:
+                    file_text = byte_data.decode(enc)
+                encoding = enc
+                break
+            except (UnicodeDecodeError, UnicodeError):
+                continue
+        else:
+            # NOTE: Use UnprocessableEntityError instead of UnicodeDecodeError to avoid
+            # logging the entire file content. UnicodeDecodeError automatically stores
+            # the complete input data, which can be problematic for large files.
+            raise UnprocessableEntityError(
+                "Unable to determine file encoding after trying all common encodings. "
+                "File may be corrupted or in an unsupported format."
+            ) from None
+
+    else:
+        # NOTE: Catch UnicodeDecodeError to avoid logging the entire file content.
+        # UnicodeDecodeError automatically stores the complete input data in its
+        # 'object' attribute, which can cause issues with large files in logging
+        # and error reporting systems.
+        try:
+            file_text = byte_data.decode(encoding)
+        except (UnicodeDecodeError, UnicodeError):
+            raise UnprocessableEntityError(
+                f"File encoding detection failed: detected '{encoding}' but decode failed. "
+                f"File may be corrupted or in an unsupported format."
+            ) from None
+
+    formatted_encoding = format_encoding_str(encoding)
+
+    return formatted_encoding, file_text
+
+
+def read_txt_file(
+    filename: str = "",
+    file: Optional[Union[bytes, IO[bytes]]] = None,
+    encoding: Optional[str] = None,
+) -> Tuple[str, str]:
+    """Extracts document metadata from a plain text document."""
+    if filename:
+        if encoding:
+            formatted_encoding = format_encoding_str(encoding)
+            with open(filename, encoding=formatted_encoding) as f:
+                try:
+                    file_text = f.read()
+                except (UnicodeDecodeError, UnicodeError) as error:
+                    raise error
+        else:
+            formatted_encoding, file_text = detect_file_encoding(filename)
+    elif file:
+        if encoding:
+            formatted_encoding = format_encoding_str(encoding)
+            try:
+                file_content = file if isinstance(file, bytes) else file.read()
+                if isinstance(file_content, bytes):
+                    file_text = file_content.decode(formatted_encoding)
+                else:
+                    file_text = file_content
+            except (UnicodeDecodeError, UnicodeError) as error:
+                raise error
+        else:
+            formatted_encoding, file_text = detect_file_encoding(file=file)
+    else:
+        raise FileNotFoundError("No filename was specified")
+
+    return formatted_encoding, file_text

unstructured/file_utils/file_conversion.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import os
+import re
+import tempfile
+from typing import IO
+
+from unstructured.errors import UnprocessableEntityError
+from unstructured.partition.common.common import exactly_one
+from unstructured.utils import requires_dependencies
+
+
+@requires_dependencies(["pypandoc"])
+def convert_file_to_text(filename: str, source_format: str, target_format: str) -> str:
+    """Uses pandoc to convert the source document to a raw text string."""
+    import pypandoc
+
+    try:
+        text: str = pypandoc.convert_file(
+            filename, target_format, format=source_format, sandbox=True
+        )
+    except FileNotFoundError as err:
+        msg = (
+            f"Error converting the file to text. Ensure you have the pandoc package installed on"
+            f" your system. Installation instructions are available at"
+            f" https://pandoc.org/installing.html. The original exception text was:\n{err}"
+        )
+        raise FileNotFoundError(msg)
+    except RuntimeError as err:
+        err_str = str(err)
+        if source_format == "epub" and (
+            "Couldn't extract ePub file" in err_str
+            or "No entry on path" in err_str
+            or re.search(r"exitcode ['\"]?64['\"]?", err_str)
+        ):
+            raise UnprocessableEntityError(f"Invalid EPUB file: {err_str}")
+
+        supported_source_formats, _ = pypandoc.get_pandoc_formats()
+
+        if source_format == "rtf" and source_format not in supported_source_formats:
+            additional_info = (
+                "Support for RTF files is not available in the current pandoc installation. "
+                "It was introduced in pandoc 2.14.2.\n"
+                "Reference: https://pandoc.org/releases.html#pandoc-2.14.2-2021-08-21"
+            )
+        else:
+            additional_info = ""
+
+        msg = (
+            f"{err}\n\n{additional_info}\n\n"
+            f"Current version of pandoc: {pypandoc.get_pandoc_version()}\n"
+            "Make sure you have the right version installed in your system. Please follow the"
+            " pandoc installation instructions in README.md to install the right version."
+        )
+        raise RuntimeError(msg)
+
+    return text
+
+
+def convert_file_to_html_text_using_pandoc(
+    source_format: str, filename: str | None = None, file: IO[bytes] | None = None
+) -> str:
+    """Converts a document to HTML raw text.
+
+    Enables the doucment to be processed using `partition_html()`.
+    """
+    exactly_one(filename=filename, file=file)
+
+    if file is not None:
+        with tempfile.TemporaryDirectory() as temp_dir_path:
+            tmp_file_path = os.path.join(temp_dir_path, f"tmp_file.{source_format}")
+            with open(tmp_file_path, "wb") as tmp_file:
+                tmp_file.write(file.read())
+            return convert_file_to_text(
+                filename=tmp_file_path, source_format=source_format, target_format="html"
+            )
+
+    assert filename is not None
+    return convert_file_to_text(
+        filename=filename, source_format=source_format, target_format="html"
+    )

unstructured/file_utils/filetype.py

@@ -0,0 +1,965 @@
+"""Automatically detect file-type based on inspection of the file's contents.
+
+Auto-detection proceeds via a sequence of strategies. The first strategy to confidently determine a
+file-type returns that value. A strategy that is not applicable, either because it lacks the input
+required or fails to determine a file-type, returns `None` and execution continues with the next
+strategy.
+
+`_FileTypeDetector` is the main object and implements the three strategies.
+
+The three strategies are:
+
+- Use MIME-type asserted by caller in the `content_type` argument.
+- Guess a MIME-type using libmagic, falling back to the `filetype` package when libmagic is
+  unavailable.
+- Map filename-extension to a `FileType` member.
+
+A file that fails all three strategies is assigned the value `FileType.UNK`, for "unknown".
+
+`_FileTypeDetectionContext` encapsulates the various arguments received by `detect_filetype()` and
+provides values derived from them. This object is immutable and can be passed to delegates of
+`_FileTypeDetector` to provide whatever context they need on the current detection instance.
+
+`_FileTypeDetector` delegates to _differentiator_ objects like `_ZipFileDifferentiator` for
+specialized discrimination and/or confirmation of ambiguous or frequently mis-identified
+MIME-types. Additional differentiators are planned, one for `application/x-ole-storage`
+(DOC, PPT, XLS, and MSG file-types) and perhaps others.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import functools
+import importlib.util
+import io
+import json
+import os
+import re
+import tempfile
+import zipfile
+from functools import cached_property
+from typing import IO, Callable, Iterator, Optional, cast
+
+import filetype as ft
+from olefile import OleFileIO
+from oxmsg.storage import Storage
+from typing_extensions import ParamSpec
+
+from unstructured.documents.elements import Element
+from unstructured.file_utils.encoding import detect_file_encoding, format_encoding_str
+from unstructured.file_utils.model import FileType
+from unstructured.logger import logger
+from unstructured.nlp.patterns import EMAIL_HEAD_RE, LIST_OF_DICTS_PATTERN
+from unstructured.partition.common.common import add_element_metadata, exactly_one
+from unstructured.partition.common.metadata import set_element_hierarchy
+from unstructured.utils import get_call_args_applying_defaults
+
+_JSON_DISAMBIGUATION_CHUNK_SIZE = 8192
+_JSON_DISAMBIGUATION_MAX_CHARS = 1024 * 1024
+
+try:
+    importlib.import_module("magic")
+    LIBMAGIC_AVAILABLE = True
+except ImportError:
+    LIBMAGIC_AVAILABLE = False  # pyright: ignore[reportConstantRedefinition]
+
+
+def detect_filetype(
+    file_path: str | None = None,
+    file: IO[bytes] | tempfile.SpooledTemporaryFile | None = None,
+    encoding: str | None = None,
+    content_type: str | None = None,
+    metadata_file_path: Optional[str] = None,
+) -> FileType:
+    """Determine file-type of specified file using libmagic and/or fallback methods.
+
+    One of `file_path` or `file` must be specified. A `file_path` that does not
+    correspond to a file on the filesystem raises `ValueError`.
+
+    Args:
+        content_type: MIME-type of document-source, when already known. Providing
+          a value for this argument disables auto-detection unless it does not map
+          to a FileType member or is ambiguous, in which case it is ignored.
+        encoding: Only used for textual file-types. When omitted, `utf-8` is
+          assumed. Should generally be omitted except to resolve a problem with
+          textual file-types like HTML.
+        metadata_file_path: Only used when `file` is provided and then only as a
+          source for a filename-extension that may be needed as a secondary
+          content-type indicator. Ignored with the document is specified using
+          `file_path`.
+
+    Returns:
+        A member of the `FileType` enumeration, `FileType.UNK` when the file type
+        could not be determined or is not supported.
+
+    Raises:
+        ValueError: when:
+        - `file_path` is specified but does not correspond to a file on the
+          filesystem.
+        - Neither `file_path` nor `file` were specified.
+    """
+    file_buffer = file
+    if isinstance(file, tempfile.SpooledTemporaryFile):
+        file_buffer = io.BytesIO(file.read())
+        file.seek(0)
+
+    ctx = _FileTypeDetectionContext.new(
+        file_path=file_path,
+        file=file_buffer,
+        encoding=encoding,
+        content_type=content_type,
+        metadata_file_path=metadata_file_path,
+    )
+    return _FileTypeDetector.file_type(ctx)
+
+
+def is_json_processable(
+    filename: Optional[str] = None,
+    file: Optional[IO[bytes]] = None,
+    file_text: Optional[str] = None,
+    encoding: Optional[str] = "utf-8",
+) -> bool:
+    """True when file looks like a JSON array of objects.
+
+    Uses regex on a file prefix, so not entirely reliable but good enough if you already know the
+    file is JSON.
+    """
+    exactly_one(filename=filename, file=file, file_text=file_text)
+
+    if file_text is None:
+        file_text = _FileTypeDetectionContext.new(
+            file_path=filename, file=file, encoding=encoding
+        ).text_head
+
+    return re.match(LIST_OF_DICTS_PATTERN, file_text) is not None
+
+
+def is_ndjson_processable(
+    filename: Optional[str] = None,
+    file: Optional[IO[bytes]] = None,
+    file_text: Optional[str] = None,
+    encoding: Optional[str] = "utf-8",
+    allow_truncated_single_line: bool = False,
+) -> bool:
+    """True when file looks like newline-delimited JSON objects.
+
+    NDJSON is a sequence of one JSON value per line, conventionally an object on each line. A
+    payload that parses as a single JSON value (e.g. a multi-line `{...}` object or a `[...]`
+    array) is *not* NDJSON and must not be matched here, otherwise `partition_ndjson` will fail
+    later when it splits the text by lines and tries to parse each fragment.
+    """
+    exactly_one(filename=filename, file=file, file_text=file_text)
+
+    allow_truncated = allow_truncated_single_line
+    if file_text is None:
+        file_text, allow_truncated = _FileTypeDetectionContext.new(
+            file_path=filename, file=file, encoding=encoding
+        ).json_disambiguation_text
+
+    text = file_text.lstrip()
+    if not text or not text.startswith("{"):
+        return False
+
+    newline_idx = text.find("\n")
+
+    if newline_idx == -1:
+        # Single-line input. A complete `{...}` parses as a dict and is treated as 1-record
+        # NDJSON (existing tests and `partition_ndjson` rely on this). When the caller knows this
+        # is a truncated first line from a JSON-like payload, a parse failure is still compatible
+        # with a long 1-record NDJSON payload.
+        try:
+            return isinstance(json.loads(text), dict)
+        except json.JSONDecodeError:
+            return allow_truncated
+
+    # Multi-line input. NDJSON requires each record to be on its own line, so the first line
+    # must independently parse as a JSON object. A pretty-printed single JSON object has its
+    # first line be just `{` (or similar fragment) which won't parse alone — that's how we
+    # distinguish it from real NDJSON.
+    first_line = text[:newline_idx].rstrip()
+    if not first_line:
+        return False
+    try:
+        return isinstance(json.loads(first_line), dict)
+    except json.JSONDecodeError:
+        return False
+
+
+class _FileTypeDetector:
+    """Determines file type from a variety of possible inputs."""
+
+    def __init__(self, ctx: _FileTypeDetectionContext):
+        self._ctx = ctx
+
+    @classmethod
+    def file_type(cls, ctx: _FileTypeDetectionContext) -> FileType:
+        """Detect file-type of document-source described by `ctx`."""
+        return cls(ctx)._file_type
+
+    @property
+    def _file_type(self) -> FileType:
+        """FileType member corresponding to this document source."""
+        # -- An explicit content-type most commonly asserted by the client/SDK and is therefore
+        # -- inherently unreliable. On the other hand, binary file-types can be detected with 100%
+        # -- accuracy. So start with binary types and only then consider an asserted content-type,
+        # -- generally as a last resort.
+
+        if (
+            (  # strategy 1: most binary types can be detected with 100% accuracy
+                predicted_file_type := self._known_binary_file_type
+            )
+            or (  # strategy 2: use content-type asserted by caller
+                predicted_file_type := self._file_type_from_content_type
+            )
+            or (  # strategy 3: guess MIME-type using libmagic and use that
+                predicted_file_type := self._file_type_from_guessed_mime_type
+            )
+            or (  # strategy 4: use filename-extension, like ".docx" -> FileType.DOCX
+                predicted_file_type := self._file_type_from_file_extension
+            )
+        ):
+            result_file_type = predicted_file_type
+        else:
+            # give up and report FileType.UNK
+            result_file_type = FileType.UNK
+
+        if result_file_type == FileType.JSON:
+            # edge case where JSON/NDJSON content without file extension
+            # (magic lib can't distinguish them)
+            result_file_type = self._disambiguate_json_file_type
+
+        return result_file_type
+
+    @property
+    def _known_binary_file_type(self) -> FileType | None:
+        """Detect file-type for binary types we can positively detect."""
+        if file_type := _OleFileDetector.file_type(self._ctx):
+            return file_type
+
+        self._ctx.rule_out_cfb_content_types()
+
+        if file_type := _ZipFileDetector.file_type(self._ctx):
+            return file_type
+
+        self._ctx.rule_out_zip_content_types()
+
+        return None
+
+    @property
+    def _file_type_from_content_type(self) -> FileType | None:
+        """Map passed content-type argument to a file-type, subject to certain rules."""
+
+        # -- when no content-type was asserted by caller, this strategy is not applicable --
+        if not self._ctx.content_type:
+            return None
+
+        # -- otherwise we trust the passed `content_type` as long as `FileType` recognizes it --
+        return FileType.from_mime_type(self._ctx.content_type)
+
+    @property
+    def _disambiguate_json_file_type(self) -> FileType:
+        """Disambiguate JSON/NDJSON file-type based on file contents.
+
+        NDJSON is detected first because it has the strictest signature (multiple JSON values
+        separated by newlines, with the first line independently parsable). Anything else that
+        libmagic flagged as JSON is classified as `FileType.JSON`; the JSON partitioner has its
+        own `is_json_processable` schema check and will reject non-conforming payloads with a
+        clear error.
+        """
+        file_text, allow_truncated_single_line = self._ctx.json_disambiguation_text
+        if is_ndjson_processable(
+            file_text=file_text,
+            allow_truncated_single_line=allow_truncated_single_line,
+        ):
+            return FileType.NDJSON
+        return FileType.JSON
+
+    @property
+    def _file_type_from_guessed_mime_type(self) -> FileType | None:
+        """FileType based on auto-detection of MIME-type by libmagic.
+
+        In some cases refinements are necessary on the magic-derived MIME-types. This process
+        includes applying those rules, most of which are accumulated through practical experience.
+        """
+        mime_type = self._ctx.mime_type
+        extension = self._ctx.extension
+
+        # -- when libmagic is not installed, the `filetype` package is used instead.
+        # -- `filetype.guess()` returns `None` for file-types it does not support, which
+        # -- unfortunately includes all the textual file-types like CSV, EML, HTML, MD, RST, RTF,
+        # -- TSV, and TXT. When we have no guessed MIME-type, this strategy is not applicable.
+        if mime_type is None:
+            return None
+
+        if mime_type.endswith("xml"):
+            return FileType.HTML if extension in (".html", ".htm") else FileType.XML
+
+        if differentiator := _TextFileDifferentiator.applies(self._ctx):
+            return differentiator.file_type
+
+        # -- All source-code files (e.g. *.py, *.js) are classified as plain text for the moment --
+        if self._ctx.has_code_mime_type:
+            return FileType.TXT
+
+        if mime_type.endswith("empty"):
+            return FileType.EMPTY
+
+        if mime_type.endswith("json") and self._ctx.extension == ".ndjson":
+            return FileType.NDJSON
+
+        # -- if no more-specific rules apply, use the MIME-type -> FileType mapping when present --
+        file_type = FileType.from_mime_type(mime_type)
+        if file_type != FileType.UNK:
+            return file_type
+
+        # -- on some environments libmagic can return a generic/unhelpful MIME-type
+        # -- like octet-stream") for files that the `filetype` package identify.
+        # -- when that happens we retry using `filetype`  `FileType.UNK` results.
+        if LIBMAGIC_AVAILABLE:
+            fallback_mime_type = (
+                ft.guess_mime(self._ctx.file_path)
+                if self._ctx.file_path
+                else ft.guess_mime(self._ctx.file_head)
+            )
+            fallback_file_type = FileType.from_mime_type(fallback_mime_type)
+            if fallback_file_type and fallback_file_type != FileType.UNK:
+                return fallback_file_type
+
+        return None
+
+    @cached_property
+    def _file_type_from_file_extension(self) -> FileType | None:
+        """Determine file-type from filename extension.
+
+        Returns `None` when no filename is available or when the extension does not map to a
+        supported file-type.
+        """
+        return FileType.from_extension(self._ctx.extension)
+
+
+class _FileTypeDetectionContext:
+    """Provides all arguments to auto-file detection and values derived from them.
+
+    NOTE that `._content_type` is mutable via `.rule_out_*_content_types()` methods, so it should
+    not be assumed to be a constant value across those calls.
+
+    This keeps computation of derived values out of the file-detection code but more importantly
+    allows the main filetype-detector to pass the full context to any delegates without coupling
+    itself to which values it might need.
+    """
+
+    def __init__(
+        self,
+        file_path: str | None = None,
+        *,
+        file: IO[bytes] | None = None,
+        encoding: str | None = None,
+        content_type: str | None = None,
+        metadata_file_path: str | None = None,
+    ):
+        self._file_path_arg = file_path
+        self._file_arg = file
+        self._encoding_arg = encoding
+        self._content_type = content_type
+        self._metadata_file_path = metadata_file_path
+
+    @classmethod
+    def new(
+        cls,
+        *,
+        file_path: str | None,
+        file: IO[bytes] | None,
+        encoding: str | None,
+        content_type: str | None = None,
+        metadata_file_path: str | None = None,
+    ) -> _FileTypeDetectionContext:
+        self = cls(
+            file_path=file_path,
+            file=file,
+            encoding=encoding,
+            content_type=content_type,
+            metadata_file_path=metadata_file_path,
+        )
+        self._validate()
+        return self
+
+    @property
+    def content_type(self) -> str | None:
+        """MIME-type asserted by caller; not based on inspection of file by this process.
+
+        Would commonly occur when the file was downloaded via HTTP and a `"Content-Type:` header was
+        present on the response. These are often ambiguous and sometimes just wrong so get some
+        further verification. All lower-case when not `None`.
+        """
+        # -- Note `._content_type` is mutable via `.invalidate_content_type()` so this cannot be a
+        # -- `@cached_property`.
+        return self._content_type.lower() if self._content_type else None
+
+    @cached_property
+    def encoding(self) -> str:
+        """Character-set used to encode text of this file.
+
+        Relevant for textual file-types only, like HTML, TXT, JSON, etc.
+        """
+        return format_encoding_str(self._encoding_arg or "utf-8")
+
+    @cached_property
+    def extension(self) -> str:
+        """Best filename-extension we can muster, "" when there is no available source."""
+        # -- get from file_path, or file when it has a name (path) --
+        with self.open() as file:
+            if hasattr(file, "name") and file.name:
+                return os.path.splitext(file.name)[1].lower()
+
+        # -- otherwise use metadata file-path when provided --
+        if file_path := self._metadata_file_path:
+            return os.path.splitext(file_path)[1].lower()
+
+        # -- otherwise empty str means no extension, same as a path like "a/b/name-no-ext" --
+        return ""
+
+    @cached_property
+    def file_head(self) -> bytes:
+        """The initial bytes of the file to be recognized, for use with libmagic detection."""
+        with self.open() as file:
+            return file.read(8192)
+
+    @cached_property
+    def file_path(self) -> str | None:
+        """Filesystem path to file to be inspected, when provided on call.
+
+        None when the caller specified the source as a file-like object instead. Useful for user
+        feedback on an error, but users of context should have little use for it otherwise.
+        """
+        if (file_path := self._file_path_arg) is None:
+            return None
+
+        return os.path.realpath(file_path) if os.path.islink(file_path) else file_path
+
+    @cached_property
+    def has_code_mime_type(self) -> bool:
+        """True when `mime_type` plausibly indicates a programming language source-code file."""
+        mime_type = self.mime_type
+
+        if mime_type is None:
+            return False
+
+        # -- check Go separately to avoid matching other MIME type containing "go" --
+        if mime_type == "text/x-go":
+            return True
+
+        return any(
+            lang in mime_type
+            for lang in [
+                "c#",
+                "c++",
+                "cpp",
+                "csharp",
+                "java",
+                "javascript",
+                "php",
+                "python",
+                "ruby",
+                "swift",
+                "typescript",
+            ]
+        )
+
+    @cached_property
+    def is_zipfile(self) -> bool:
+        """True when file is a Zip archive."""
+        with self.open() as file:
+            return zipfile.is_zipfile(file)
+
+    @cached_property
+    def mime_type(self) -> str | None:
+        """The best MIME-type we can get from `magic` (or `filetype` package).
+
+        A `str` return value is always in lower-case.
+        """
+        file_path = self.file_path
+
+        if LIBMAGIC_AVAILABLE:
+            import magic
+
+            magic_mime = (
+                magic.from_file(file_path, mime=True)
+                if file_path
+                else magic.from_buffer(self.file_head, mime=True)
+            )
+            magic_mime = magic_mime.lower() if magic_mime else None
+
+            # When libmagic returns None or "application/octet-stream", try the filetype package
+            # (magic-byte signatures) for formats libmagic often mis-detects (e.g. BMP, HEIC, WAV).
+            if magic_mime and magic_mime != "application/octet-stream":
+                return magic_mime
+
+            ft_mime = ft.guess_mime(file_path) if file_path else ft.guess_mime(self.file_head)
+            if ft_mime:
+                return ft_mime.lower()
+            # filetype could not identify; same outcome as if we had not tried it.
+            return magic_mime
+
+        mime_type = ft.guess_mime(file_path) if file_path else ft.guess_mime(self.file_head)
+
+        if mime_type is None:
+            logger.warning(
+                "libmagic is unavailable but assists in filetype detection. Please consider"
+                " installing libmagic for better results."
+            )
+            return None
+
+        return mime_type.lower()
+
+    @contextlib.contextmanager
+    def open(self) -> Iterator[IO[bytes]]:
+        """Encapsulates complexity of dealing with file-path or file-like-object.
+
+        Provides an `IO[bytes]` object as the "common-denominator" document source.
+
+        Must be used as a context manager using a `with` statement:
+
+            with self._file as file:
+                do things with file
+
+        File is guaranteed to be at read position 0 when called.
+        """
+        if self.file_path:
+            with open(self.file_path, "rb") as f:
+                yield f
+        else:
+            file = self._file_arg
+            assert file is not None  # -- guaranteed by `._validate()` --
+            file.seek(0)
+            yield file
+
+    def rule_out_cfb_content_types(self) -> None:
+        """Invalidate content-type when a legacy MS-Office file-type is asserted.
+
+        Used before returning `None`; at that point we know the file is not one of these formats
+        so if the asserted `content-type` is a legacy MS-Office type we know it's wrong and should
+        not be used as a fallback later in the detection process.
+        """
+        if FileType.from_mime_type(self._content_type) in (
+            FileType.DOC,
+            FileType.MSG,
+            FileType.PPT,
+            FileType.XLS,
+        ):
+            self._content_type = None
+
+    def rule_out_zip_content_types(self) -> None:
+        """Invalidate content-type when an MS-Office 2007+ file-type is asserted.
+
+        Used before returning `None`; at that point we know the file is not one of these formats
+        so if the asserted `content-type` is an MS-Office 2007+ type we know it's wrong and should
+        not be used as a fallback later in the detection process.
+        """
+        if FileType.from_mime_type(self._content_type) in (
+            FileType.DOCX,
+            FileType.EPUB,
+            FileType.ODT,
+            FileType.PPTX,
+            FileType.XLSX,
+            FileType.ZIP,
+        ):
+            self._content_type = None
+
+    @cached_property
+    def text_head(self) -> str:
+        """The initial characters of the text file for use with text-format differentiation.
+
+        Raises:
+            UnicodeDecodeError if file cannot be read as text.
+        """
+        # TODO: only attempts fallback character-set detection for file-path case, not for
+        # file-like object case. Seems like we should do both.
+
+        if file := self._file_arg:
+            file.seek(0)
+            content = file.read(4096)
+            file.seek(0)
+            return (
+                content
+                if isinstance(content, str)
+                else content.decode(encoding=self.encoding, errors="ignore")
+            )
+
+        file_path = self.file_path
+        assert file_path is not None  # -- guaranteed by `._validate` --
+
+        try:
+            with open(file_path, encoding=self.encoding) as f:
+                return f.read(4096)
+        except UnicodeDecodeError:
+            encoding, _ = detect_file_encoding(filename=file_path)
+            with open(file_path, encoding=encoding) as f:
+                return f.read(4096)
+
+    @cached_property
+    def json_disambiguation_text(self) -> tuple[str, bool]:
+        """Text prefix for JSON/NDJSON disambiguation and whether the first line was truncated."""
+
+        if file := self._file_arg:
+            file.seek(0)
+            content, first_line_truncated = self._read_until_newline_or_limit(file)
+            file.seek(0)
+            if isinstance(content, str):
+                return content, first_line_truncated
+            return content.decode(encoding=self.encoding, errors="ignore"), first_line_truncated
+
+        file_path = self.file_path
+        assert file_path is not None  # -- guaranteed by `._validate` --
+
+        try:
+            with open(file_path, encoding=self.encoding) as f:
+                content, first_line_truncated = self._read_until_newline_or_limit(f)
+                assert isinstance(content, str)
+                return content, first_line_truncated
+        except UnicodeDecodeError:
+            encoding, _ = detect_file_encoding(filename=file_path)
+            with open(file_path, encoding=encoding) as f:
+                content, first_line_truncated = self._read_until_newline_or_limit(f)
+                assert isinstance(content, str)
+                return content, first_line_truncated
+
+    def _validate(self) -> None:
+        """Raise if the context is invalid."""
+        if self.file_path and not os.path.isfile(self.file_path):
+            raise FileNotFoundError(f"no such file {self._file_path_arg}")
+        if not self.file_path and not self._file_arg:
+            raise ValueError("either `file_path` or `file` argument must be provided")
+
+    @staticmethod
+    def _read_until_newline_or_limit(file: IO) -> tuple[str | bytes, bool]:
+        """Read through the first newline, stopping at a bounded prefix if none is found."""
+        chunks: list[str | bytes] = []
+        chars_read = 0
+
+        while chars_read < _JSON_DISAMBIGUATION_MAX_CHARS:
+            chars_to_read = min(
+                _JSON_DISAMBIGUATION_CHUNK_SIZE,
+                _JSON_DISAMBIGUATION_MAX_CHARS - chars_read,
+            )
+            chunk = file.read(chars_to_read)
+            if not chunk:
+                return _FileTypeDetectionContext._join_text_chunks(chunks), False
+
+            newline = b"\n" if isinstance(chunk, bytes) else "\n"
+            newline_idx = chunk.find(newline)
+            if newline_idx != -1:
+                chunks.append(chunk[: newline_idx + 1])
+                return _FileTypeDetectionContext._join_text_chunks(chunks), False
+
+            chunks.append(chunk)
+            chars_read += len(chunk)
+
+        return _FileTypeDetectionContext._join_text_chunks(chunks), True
+
+    @staticmethod
+    def _join_text_chunks(chunks: list[str | bytes]) -> str | bytes:
+        """Join chunks without mixing text and bytes types."""
+        if chunks and isinstance(chunks[0], bytes):
+            return b"".join(cast(list[bytes], chunks))
+        return "".join(cast(list[str], chunks))
+
+
+class _OleFileDetector:
+    """Detect and differentiate a CFB file, aka. "OLE" file.
+
+    Compound File Binary Format (CFB), aka. OLE file, is use by Microsoft for legacy MS Office
+    files (DOC, PPT, XLS) as well as for Outlook MSG files.
+    """
+
+    def __init__(self, ctx: _FileTypeDetectionContext):
+        self._ctx = ctx
+
+    @classmethod
+    def file_type(cls, ctx: _FileTypeDetectionContext) -> FileType | None:
+        """Specific file-type when file is a CFB file, `None` otherwise."""
+        return cls(ctx)._file_type
+
+    @property
+    def _file_type(self) -> FileType | None:
+        """Differentiated file-type for Microsoft Compound File Binary Format (CFBF).
+
+        Returns one of:
+        - `FileType.DOC`
+        - `FileType.PPT`
+        - `FileType.XLS`
+        - `FileType.MSG`
+        - `None` when the file is not one of these.
+        """
+        # -- all CFB files share common magic number, start with that --
+        if not self._is_ole_file:
+            return None
+
+        # -- check storage contents of the ole file for file-type specific stream names --
+        if (ole_file_type := self._ole_file_type) is not None:
+            return ole_file_type
+
+        return None
+
+    @cached_property
+    def _is_ole_file(self) -> bool:
+        """True when file has CFB magic first 8 bytes."""
+        with self._ctx.open() as file:
+            return file.read(8) == b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
+
+    @cached_property
+    def _ole_file_type(self) -> FileType | None:
+        with self._ctx.open() as f:
+            ole = OleFileIO(f)  # pyright: ignore[reportUnknownVariableType]
+            root_storage = Storage.from_ole(ole)  # pyright: ignore[reportUnknownMemberType]
+
+        for stream in root_storage.streams:
+            if stream.name == "WordDocument":
+                return FileType.DOC
+            elif stream.name == "PowerPoint Document":
+                return FileType.PPT
+            elif stream.name == "Workbook":
+                return FileType.XLS
+            elif stream.name == "__properties_version1.0":
+                return FileType.MSG
+
+        return None
+
+
+class _TextFileDifferentiator:
+    """Refine a textual file-type that may not be as specific as it could be."""
+
+    def __init__(self, ctx: _FileTypeDetectionContext):
+        self._ctx = ctx
+
+    @classmethod
+    def applies(cls, ctx: _FileTypeDetectionContext) -> _TextFileDifferentiator | None:
+        """Constructs an instance, but only if this differentiator applies in `ctx`."""
+        mime_type = ctx.mime_type
+        return (
+            cls(ctx)
+            if mime_type and (mime_type == "message/rfc822" or mime_type.startswith("text"))
+            else None
+        )
+
+    @cached_property
+    def file_type(self) -> FileType:
+        """Differentiated file-type for textual content.
+
+        Always produces a file-type, worst case that's `FileType.TXT` when nothing more specific
+        applies.
+        """
+        extension = self._ctx.extension
+
+        if extension in [
+            ".csv",
+            ".eml",
+            ".html",
+            ".json",
+            ".markdown",
+            ".md",
+            ".org",
+            ".p7s",
+            ".rst",
+            ".rtf",
+            ".tab",
+            ".tsv",
+        ]:
+            return FileType.from_extension(extension) or FileType.TXT
+
+        # NOTE(crag): for older versions of the OS libmagic package, such as is currently
+        # installed on the Unstructured docker image, .json files resolve to "text/plain"
+        # rather than "application/json". this corrects for that case.
+        if self._is_json:
+            return FileType.JSON
+
+        if self._is_csv:
+            return FileType.CSV
+
+        if self._is_eml:
+            return FileType.EML
+
+        if extension in (".text", ".txt"):
+            return FileType.TXT
+
+        # Safety catch
+        if file_type := FileType.from_mime_type(self._ctx.mime_type):
+            return file_type
+
+        return FileType.TXT
+
+    @cached_property
+    def _is_csv(self) -> bool:
+        """True when file is plausibly in Comma Separated Values (CSV) format."""
+
+        def count_commas(text: str):
+            """Counts the number of commas in a line, excluding commas in quotes."""
+            pattern = r"(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$),"
+            matches = re.findall(pattern, text)
+            return len(matches)
+
+        lines = self._ctx.text_head.strip().splitlines()
+        if len(lines) < 2:
+            return False
+        # -- check at most the first 10 lines --
+        lines = lines[: len(lines)] if len(lines) < 10 else lines[:10]
+        # -- any lines without at least one comma disqualifies the file --
+        if any("," not in line for line in lines):
+            return False
+        header_count = count_commas(lines[0])
+        return all(count_commas(line) == header_count for line in lines[1:])
+
+    @cached_property
+    def _is_eml(self) -> bool:
+        """Checks if a text/plain file is actually a .eml file.
+
+        Uses a regex pattern to see if the start of the file matches the typical pattern for a .eml
+        file.
+        """
+        return EMAIL_HEAD_RE.match(self._ctx.text_head) is not None
+
+    @cached_property
+    def _is_json(self) -> bool:
+        """True when file is JSON collection.
+
+        A JSON file that contains only a string, number, or boolean, while valid JSON, will fail
+        this test since it is not partitionable.
+        """
+        text_head = self._ctx.text_head
+
+        # -- an empty file is not JSON --
+        if not text_head.lstrip():
+            return False
+
+        # -- has to be a list or object, no string, number, or bool --
+        if text_head.lstrip()[0] not in "[{":
+            return False
+
+        try:
+            with self._ctx.open() as file:
+                json.load(file)
+            return True
+        except json.JSONDecodeError:
+            return False
+
+
+class _ZipFileDetector:
+    """Detect and differentiate a Zip-archive file."""
+
+    def __init__(self, ctx: _FileTypeDetectionContext):
+        self._ctx = ctx
+
+    @classmethod
+    def file_type(cls, ctx: _FileTypeDetectionContext) -> FileType | None:
+        """Most specific file-type available when file is a Zip file, `None` otherwise.
+
+        MS-Office 2007+ files are detected with 100% accuracy. Otherwise this returns `None`, even
+        when we can tell it's a Zip file, so later strategies can have a crack at it. In
+        particular, ODT and EPUB files are Zip archives but are not detected here.
+        """
+        return cls(ctx)._file_type
+
+    @cached_property
+    def _file_type(self) -> FileType | None:
+        """Differentiated file-type for a Zip archive.
+
+        Returns `FileType.DOCX`, `FileType.PPTX`, or `FileType.XLSX` when one of those applies,
+        `None` otherwise.
+        """
+        if not self._ctx.is_zipfile:
+            return None
+
+        with self._ctx.open() as file:
+            zip = zipfile.ZipFile(file)
+
+            filenames = zip.namelist()
+
+            if any(re.match(r"word/document.*\.xml$", filename) for filename in filenames):
+                return FileType.DOCX
+
+            if any(re.match(r"xl/workbook.*\.xml$", filename) for filename in filenames):
+                return FileType.XLSX
+
+            if any(re.match(r"ppt/presentation.*\.xml$", filename) for filename in filenames):
+                return FileType.PPTX
+
+            # -- ODT and EPUB files place their MIME-type in `mimetype` in the archive root --
+            if "mimetype" in filenames:
+                with zip.open("mimetype") as f:
+                    mime_type = f.read().decode("utf-8").strip()
+                    return FileType.from_mime_type(mime_type)
+
+        return FileType.ZIP
+
+
+_P = ParamSpec("_P")
+
+
+def add_metadata(func: Callable[_P, list[Element]]) -> Callable[_P, list[Element]]:
+    @functools.wraps(func)
+    def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> list[Element]:
+        elements = func(*args, **kwargs)
+        call_args = get_call_args_applying_defaults(func, *args, **kwargs)
+
+        if call_args.get("metadata_filename"):
+            call_args["filename"] = call_args.get("metadata_filename")
+
+        metadata_kwargs = {
+            kwarg: call_args.get(kwarg) for kwarg in ("filename", "url", "text_as_html")
+        }
+        # NOTE (yao): do not use cast here as cast(None) still is None
+        if not str(kwargs.get("model_name", "")).startswith("chipper"):
+            # NOTE(alan): Skip hierarchy if using chipper, as it should take care of that
+            elements = set_element_hierarchy(elements)
+
+        for element in elements:
+            # NOTE(robinson) - Attached files have already run through this logic
+            # in their own partitioning function
+            if element.metadata.attached_to_filename is None:
+                add_element_metadata(element, **metadata_kwargs)
+
+        return elements
+
+    return wrapper
+
+
+def add_filetype(
+    filetype: FileType,
+) -> Callable[[Callable[_P, list[Element]]], Callable[_P, list[Element]]]:
+    """Post-process element-metadata for list[Element] from partitioning.
+
+    This decorator adds a post-processing step to a document partitioner.
+
+    - Adds `.metadata.filetype` (source-document MIME-type) metadata value
+
+    This "partial" decorator is present because `partition_image()` does not apply
+    `.metadata.filetype` this way since each image type has its own MIME-type (e.g. `image.jpeg`,
+    `image/png`, etc.).
+    """
+
+    def decorator(func: Callable[_P, list[Element]]) -> Callable[_P, list[Element]]:
+        @functools.wraps(func)
+        def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> list[Element]:
+            elements = func(*args, **kwargs)
+
+            for element in elements:
+                # NOTE(robinson) - Attached files have already run through this logic
+                # in their own partitioning function
+                if element.metadata.attached_to_filename is None:
+                    add_element_metadata(element, filetype=filetype.mime_type)
+
+            return elements
+
+        return wrapper
+
+    return decorator
+
+
+def add_metadata_with_filetype(
+    filetype: FileType,
+) -> Callable[[Callable[_P, list[Element]]], Callable[_P, list[Element]]]:
+    """..."""
+
+    def decorator(func: Callable[_P, list[Element]]) -> Callable[_P, list[Element]]:
+        return add_filetype(filetype=filetype)(add_metadata(func))
+
+    return decorator

unstructured/file_utils/google_filetype.py

@@ -0,0 +1,9 @@
+GOOGLE_DRIVE_EXPORT_TYPES = {
+    "application/vnd.google-apps.document": "application/"
+    "vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "application/vnd.google-apps.spreadsheet": "application/"
+    "vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "application/vnd.google-apps.presentation": "application/"
+    "vnd.openxmlformats-officedocument.presentationml.presentation",
+    "application/vnd.google-apps.photo": "image/jpeg",
+}

unstructured/file_utils/model.py

@@ -0,0 +1,591 @@
+"""Domain-model for file-types."""
+
+from __future__ import annotations
+
+import enum
+from typing import TYPE_CHECKING, Callable, Iterable, Type, cast
+
+from typing_extensions import ParamSpec
+
+if TYPE_CHECKING:
+    from unstructured.documents.elements import Element
+else:
+    Element = None
+
+
+def _create_file_type_enum(
+    cls: Type["FileType"],
+    value: str,
+    partitioner_shortname: str | None,
+    importable_package_dependencies: Iterable[str],
+    extra_name: str | None,
+    extensions: Iterable[str],
+    canonical_mime_type: str,
+    alias_mime_types: Iterable[str],
+    partitioner_full_module_path: str | None = None,
+) -> "FileType":
+    """
+    Moving here instead of directly in the FileType.__new__ allows us
+    to dynamically create new enum properties.
+
+    FileType.__new__ does not work with dynamic properties.
+    """
+    val = object.__new__(cls)
+    val._value_ = value
+    val._partitioner_shortname = partitioner_shortname
+    val._importable_package_dependencies = tuple(importable_package_dependencies)
+    val._extra_name = extra_name
+    val._extensions = tuple(extensions)
+    val._canonical_mime_type = canonical_mime_type
+    val._alias_mime_types = tuple(alias_mime_types)
+    val._partitioner_full_module_path = partitioner_full_module_path
+    return val
+
+
+class FileType(enum.Enum):
+    """The collection of file-types recognized by `unstructured`.
+
+    Note not all of these can be partitioned, e.g. ZIP has no partitioner.
+    """
+
+    _partitioner_shortname: str | None
+    """Like "docx", from which partitioner module and function-name can be derived via template."""
+
+    _importable_package_dependencies: tuple[str, ...]
+    """Packages that must be available for import for this file-type's partitioner to work."""
+
+    _extra_name: str | None
+    """`pip install` extra that provides package dependencies for this file-type."""
+
+    _extensions: tuple[str, ...]
+    """Filename-extensions recognized as this file-type. Use for secondary identification only."""
+
+    _canonical_mime_type: str
+    """The MIME-type used as `.metadata.filetype` for this file-type."""
+
+    _alias_mime_types: tuple[str, ...]
+    """MIME-types accepted as identifying this file-type."""
+
+    _partitioner_full_module_path: str | None
+    """Fully-qualified name of module providing partitioner for this file-type."""
+
+    def __new__(
+        cls,
+        value: str,
+        partitioner_shortname: str | None,
+        importable_package_dependencies: Iterable[str],
+        extra_name: str | None,
+        extensions: Iterable[str],
+        canonical_mime_type: str,
+        alias_mime_types: Iterable[str],
+        partitioner_full_module_path: str | None = None,
+    ):
+        return _create_file_type_enum(
+            cls,
+            value,
+            partitioner_shortname,
+            importable_package_dependencies,
+            extra_name,
+            extensions,
+            canonical_mime_type,
+            alias_mime_types,
+            partitioner_full_module_path,
+        )
+
+    def __lt__(self, other: FileType) -> bool:
+        """Makes `FileType` members comparable with relational operators, at least with `<`.
+
+        This makes them sortable, in particular it supports sorting for pandas groupby functions.
+        """
+        return self.name < other.name
+
+    @classmethod
+    def from_extension(cls, extension: str | None) -> FileType | None:
+        """Select a FileType member based on an extension.
+
+        `extension` must include the leading period, like `".pdf"`. Extension is suitable as a
+        secondary file-type identification method but is unreliable for primary identification.
+
+        Returns `None` when `extension` is not registered for any supported file-type.
+        """
+        if extension in (None, "", "."):
+            return None
+        # -- not super efficient but plenty fast enough for once-or-twice-per-file use and avoids
+        # -- limitations on defining a class variable on an Enum.
+        for m in cls.__members__.values():
+            if extension in m._extensions:
+                return m
+        return None
+
+    @classmethod
+    def from_mime_type(cls, mime_type: str | None) -> FileType | None:
+        """Select a FileType member based on a MIME-type.
+
+        Returns `None` when `mime_type` is `None` or does not map to the canonical MIME-type of a
+        `FileType` member or one of its alias MIME-types.
+        """
+        if mime_type is None:
+            return None
+        # -- not super efficient but plenty fast enough for once-or-twice-per-file use and avoids
+        # -- limitations on defining a class variable on an Enum.
+        for m in cls.__members__.values():
+            if mime_type == m._canonical_mime_type or mime_type in m._alias_mime_types:
+                return m
+        return None
+
+    @property
+    def extra_name(self) -> str | None:
+        """The `pip` "extra" that must be installed to provide this file-type's dependencies.
+
+        Like "image" for PNG, as in `pip install "unstructured[image]"`.
+
+        `None` when partitioning this file-type requires only the base `unstructured` install.
+        """
+        return self._extra_name
+
+    @property
+    def importable_package_dependencies(self) -> tuple[str, ...]:
+        """Packages that must be importable for this file-type's partitioner to work.
+
+        In general, these are the packages provided by the `pip install` "extra" for this file-type,
+        like `pip install "unstructured[docx]"` loads the `python-docx` package.
+
+        Note that these names are the ones used in an `import` statement, which is not necessarily
+        the same as the _distribution_ package name used by `pip`. For example, the DOCX
+        distribution package name is `"python-docx"` whereas the _importable_ package name is
+        `"docx"`. This latter name as it appears like `import docx` is what is provided by this
+        property.
+
+        The return value is an empty tuple for file-types that do not require optional dependencies.
+
+        Note this property does not complain when accessed on a non-partitionable file-type, it
+        simply returns an empty tuple because file-types that are not partitionable require no
+        optional dependencies.
+        """
+        return self._importable_package_dependencies
+
+    @property
+    def is_partitionable(self) -> bool:
+        """True when there is a partitioner for this file-type.
+
+        Note this does not check whether the dependencies for this file-type are installed so
+        attempting to partition a file of this type may still fail. This is meant for
+        distinguishing file-types like ZIP, EMPTY, and UNK which are legitimate file-types
+        but have no associated partitioner.
+        """
+        return bool(self._partitioner_shortname) or bool(self._partitioner_full_module_path)
+
+    @property
+    def mime_type(self) -> str:
+        """The canonical MIME-type for this file-type, suitable for use in metadata.
+
+        This value is used in `.metadata.filetype` for elements partitioned from files of this
+        type. In general it is the "offical", "recommended", or "defacto-standard" MIME-type for
+        files of this type, in that order, as available.
+        """
+        return self._canonical_mime_type
+
+    @property
+    def partitioner_function_name(self) -> str:
+        """Name of partitioner function for this file-type. Like "partition_docx".
+
+        Raises when this property is accessed on a file-type that is not partitionable. Use
+        `.is_partitionable` to avoid exceptions when partitionability is unknown.
+        """
+        # -- Raise when this property is accessed on a FileType member that has no partitioner
+        # -- shortname. This prevents a harder-to-find bug from appearing far away from this call
+        # -- when code would try to `getattr(module, None)` or whatever.
+        if full_module_path := self._partitioner_full_module_path:
+            return full_module_path.split(".")[-1]
+
+        if (shortname := self._partitioner_shortname) is None:
+            raise ValueError(
+                f"`.partitioner_function_name` is undefined because FileType.{self.name} is not"
+                f" partitionable. Use `.is_partitionable` to determine whether a `FileType`"
+                f" is partitionable."
+            )
+        return f"partition_{shortname}"
+
+    @property
+    def partitioner_module_qname(self) -> str:
+        """Fully-qualified name of module providing partitioner for this file-type.
+
+        e.g. "unstructured.partition.docx" for FileType.DOCX.
+        """
+        # -- Raise when this property is accessed on a FileType member that has no partitioner
+        # -- shortname. This prevents a harder-to-find bug from appearing far away from this call
+        # -- when code would try to `importlib.import_module(None)` or whatever.
+        if full_module_path := self._partitioner_full_module_path:
+            return ".".join(full_module_path.split(".")[:-1])
+
+        if (shortname := self._partitioner_shortname) is None:
+            raise ValueError(
+                f"`.partitioner_module_qname` is undefined because FileType.{self.name} is not"
+                f" partitionable. Use `.is_partitionable` to determine whether a `FileType`"
+                f" is partitionable."
+            )
+        return f"unstructured.partition.{shortname}"
+
+    @property
+    def partitioner_shortname(self) -> str | None:
+        """Familiar name of partitioner, like "image" for file-types that use `partition_image()`.
+
+        One use is to determine whether a file-type is one of the five image types, all of which
+        are processed by `partition_image()`.
+
+        `None` for file-types that are not partitionable, although `.is_partitionable` is the
+        preferred way of discovering that.
+        """
+        return self._partitioner_shortname
+
+    BMP = (
+        "bmp",  # -- value for this Enum member, like BMP = "bmp" in a simple enum --
+        "image",  # -- partitioner_shortname --
+        ["unstructured_inference"],  # -- importable_package_dependencies --
+        "image",  # -- extra_name - like `pip install "unstructured[image]"` in this case --
+        [".bmp"],  # -- extensions - filename extensions that map to this file-type --
+        "image/bmp",  # -- canonical_mime_type -  MIME-type written to `.metadata.filetype` --
+        [
+            "image/x-bmp",
+            "image/x-ms-bmp",  # returned by older libmagic versions
+        ],
+    )
+    CSV = (
+        "csv",
+        "csv",
+        ["pandas"],
+        "csv",
+        [".csv"],
+        "text/csv",
+        [
+            "application/csv",
+            "application/x-csv",
+            "text/comma-separated-values",
+            "text/x-comma-separated-values",
+            "text/x-csv",
+        ],
+    )
+    DOC = ("doc", "doc", ["docx"], "doc", [".doc"], "application/msword", cast(list[str], []))
+    DOCX = (
+        "docx",
+        "docx",
+        ["docx"],
+        "docx",
+        [".docx"],
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        cast(list[str], []),
+    )
+    EML = (
+        "eml",
+        "email",
+        cast(list[str], []),
+        None,
+        [".eml", ".p7s"],
+        "message/rfc822",
+        cast(list[str], []),
+    )
+    EPUB = (
+        "epub",
+        "epub",
+        ["pypandoc"],
+        "epub",
+        [".epub"],
+        "application/epub",
+        ["application/epub+zip"],
+    )
+    FLAC = (
+        "flac",
+        "audio",
+        cast(list[str], []),  # STT agent deps validated at runtime by the chosen agent
+        "audio",
+        [".flac"],
+        "audio/flac",
+        ["audio/x-flac"],
+    )
+    HEIC = (
+        "heic",
+        "image",
+        ["unstructured_inference"],
+        "image",
+        [".heic"],
+        "image/heic",
+        ["image/x-heic"],
+    )
+    HTML = (
+        "html",
+        "html",
+        cast(list[str], []),
+        None,
+        [".html", ".htm"],
+        "text/html",
+        cast(list[str], []),
+    )
+    JPG = (
+        "jpg",
+        "image",
+        ["unstructured_inference"],
+        "image",
+        [".jpeg", ".jpg"],
+        "image/jpeg",
+        cast(list[str], []),
+    )
+    JSON = (
+        "json",
+        "json",
+        cast(list[str], []),
+        None,
+        [".json"],
+        "application/json",
+        cast(list[str], []),
+    )
+    M4A = (
+        "m4a",
+        "audio",
+        cast(list[str], []),  # STT agent deps validated at runtime by the chosen agent
+        "audio",
+        [".m4a"],
+        "audio/mp4",
+        ["audio/x-m4a"],
+    )
+    MD = ("md", "md", ["markdown"], "md", [".md"], "text/markdown", ["text/x-markdown"])
+    MP3 = (
+        "mp3",
+        "audio",
+        cast(list[str], []),  # STT agent deps validated at runtime by the chosen agent
+        "audio",
+        [".mp3"],
+        "audio/mpeg",
+        ["audio/mp3", "audio/x-mp3", "audio/x-mpeg"],
+    )
+    MSG = (
+        "msg",
+        "msg",
+        ["oxmsg"],
+        "msg",
+        [".msg"],
+        "application/vnd.ms-outlook",
+        cast(list[str], []),
+    )
+    NDJSON = (
+        "ndjson",
+        "ndjson",
+        cast(list[str], []),
+        None,
+        [".ndjson"],
+        "application/x-ndjson",
+        cast(list[str], []),
+    )
+    ODT = (
+        "odt",
+        "odt",
+        ["docx", "pypandoc"],
+        "odt",
+        [".odt"],
+        "application/vnd.oasis.opendocument.text",
+        cast(list[str], []),
+    )
+    OGG = (
+        "ogg",
+        "audio",
+        cast(list[str], []),  # STT agent deps validated at runtime by the chosen agent
+        "audio",
+        [".ogg", ".oga"],
+        "audio/ogg",
+        ["audio/x-ogg"],
+    )
+    OPUS = (
+        "opus",
+        "audio",
+        cast(list[str], []),  # STT agent deps validated at runtime by the chosen agent
+        "audio",
+        [".opus"],
+        "audio/opus",
+        cast(list[str], []),
+    )
+    ORG = ("org", "org", ["pypandoc"], "org", [".org"], "text/org", cast(list[str], []))
+    PDF = (
+        "pdf",
+        "pdf",
+        ["pdf2image", "pdfminer", "PIL"],
+        "pdf",
+        [".pdf"],
+        "application/pdf",
+        cast(list[str], []),
+    )
+    PNG = (
+        "png",
+        "image",
+        ["unstructured_inference"],
+        "image",
+        [".png"],
+        "image/png",
+        cast(list[str], []),
+    )
+    PPT = (
+        "ppt",
+        "ppt",
+        ["pptx"],
+        "ppt",
+        [".ppt"],
+        "application/vnd.ms-powerpoint",
+        cast(list[str], []),
+    )
+    PPTX = (
+        "pptx",
+        "pptx",
+        ["pptx"],
+        "pptx",
+        [".pptx"],
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        cast(list[str], []),
+    )
+    RST = ("rst", "rst", ["pypandoc"], "rst", [".rst"], "text/x-rst", cast(list[str], []))
+    RTF = ("rtf", "rtf", ["pypandoc"], "rtf", [".rtf"], "text/rtf", ["application/rtf"])
+    TIFF = (
+        "tiff",
+        "image",
+        ["unstructured_inference"],
+        "image",
+        [".tiff"],
+        "image/tiff",
+        cast(list[str], []),
+    )
+    TSV = ("tsv", "tsv", ["pandas"], "tsv", [".tab", ".tsv"], "text/tsv", cast(list[str], []))
+    TXT = (
+        "txt",
+        "text",
+        cast(list[str], []),
+        None,
+        [
+            ".txt",
+            ".text",
+            # NOTE(robinson) - for now we are treating code files as plain text
+            ".c",
+            ".cc",
+            ".cpp",
+            ".cs",
+            ".cxx",
+            ".go",
+            ".java",
+            ".js",
+            ".log",
+            ".php",
+            ".py",
+            ".rb",
+            ".swift",
+            ".ts",
+            ".yaml",
+            ".yml",
+        ],
+        "text/plain",
+        [
+            # NOTE(robinson) - In the future, we may have special processing for YAML files
+            # instead of treating them as plaintext.
+            "text/yaml",
+            "application/x-yaml",
+            "application/yaml",
+            "text/x-yaml",
+        ],
+    )
+    WAV = (
+        "wav",
+        "audio",
+        cast(list[str], []),  # STT agent deps validated at runtime by the chosen agent
+        "audio",
+        [".wav"],
+        "audio/wav",
+        [
+            "audio/vnd.wav",
+            "audio/vnd.wave",
+            "audio/wave",
+            "audio/x-pn-wav",
+            "audio/x-wav",
+        ],
+    )
+    WEBM = (
+        "webm",
+        "audio",
+        cast(list[str], []),  # STT agent deps validated at runtime by the chosen agent
+        "audio",
+        [".webm"],
+        "audio/webm",
+        [],  # Do not alias video/webm: WebM is a container; this type is for audio-only.
+    )
+    XLS = (
+        "xls",
+        "xlsx",
+        ["pandas", "openpyxl"],
+        "xlsx",
+        [".xls"],
+        "application/vnd.ms-excel",
+        cast(list[str], []),
+    )
+    XLSX = (
+        "xlsx",
+        "xlsx",
+        ["pandas", "openpyxl"],
+        "xlsx",
+        [".xlsx"],
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        cast(list[str], []),
+    )
+    XML = ("xml", "xml", cast(list[str], []), None, [".xml"], "application/xml", ["text/xml"])
+    ZIP = ("zip", None, cast(list[str], []), None, [".zip"], "application/zip", cast(list[str], []))
+
+    UNK = (
+        "unk",
+        None,
+        cast(list[str], []),
+        None,
+        cast(list[str], []),
+        "application/octet-stream",
+        cast(list[str], []),
+    )
+    EMPTY = (
+        "empty",
+        None,
+        cast(list[str], []),
+        None,
+        cast(list[str], []),
+        "inode/x-empty",
+        cast(list[str], []),
+    )
+
+
+def create_file_type(
+    name: str,
+    *,
+    canonical_mime_type: str,
+    importable_package_dependencies: Iterable[str] | None = None,
+    extra_name: str | None = None,
+    extensions: Iterable[str] | None = None,
+    alias_mime_types: Iterable[str] | None = None,
+) -> FileType:
+    """Register a new FileType member."""
+    type_ = _create_file_type_enum(
+        FileType,
+        name,
+        None,
+        importable_package_dependencies or cast(list[str], []),
+        extra_name,
+        extensions or cast(list[str], []),
+        canonical_mime_type,
+        alias_mime_types or cast(list[str], []),
+        None,
+    )
+    type_._name_ = name
+    FileType._member_map_[name] = type_
+    return type_
+
+
+_P = ParamSpec("_P")
+
+
+def register_partitioner(
+    file_type: FileType,
+) -> Callable[[Callable[_P, list[Element]]], Callable[_P, list[Element]]]:
+    def decorator(func: Callable[_P, list[Element]]) -> Callable[_P, list[Element]]:
+        file_type._partitioner_full_module_path = func.__module__ + "." + func.__name__
+        return func
+
+    return decorator

unstructured/file_utils/ndjson.py

@@ -0,0 +1,67 @@
+"""
+Adds support for working with newline-delimited JSON (ndjson) files. This format is useful for
+streaming json content that would otherwise not be possible using raw JSON files.
+"""
+
+import json
+from typing import IO, Any
+
+
+def dumps(obj: list[dict[str, Any]], **kwargs) -> str:
+    """
+    Converts the list of dictionaries into string representation
+
+    Args:
+        obj (list[dict[str, Any]]): List of dictionaries to convert
+        **kwargs: Additional keyword arguments to pass to json.dumps
+
+    Returns:
+        str: string representation of the list of dictionaries
+    """
+    return "\n".join(json.dumps(each, **kwargs) for each in obj)
+
+
+def dump(obj: list[dict[str, Any]], fp: IO, **kwargs) -> None:
+    """
+    Writes the list of dictionaries to a newline-delimited file
+
+    Args:
+        obj (list[dict[str, Any]]): List of dictionaries to convert
+        fp (IO): File pointer to write the string representation to
+        **kwargs: Additional keyword arguments to pass to json.dumps
+
+    Returns:
+        None
+    """
+    # Indent breaks ndjson formatting
+    kwargs["indent"] = None
+    text = dumps(obj, **kwargs)
+    fp.write(text)
+
+
+def loads(s: str, **kwargs) -> list[dict[str, Any]]:
+    """
+    Converts the raw string into a list of dictionaries
+
+    Args:
+        s (str): Raw string to convert
+        **kwargs: Additional keyword arguments to pass to json.loads
+
+    Returns:
+        list[dict[str, Any]]: List of dictionaries parsed from the input string
+    """
+    return [json.loads(line, **kwargs) for line in s.splitlines()]
+
+
+def load(fp: IO, **kwargs) -> list[dict[str, Any]]:
+    """
+    Converts the contents of the file into a list of dictionaries
+
+    Args:
+        fp (IO): File pointer to read the string representation from
+        **kwargs: Additional keyword arguments to pass to json.loads
+
+    Returns:
+        list[dict[str, Any]]: List of dictionaries parsed from the file
+    """
+    return loads(fp.read(), **kwargs)

unstructured/logger.py

@@ -0,0 +1,18 @@
+import logging
+
+logger = logging.getLogger("unstructured")
+trace_logger = logging.getLogger("unstructured.trace")
+
+# Create a custom logging level
+DETAIL = 15
+logging.addLevelName(DETAIL, "DETAIL")
+
+
+# Create a custom log method for the "DETAIL" level
+def detail(self, message, *args, **kws):
+    if self.isEnabledFor(DETAIL):
+        self._log(DETAIL, message, args, **kws)
+
+
+# Add the custom log method to the logging.Logger class
+logging.Logger.detail = detail  # type: ignore

unstructured/metrics/element_type.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import json
+
+from typing_extensions import TypeAlias
+
+FrequencyDict: TypeAlias = "dict[tuple[str, int | None], int]"
+"""Like:
+    {
+        ("ListItem", 0): 2,
+        ("NarrativeText", None): 2,
+        ("Title", 0): 5,
+        ("UncategorizedText", None): 6,
+    }
+"""
+
+
+def get_element_type_frequency(
+    elements: str,
+) -> FrequencyDict:
+    """
+    Calculate the frequency of Element Types from a list of elements.
+
+    Args:
+        elements (str): String-formatted json of all elements (as a result of elements_to_json).
+    Returns:
+        Element type and its frequency in dictionary format.
+    """
+    frequency: dict[tuple[str, int | None], int] = {}
+    if len(elements) == 0:
+        return frequency
+    for element in json.loads(elements):
+        type = element.get("type")
+        category_depth = element["metadata"].get("category_depth")
+        key = (type, category_depth)
+        if key not in frequency:
+            frequency[key] = 1
+        else:
+            frequency[key] += 1
+    return frequency
+
+
+def calculate_element_type_percent_match(
+    output: FrequencyDict,
+    source: FrequencyDict,
+    category_depth_weight: float = 0.5,
+) -> float:
+    """Calculate the percent match between two frequency dictionary.
+
+    Intended to use with `get_element_type_frequency` function. The function counts the absolute
+    exact match (type and depth), and counts the weighted match (correct type but different depth),
+    then normalized with source's total elements.
+    """
+    if len(output) == 0 or len(source) == 0:
+        return 0.0
+
+    output_copy = output.copy()
+    source_copy = source.copy()
+    total_source_element_count = 0
+    total_match_element_count = 0
+
+    unmatched_depth_output: dict[str, int] = {}
+    unmatched_depth_source: dict[str, int] = {}
+
+    # loop through the output list to find match with source
+    for k, _ in output_copy.items():
+        if k in source_copy:
+            match_count = min(output_copy[k], source_copy[k])
+            total_match_element_count += match_count
+            total_source_element_count += match_count
+
+            # update the dictionary by removing already matched values
+            output_copy[k] -= match_count
+            source_copy[k] -= match_count
+
+        # add unmatched leftovers from output_copy to a new dictionary
+        element_type = k[0]
+        if element_type not in unmatched_depth_output:
+            unmatched_depth_output[element_type] = output_copy[k]
+        else:
+            unmatched_depth_output[element_type] += output_copy[k]
+
+    # add unmatched leftovers from source_copy to a new dictionary
+    unmatched_depth_source = _convert_to_frequency_without_depth(source_copy)
+
+    # loop through the source list to match any existing partial match left
+    for k, _ in unmatched_depth_source.items():
+        total_source_element_count += unmatched_depth_source[k]
+        if k in unmatched_depth_output:
+            match_count = min(unmatched_depth_output[k], unmatched_depth_source[k])
+            total_match_element_count += match_count * category_depth_weight
+
+    return min(max(total_match_element_count / total_source_element_count, 0.0), 1.0)
+
+
+def _convert_to_frequency_without_depth(d: FrequencyDict) -> dict[str, int]:
+    """
+    Takes in element frequency with depth of format (type, depth): value
+    and converts to dictionary without depth of format type: value
+    """
+    res: dict[str, int] = {}
+    for k, v in d.items():
+        element_type = k[0]
+        if element_type not in res:
+            res[element_type] = v
+        else:
+            res[element_type] += v
+    return res

unstructured/metrics/evaluate.py

@@ -0,0 +1,898 @@
+#! /usr/bin/env python3
+
+from __future__ import annotations
+
+import concurrent.futures
+import json
+import logging
+import os
+import sys
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import List, Optional, Union
+
+import numpy as np
+import pandas as pd
+from tqdm import tqdm
+
+from unstructured.metrics.element_type import (
+    calculate_element_type_percent_match,
+    get_element_type_frequency,
+)
+from unstructured.metrics.object_detection import (
+    ObjectDetectionEvalProcessor,
+)
+from unstructured.metrics.table.table_eval import TableEvalProcessor
+from unstructured.metrics.text_extraction import calculate_accuracy, calculate_percent_missing_text
+from unstructured.metrics.utils import (
+    _count,
+    _display,
+    _format_grouping_output,
+    _mean,
+    _prepare_output_cct,
+    _pstdev,
+    _read_text_file,
+    _rename_aggregated_columns,
+    _stdev,
+    _write_to_file,
+)
+
+logger = logging.getLogger("unstructured.eval")
+handler = logging.StreamHandler()
+handler.name = "eval_log_handler"
+formatter = logging.Formatter("%(asctime)s %(processName)-10s %(levelname)-8s %(message)s")
+handler.setFormatter(formatter)
+
+# Only want to add the handler once
+if "eval_log_handler" not in [h.name for h in logger.handlers]:
+    logger.addHandler(handler)
+
+logger.setLevel(logging.DEBUG)
+
+AGG_HEADERS = ["metric", "average", "sample_sd", "population_sd", "count"]
+AGG_HEADERS_MAPPING = {
+    "index": "metric",
+    "_mean": "average",
+    "_stdev": "sample_sd",
+    "_pstdev": "population_sd",
+    "_count": "count",
+}
+OUTPUT_TYPE_OPTIONS = ["json", "txt"]
+
+
+@dataclass
+class BaseMetricsCalculator(ABC):
+    """Foundation class for specialized metrics calculators.
+
+    It provides a common interface for calculating metrics based on outputs and ground truths.
+    Those can be provided as either directories or lists of files.
+    """
+
+    documents_dir: str | Path
+    ground_truths_dir: str | Path
+
+    def __post_init__(self):
+        """Discover all files in the provided directories."""
+        self.documents_dir = Path(self.documents_dir).resolve()
+        self.ground_truths_dir = Path(self.ground_truths_dir).resolve()
+
+        # -- auto-discover all files in the directories --
+        self._document_paths = [
+            path.relative_to(self.documents_dir)
+            for path in self.documents_dir.glob("*")
+            if path.is_file()
+        ]
+        self._ground_truth_paths = [
+            path.relative_to(self.ground_truths_dir)
+            for path in self.ground_truths_dir.glob("*")
+            if path.is_file()
+        ]
+
+    @property
+    @abstractmethod
+    def default_tsv_name(self):
+        """Default name for the per-document metrics TSV file."""
+
+    @property
+    @abstractmethod
+    def default_agg_tsv_name(self):
+        """Default name for the aggregated metrics TSV file."""
+
+    @abstractmethod
+    def _generate_dataframes(self, rows: list) -> tuple[pd.DataFrame, pd.DataFrame]:
+        """Generates pandas DataFrames from the list of rows.
+
+        The first DF (index 0) is a dataframe containing metrics per file.
+        The second DF (index 1) is a dataframe containing the aggregated
+            metrics.
+        """
+
+    def on_files(
+        self,
+        document_paths: Optional[list[str | Path]] = None,
+        ground_truth_paths: Optional[list[str | Path]] = None,
+    ) -> BaseMetricsCalculator:
+        """Overrides the default list of files to process."""
+        if document_paths:
+            self._document_paths = [Path(p) for p in document_paths]
+
+        if ground_truth_paths:
+            self._ground_truth_paths = [Path(p) for p in ground_truth_paths]
+
+        return self
+
+    def calculate(
+        self,
+        executor: Optional[concurrent.futures.Executor] = None,
+        export_dir: Optional[str | Path] = None,
+        visualize_progress: bool = True,
+        display_agg_df: bool = True,
+    ) -> pd.DataFrame:
+        """Calculates metrics for each document using the provided executor.
+
+        * Optionally, the results can be exported and displayed.
+        * It loops through the list of structured output from all of `documents_dir` or
+        selected files from `document_paths`, and compares them with gold-standard
+        of the same file name under `ground_truths_dir` or selected files from `ground_truth_paths`.
+
+        Args:
+            executor: concurrent.futures.Executor instance
+            export_dir: directory to export the results
+            visualize_progress: whether to display progress bar
+            display_agg_df: whether to display the aggregated results
+
+        Returns:
+            Metrics for each document as a pandas DataFrame
+        """
+        if executor is None:
+            executor = self._default_executor()
+        rows = self._process_all_documents(executor, visualize_progress)
+        df, agg_df = self._generate_dataframes(rows)
+
+        if export_dir is not None:
+            _write_to_file(export_dir, self.default_tsv_name, df)
+            _write_to_file(export_dir, self.default_agg_tsv_name, agg_df)
+
+        if display_agg_df is True:
+            _display(agg_df)
+        return df
+
+    @classmethod
+    def _default_executor(cls):
+        max_processors = int(os.environ.get("MAX_PROCESSES", os.cpu_count()))
+        logger.info(f"Configuring a pool of {max_processors} processors for parallel processing.")
+        return cls._get_executor_class()(max_workers=max_processors)
+
+    @classmethod
+    def _get_executor_class(
+        cls,
+    ) -> type[concurrent.futures.ThreadPoolExecutor] | type[concurrent.futures.ProcessPoolExecutor]:
+        return concurrent.futures.ProcessPoolExecutor
+
+    def _process_all_documents(
+        self, executor: concurrent.futures.Executor, visualize_progress: bool
+    ) -> list:
+        """Triggers processing of all documents using the provided executor.
+
+        Failures are omitted from the returned result.
+        """
+        with executor:
+            return [
+                row
+                for row in tqdm(
+                    executor.map(self._try_process_document, self._document_paths),
+                    total=len(self._document_paths),
+                    leave=False,
+                    disable=not visualize_progress,
+                )
+                if row is not None
+            ]
+
+    def _try_process_document(self, doc: Path) -> Optional[list]:
+        """Safe wrapper around the document processing method."""
+        logger.info(f"Processing {doc}")
+        try:
+            return self._process_document(doc)
+        except Exception as e:
+            logger.error(f"Failed to process document {doc}: {e}")
+            return None
+
+    @abstractmethod
+    def _process_document(self, doc: Path) -> Optional[list]:
+        """Should return all metadata and metrics for a single document."""
+
+
+@dataclass
+class TableStructureMetricsCalculator(BaseMetricsCalculator):
+    """Calculates the following metrics for tables:
+        - tables found accuracy
+        - table-level accuracy
+        - element in column index accuracy
+        - element in row index accuracy
+        - element's column content accuracy
+        - element's row content accuracy
+    It also calculates the aggregated accuracy.
+    """
+
+    cutoff: Optional[float] = None
+    weighted_average: bool = True
+    include_false_positives: bool = True
+
+    def __post_init__(self):
+        super().__post_init__()
+
+    @property
+    def supported_metric_names(self):
+        return [
+            "total_tables",
+            "table_level_acc",
+            "table_detection_recall",
+            "table_detection_precision",
+            "table_detection_f1",
+            "composite_structure_acc",
+            "element_col_level_index_acc",
+            "element_row_level_index_acc",
+            "element_col_level_content_acc",
+            "element_row_level_content_acc",
+        ]
+
+    @property
+    def default_tsv_name(self):
+        return "all-docs-table-structure-accuracy.tsv"
+
+    @property
+    def default_agg_tsv_name(self):
+        return "aggregate-table-structure-accuracy.tsv"
+
+    def _process_document(self, doc: Path) -> Optional[list]:
+        doc_path = Path(doc)
+        out_filename = doc_path.stem
+        doctype = Path(out_filename).suffix
+        src_gt_filename = out_filename + ".json"
+        connector = doc_path.parts[-2] if len(doc_path.parts) > 1 else None
+
+        if src_gt_filename in self._ground_truth_paths:  # type: ignore
+            return None
+
+        prediction_file = self.documents_dir / doc
+        if not prediction_file.exists():
+            logger.warning(f"Prediction file {prediction_file} does not exist, skipping")
+            return None
+
+        ground_truth_file = self.ground_truths_dir / src_gt_filename
+        if not ground_truth_file.exists():
+            logger.warning(f"Ground truth file {ground_truth_file} does not exist, skipping")
+            return None
+
+        processor_from_text_as_html = TableEvalProcessor.from_json_files(
+            prediction_file=prediction_file,
+            ground_truth_file=ground_truth_file,
+            cutoff=self.cutoff,
+            source_type="html",
+        )
+        report_from_html = processor_from_text_as_html.process_file()
+        return [
+            out_filename,
+            doctype,
+            connector,
+            report_from_html.total_predicted_tables,
+        ] + [getattr(report_from_html, metric) for metric in self.supported_metric_names]
+
+    def _generate_dataframes(self, rows):
+        headers = [
+            "filename",
+            "doctype",
+            "connector",
+            "total_predicted_tables",
+        ] + self.supported_metric_names
+
+        df = pd.DataFrame(rows, columns=headers)
+        df["_table_weights"] = df["total_tables"]
+
+        if self.include_false_positives:
+            # we give false positive tables a 1 table worth of weight in computing table level acc
+            _fp_mask = df.total_tables.eq(0) & df.total_predicted_tables.gt(0)
+            df.loc[_fp_mask, "_table_weights"] = 1
+
+        # filter down to only those with actual and/or predicted tables
+        has_tables_df = df[df["_table_weights"] > 0]
+
+        if not self.weighted_average:
+            # for all non zero elements assign them value 1
+            df["_table_weights"] = df["_table_weights"].apply(
+                lambda table_weight: 1 if table_weight != 0 else 0
+            )
+
+        if has_tables_df.empty:
+            agg_df = pd.DataFrame(
+                [[metric, None, None, None, 0] for metric in self.supported_metric_names]
+            ).reset_index()
+        else:
+            element_metrics_results = {}
+            for metric in self.supported_metric_names:
+                metric_df = has_tables_df[has_tables_df[metric].notnull()]
+                agg_metric = metric_df[metric].agg([_stdev, _pstdev, _count]).transpose()
+                if metric.startswith("total_tables"):
+                    agg_metric["_mean"] = metric_df[metric].mean()
+                elif metric.startswith("table_level_acc"):
+                    agg_metric["_mean"] = np.round(
+                        np.average(metric_df[metric], weights=metric_df["_table_weights"]),
+                        3,
+                    )
+                else:
+                    # false positive tables do not contribute to table structure and content
+                    # extraction metrics
+                    agg_metric["_mean"] = np.round(
+                        np.average(metric_df[metric], weights=metric_df["total_tables"]),
+                        3,
+                    )
+                if agg_metric.empty:
+                    element_metrics_results[metric] = pd.Series(
+                        data=[None, None, None, 0], index=["_mean", "_stdev", "_pstdev", "_count"]
+                    )
+                else:
+                    element_metrics_results[metric] = agg_metric
+            agg_df = pd.DataFrame(element_metrics_results).transpose().reset_index()
+        agg_df = agg_df.rename(columns=AGG_HEADERS_MAPPING)
+        return df, agg_df
+
+
+@dataclass
+class TextExtractionMetricsCalculator(BaseMetricsCalculator):
+    """Calculates text accuracy and percent missing between document and ground truth texts.
+
+    It also calculates the aggregated accuracy and percent missing.
+    """
+
+    group_by: Optional[str] = None
+    weights: tuple[int, int, int] = (1, 1, 1)
+    document_type: str = "json"
+
+    def __post_init__(self):
+        super().__post_init__()
+        self._validate_inputs()
+
+    @property
+    def default_tsv_name(self) -> str:
+        return "all-docs-cct.tsv"
+
+    @property
+    def default_agg_tsv_name(self) -> str:
+        return "aggregate-scores-cct.tsv"
+
+    def calculate(
+        self,
+        executor: Optional[concurrent.futures.Executor] = None,
+        export_dir: Optional[str | Path] = None,
+        visualize_progress: bool = True,
+        display_agg_df: bool = True,
+    ) -> pd.DataFrame:
+        """See the parent class for the method's docstring."""
+        df = super().calculate(
+            executor=executor,
+            export_dir=export_dir,
+            visualize_progress=visualize_progress,
+            display_agg_df=display_agg_df,
+        )
+
+        if export_dir is not None and self.group_by:
+            get_mean_grouping(self.group_by, df, export_dir, "text_extraction")
+        return df
+
+    def _validate_inputs(self):
+        if not self._document_paths:
+            logger.info("No output files to calculate to edit distances for, exiting")
+            sys.exit(0)
+        if self.document_type not in OUTPUT_TYPE_OPTIONS:
+            raise ValueError(
+                "Specified file type under `documents_dir` or `output_list` should be one of "
+                f"`json` or `txt`. The given file type is {self.document_type}, exiting."
+            )
+        for path in self._document_paths:
+            try:
+                path.suffixes[-1]
+            except IndexError:
+                logger.error(f"File {path} does not have a suffix, skipping")
+                continue
+            if path.suffixes[-1] != f".{self.document_type}":
+                logger.warning(
+                    "The directory contains file type inconsistent with the given input. "
+                    "Please note that some files will be skipped."
+                )
+        if not all(path.suffixes[-1] == f".{self.document_type}" for path in self._document_paths):
+            logger.warning(
+                "The directory contains file type inconsistent with the given input. "
+                "Please note that some files will be skipped."
+            )
+
+    def _process_document(self, doc: Path) -> Optional[list]:
+        filename = doc.stem
+        doctype = doc.suffixes[-2]
+        connector = doc.parts[0] if len(doc.parts) > 1 else None
+
+        output_cct, source_cct = self._get_ccts(doc)
+        # NOTE(amadeusz): Levenshtein distance calculation takes too long
+        # skip it if file sizes differ wildly
+        if 0.5 < len(output_cct.encode()) / len(source_cct.encode()) < 2.0:
+            accuracy = round(calculate_accuracy(output_cct, source_cct, self.weights), 3)
+        else:
+            # 0.01 to distinguish it was set manually
+            accuracy = 0.01
+        percent_missing = round(calculate_percent_missing_text(output_cct, source_cct), 3)
+        return [filename, doctype, connector, accuracy, percent_missing]
+
+    def _get_ccts(self, doc: Path) -> tuple[str, str]:
+        output_cct = _prepare_output_cct(
+            docpath=self.documents_dir / doc, output_type=self.document_type
+        )
+        source_cct = _read_text_file(self.ground_truths_dir / doc.with_suffix(".txt"))
+
+        return output_cct, source_cct
+
+    def _generate_dataframes(self, rows):
+        headers = ["filename", "doctype", "connector", "cct-accuracy", "cct-%missing"]
+        df = pd.DataFrame(rows, columns=headers)
+
+        acc = df[["cct-accuracy"]].agg([_mean, _stdev, _pstdev, _count]).transpose()
+        miss = df[["cct-%missing"]].agg([_mean, _stdev, _pstdev, _count]).transpose()
+        if acc.shape[1] == 0 and miss.shape[1] == 0:
+            agg_df = pd.DataFrame(columns=AGG_HEADERS)
+        else:
+            agg_df = pd.concat((acc, miss)).reset_index()
+            agg_df.columns = AGG_HEADERS
+
+        return df, agg_df
+
+
+@dataclass
+class ElementTypeMetricsCalculator(BaseMetricsCalculator):
+    """
+    Calculates element type frequency accuracy, percent missing and
+    aggregated accuracy between document and ground truth.
+    """
+
+    group_by: Optional[str] = None
+
+    def calculate(
+        self,
+        executor: Optional[concurrent.futures.Executor] = None,
+        export_dir: Optional[str | Path] = None,
+        visualize_progress: bool = True,
+        display_agg_df: bool = False,
+    ) -> pd.DataFrame:
+        """See the parent class for the method's docstring."""
+        df = super().calculate(
+            executor=executor,
+            export_dir=export_dir,
+            visualize_progress=visualize_progress,
+            display_agg_df=display_agg_df,
+        )
+
+        if export_dir is not None and self.group_by:
+            get_mean_grouping(self.group_by, df, export_dir, "element_type")
+        return df
+
+    @property
+    def default_tsv_name(self) -> str:
+        return "all-docs-element-type-frequency.tsv"
+
+    @property
+    def default_agg_tsv_name(self) -> str:
+        return "aggregate-scores-element-type.tsv"
+
+    def _process_document(self, doc: Path) -> Optional[list]:
+        filename = doc.stem
+        doctype = doc.suffixes[-2]
+        connector = doc.parts[0] if len(doc.parts) > 1 else None
+
+        output = get_element_type_frequency(_read_text_file(self.documents_dir / doc))
+        source = get_element_type_frequency(
+            _read_text_file(self.ground_truths_dir / doc.with_suffix(".json"))
+        )
+        accuracy = round(calculate_element_type_percent_match(output, source), 3)
+        return [filename, doctype, connector, accuracy]
+
+    def _generate_dataframes(self, rows):
+        headers = ["filename", "doctype", "connector", "element-type-accuracy"]
+        df = pd.DataFrame(rows, columns=headers)
+        if df.empty:
+            agg_df = pd.DataFrame(["element-type-accuracy", None, None, None, 0]).transpose()
+        else:
+            agg_df = df.agg({"element-type-accuracy": [_mean, _stdev, _pstdev, _count]}).transpose()
+            agg_df = agg_df.reset_index()
+
+        agg_df.columns = AGG_HEADERS
+
+        return df, agg_df
+
+
+def get_mean_grouping(
+    group_by: str,
+    data_input: Union[pd.DataFrame, str],
+    export_dir: str,
+    eval_name: str,
+    agg_name: Optional[str] = None,
+    export_filename: Optional[str] = None,
+) -> None:
+    """Aggregates accuracy and missing metrics by column name 'doctype' or 'connector',
+    or 'all' for all rows. Export to TSV.
+    If `all`, passing export_name is recommended.
+
+    Args:
+        group_by (str): Grouping category ('doctype' or 'connector' or 'all').
+        data_input (Union[pd.DataFrame, str]): DataFrame or path to a CSV/TSV file.
+        export_dir (str): Directory for the exported TSV file.
+        eval_name (str): Evaluated metric ('text_extraction' or 'element_type').
+        agg_name (str, optional): String to use with export filename. Default is `cct` for
+            group_by `text_extraction` and `element-type` for `element_type`
+        export_name (str, optional): Export filename.
+    """
+    if group_by not in ("doctype", "connector") and group_by != "all":
+        raise ValueError("Invalid grouping category. Returning a non-group evaluation.")
+
+    if eval_name == "text_extraction":
+        agg_fields = ["cct-accuracy", "cct-%missing"]
+        agg_name = "cct"
+    elif eval_name == "element_type":
+        agg_fields = ["element-type-accuracy"]
+        agg_name = "element-type"
+    elif eval_name == "object_detection":
+        agg_fields = ["f1_score", "m_ap"]
+        agg_name = "object-detection"
+    else:
+        raise ValueError(
+            f"Unknown metric for eval {eval_name}. "
+            f"Expected `text_extraction` or `element_type` or `table_extraction`."
+        )
+
+    if isinstance(data_input, str):
+        if not os.path.exists(data_input):
+            raise FileNotFoundError(f"File {data_input} not found.")
+        if data_input.endswith(".csv"):
+            df = pd.read_csv(data_input, header=None)
+        elif data_input.endswith(".tsv"):
+            df = pd.read_csv(data_input, sep="\t")
+        elif data_input.endswith(".txt"):
+            df = pd.read_csv(data_input, sep="\t", header=None)
+        else:
+            raise ValueError("Please provide a .csv or .tsv file.")
+    else:
+        df = data_input
+
+    if df.empty:
+        raise SystemExit("Data is empty. Exiting.")
+    elif group_by != "all" and (group_by not in df.columns or df[group_by].isnull().all()):
+        raise SystemExit(
+            f"Data cannot be aggregated by `{group_by}`."
+            f" Check if it's empty or the column is missing/empty."
+        )
+
+    grouped_df = []
+    if group_by and group_by != "all":
+        for field in agg_fields:
+            grouped_df.append(
+                _rename_aggregated_columns(
+                    df.groupby(group_by).agg({field: [_mean, _stdev, _pstdev, _count]})
+                )
+            )
+    if group_by == "all":
+        df["grouping_key"] = 0
+        for field in agg_fields:
+            grouped_df.append(
+                _rename_aggregated_columns(
+                    df.groupby("grouping_key").agg({field: [_mean, _stdev, _pstdev, _count]})
+                )
+            )
+    grouped_df = _format_grouping_output(*grouped_df)
+    if "grouping_key" in grouped_df.columns.get_level_values(0):
+        grouped_df = grouped_df.drop("grouping_key", axis=1, level=0)
+
+    if export_filename:
+        if not export_filename.endswith(".tsv"):
+            export_filename = export_filename + ".tsv"
+        _write_to_file(export_dir, export_filename, grouped_df)
+    else:
+        _write_to_file(export_dir, f"all-{group_by}-agg-{agg_name}.tsv", grouped_df)
+
+
+def filter_metrics(
+    data_input: Union[str, pd.DataFrame],
+    filter_list: Union[str, List[str]],
+    filter_by: str = "filename",
+    export_filename: Optional[str] = None,
+    export_dir: str = "metrics",
+    return_type: str = "file",
+) -> Optional[pd.DataFrame]:
+    """Reads the data_input file and filter only selected row available in filter_list.
+
+    Args:
+        data_input (str, dataframe): the source data, path to file or dataframe
+        filter_list (str, list): the filter, path to file or list of string
+        filter_by (str): data_input's column to filter the filter_list to
+        export_filename (str, optional): export filename. required when return_type is "file"
+        export_dir (str, optional): export directory. default to <current directory>/metrics
+        return_type (str): "file" or "dataframe"
+    """
+    if isinstance(data_input, str):
+        if not os.path.exists(data_input):
+            raise FileNotFoundError(f"File {data_input} not found.")
+        if data_input.endswith(".csv"):
+            df = pd.read_csv(data_input, header=None)
+        elif data_input.endswith(".tsv"):
+            df = pd.read_csv(data_input, sep="\t")
+        elif data_input.endswith(".txt"):
+            df = pd.read_csv(data_input, sep="\t", header=None)
+        else:
+            raise ValueError("Please provide a .csv or .tsv file.")
+    else:
+        df = data_input
+
+    if isinstance(filter_list, str):
+        if not os.path.exists(filter_list):
+            raise FileNotFoundError(f"File {filter_list} not found.")
+        if filter_list.endswith(".csv"):
+            filter_df = pd.read_csv(filter_list, header=None)
+        elif filter_list.endswith(".tsv"):
+            filter_df = pd.read_csv(filter_list, sep="\t")
+        elif filter_list.endswith(".txt"):
+            filter_df = pd.read_csv(filter_list, sep="\t", header=None)
+        else:
+            raise ValueError("Please provide a .csv or .tsv file.")
+        filter_list = filter_df.iloc[:, 0].astype(str).values.tolist()
+    elif not isinstance(filter_list, list):
+        raise ValueError("Please provide a List of strings or path to file.")
+
+    if filter_by not in df.columns:
+        raise ValueError("`filter_by` key does not exists in the data provided.")
+
+    res = df[df[filter_by].isin(filter_list)]
+
+    if res.empty:
+        raise SystemExit("No common file names between data_input and filter_list. Exiting.")
+
+    if return_type == "dataframe":
+        return res
+    elif return_type == "file" and export_filename:
+        _write_to_file(export_dir, export_filename, res)
+    elif return_type == "file" and not export_filename:
+        raise ValueError("Please provide `export_filename`.")
+    else:
+        raise ValueError("Return type must be either `dataframe` or `file`.")
+
+
+@dataclass
+class ObjectDetectionMetricsCalculatorBase(BaseMetricsCalculator, ABC):
+    """
+    Calculates object detection metrics for each document:
+    - f1 score
+    - precision
+    - recall
+    - average precision (mAP)
+    It also calculates aggregated metrics.
+    """
+
+    def __post_init__(self):
+        super().__post_init__()
+        self._document_paths = [
+            path.relative_to(self.documents_dir)
+            for path in self.documents_dir.rglob("analysis/*/layout_dump/object_detection.json")
+            if path.is_file()
+        ]
+
+    @property
+    def supported_metric_names(self):
+        return ["f1_score", "precision", "recall", "m_ap"]
+
+    @property
+    def default_tsv_name(self):
+        return "all-docs-object-detection-metrics.tsv"
+
+    @property
+    def default_agg_tsv_name(self):
+        return "aggregate-object-detection-metrics.tsv"
+
+    def _find_file_in_ground_truth(self, file_stem: str) -> Optional[Path]:
+        """Find the file corresponding to OD model dump file among the set of ground truth files
+
+        The files in ground truth paths keep the original extension and have .json suffix added,
+        e.g.:
+        some_document.pdf.json
+        poster.jpg.json
+
+        To compare to `file_stem` we need to take the prefix part of the file, thus double-stem
+        is applied.
+        """
+        for path in self._ground_truth_paths:
+            if Path(path.stem).stem == file_stem:
+                return path
+        return None
+
+    def _get_paths(self, doc: Path) -> tuple(str, Path, Path):
+        """Resolves ground doctype, prediction file path and ground truth path.
+
+        As OD dump directory structure differes from other simple outputs, it needs
+        a specific processing to match the output OD dump file with corresponding
+        OD GT file.
+
+        The outputs are placed in a dicrectory structure:
+
+        analysis
+        |- document_name
+            |- layout_dump
+                |- object_detection.json
+            |- bboxes # not used in this evaluation
+
+        and the GT file is pleced in od_gt directory for given dataset
+
+        dataset_name
+        |- od_gt
+            |- document_name.pdf.json
+
+        Args:
+            doc (Path): path to the OD dump file
+
+        Returns:
+            tuple: doctype, prediction file path, ground truth path
+        """
+        od_dump_path = Path(doc)
+        file_stem = od_dump_path.parts[-3]  # we take the `document_name` - so the filename stem
+
+        src_gt_filename = self._find_file_in_ground_truth(file_stem)
+
+        if src_gt_filename not in self._ground_truth_paths:
+            raise ValueError(f"Ground truth file {src_gt_filename} not found in list of GT files")
+
+        doctype = Path(src_gt_filename.stem).suffix[1:]
+
+        prediction_file = self.documents_dir / doc
+        if not prediction_file.exists():
+            logger.warning(f"Prediction file {prediction_file} does not exist, skipping")
+            raise ValueError(f"Prediction file {prediction_file} does not exist")
+
+        ground_truth_file = self.ground_truths_dir / src_gt_filename
+        if not ground_truth_file.exists():
+            logger.warning(f"Ground truth file {ground_truth_file} does not exist, skipping")
+            raise ValueError(f"Ground truth file {ground_truth_file} does not exist")
+
+        return doctype, prediction_file, ground_truth_file
+
+    def _generate_dataframes(self, rows) -> tuple[pd.DataFrame, pd.DataFrame]:
+        headers = ["filename", "doctype", "connector"] + self.supported_metric_names
+        df = pd.DataFrame(rows, columns=headers)
+
+        if df.empty:
+            agg_df = pd.DataFrame(columns=AGG_HEADERS)
+        else:
+            element_metrics_results = {}
+            for metric in self.supported_metric_names:
+                metric_df = df[df[metric].notnull()]
+                agg_metric = metric_df[metric].agg([_mean, _stdev, _pstdev, _count]).transpose()
+                if agg_metric.empty:
+                    element_metrics_results[metric] = pd.Series(
+                        data=[None, None, None, 0], index=["_mean", "_stdev", "_pstdev", "_count"]
+                    )
+                else:
+                    element_metrics_results[metric] = agg_metric
+            agg_df = pd.DataFrame(element_metrics_results).transpose().reset_index()
+        agg_df.columns = AGG_HEADERS
+
+        return df, agg_df
+
+
+class ObjectDetectionPerClassMetricsCalculator(ObjectDetectionMetricsCalculatorBase):
+    def __post_init__(self):
+        super().__post_init__()
+        self.per_class_metric_names: list[str] | None = None
+        self._set_supported_metrics()
+
+    @property
+    def supported_metric_names(self):
+        if self.per_class_metric_names:
+            return self.per_class_metric_names
+        else:
+            raise ValueError("per_class_metrics not initialized - cannot get class names")
+
+    @property
+    def default_tsv_name(self):
+        return "all-docs-object-detection-metrics-per-class.tsv"
+
+    @property
+    def default_agg_tsv_name(self):
+        return "aggregate-object-detection-metrics-per-class.tsv"
+
+    def _process_document(self, doc: Path) -> Optional[list]:
+        """Calculate both class-aggregated and per-class metrics for a single document.
+
+        Args:
+            doc (Path): path to the OD dump file
+
+        Returns:
+            tuple: a tuple of aggregated and per-class metrics for a single document
+        """
+        try:
+            doctype, prediction_file, ground_truth_file = self._get_paths(doc)
+        except ValueError as e:
+            logger.error(f"Failed to process document {doc}: {e}")
+            return None
+
+        processor = ObjectDetectionEvalProcessor.from_json_files(
+            prediction_file_path=prediction_file,
+            ground_truth_file_path=ground_truth_file,
+        )
+        _, per_class_metrics = processor.get_metrics()
+
+        per_class_metrics_row = [
+            ground_truth_file.stem,
+            doctype,
+            None,  # connector
+        ]
+
+        for combined_metric_name in self.supported_metric_names:
+            metric = "_".join(combined_metric_name.split("_")[:-1])
+            class_name = combined_metric_name.split("_")[-1]
+            class_metrics = getattr(per_class_metrics, metric)
+            per_class_metrics_row.append(class_metrics[class_name])
+        return per_class_metrics_row
+
+    def _set_supported_metrics(self):
+        """Sets the supported metrics based on the classes found in the ground truth files.
+        The difference between per class and aggregated calculator is that the list of classes
+        (so the metrics) bases on the contents of the GT / prediction files.
+        """
+        metrics = ["f1_score", "precision", "recall", "m_ap"]
+        classes = set()
+        for gt_file in self._ground_truth_paths:
+            gt_file_path = self.ground_truths_dir / gt_file
+            with open(gt_file_path) as f:
+                gt = json.load(f)
+                gt_classes = gt["object_detection_classes"]
+                classes.update(gt_classes)
+        per_class_metric_names = []
+        for metric in metrics:
+            for class_name in classes:
+                per_class_metric_names.append(f"{metric}_{class_name}")
+        self.per_class_metric_names = sorted(per_class_metric_names)
+
+
+class ObjectDetectionAggregatedMetricsCalculator(ObjectDetectionMetricsCalculatorBase):
+    """Calculates object detection metrics for each document and aggregates by all classes"""
+
+    @property
+    def supported_metric_names(self):
+        return ["f1_score", "precision", "recall", "m_ap"]
+
+    @property
+    def default_tsv_name(self):
+        return "all-docs-object-detection-metrics.tsv"
+
+    @property
+    def default_agg_tsv_name(self):
+        return "aggregate-object-detection-metrics.tsv"
+
+    def _process_document(self, doc: Path) -> Optional[list]:
+        """Calculate both class-aggregated and per-class metrics for a single document.
+
+        Args:
+            doc (Path): path to the OD dump file
+
+        Returns:
+            list: a list of aggregated metrics for a single document
+        """
+        try:
+            doctype, prediction_file, ground_truth_file = self._get_paths(doc)
+        except ValueError as e:
+            logger.error(f"Failed to process document {doc}: {e}")
+            return None
+
+        processor = ObjectDetectionEvalProcessor.from_json_files(
+            prediction_file_path=prediction_file,
+            ground_truth_file_path=ground_truth_file,
+        )
+        metrics, _ = processor.get_metrics()
+
+        return [
+            ground_truth_file.stem,
+            doctype,
+            None,  # connector
+        ] + [getattr(metrics, metric) for metric in self.supported_metric_names]

unstructured/metrics/object_detection.py

@@ -0,0 +1,721 @@
+"""
+Implements object detection metrics: average precision, precision, recall, and f1 score.
+"""
+
+import json
+from dataclasses import dataclass
+from pathlib import Path
+
+import numpy as np
+import torch
+
+IOU_THRESHOLDS = torch.tensor(
+    [0.5000, 0.5500, 0.6000, 0.6500, 0.7000, 0.7500, 0.8000, 0.8500, 0.9000, 0.9500]
+)
+SCORE_THRESHOLD = 0.1
+RECALL_THRESHOLDS = torch.arange(0, 1.01, 0.01)
+
+
+@dataclass
+class ObjectDetectionAggregatedEvaluation:
+    """Class representing a gathered class-aggregated object detection metrics"""
+
+    f1_score: float
+    precision: float
+    recall: float
+    m_ap: float
+
+
+@dataclass
+class ObjectDetectionPerClassEvaluation:
+    """Class representing a gathered object detection metrics per-class"""
+
+    f1_score: dict[str, float]
+    precision: dict[str, float]
+    recall: dict[str, float]
+    m_ap: dict[str, float]
+
+    @classmethod
+    def from_tensors(cls, ap, precision, recall, f1, class_labels):
+        f1_score = {class_labels[i]: f1[i] for i in range(len(class_labels))}
+        precision = {class_labels[i]: precision[i] for i in range(len(class_labels))}
+        recall = {class_labels[i]: recall[i] for i in range(len(class_labels))}
+        m_ap = {class_labels[i]: ap[i] for i in range(len(class_labels))}
+
+        return cls(f1_score, precision, recall, m_ap)
+
+
+class ObjectDetectionEvalProcessor:
+    iou_thresholds = IOU_THRESHOLDS
+    score_threshold = SCORE_THRESHOLD
+    recall_thresholds = RECALL_THRESHOLDS
+
+    def __init__(
+        self,
+        document_preds: list[torch.Tensor],
+        document_targets: list[torch.Tensor],
+        pages_height: list[int],
+        pages_width: list[int],
+        class_labels: list[str],
+        device: str = "cpu",
+    ):
+        """
+        Initializes the ObjectDetection prediction and ground truth.
+
+        Args:
+            document_preds (list):      list (of length pages of document) of
+                                        Tensors of shape (num_predictions, 6)
+                                        format: (x1, y1, x2, y2, confidence,class_label)
+                                        where x1,y1,x2,y2 are according to image size
+            document_targets (list):    list (of length pages of document) of
+                                        Tensors of shape (num_targets, 6)
+                                        format: (label, x1, y1, x2, y2)
+                                        where x,y,w,h are according to image size
+            pages_height (list):        list of height of each page in the document
+            pages_width (list):         list of width of each page in the document
+            class_labels (list):        list of class labels
+        """
+        self.device = device
+        self.document_preds = [pred.to(device) for pred in document_preds]
+        self.document_targets = [target.to(device) for target in document_targets]
+        self.pages_height = pages_height
+        self.pages_width = pages_width
+        self.class_labels = class_labels
+
+    @classmethod
+    def from_json_files(
+        cls,
+        prediction_file_path: Path,
+        ground_truth_file_path: Path,
+    ) -> "ObjectDetectionEvalProcessor":
+        """
+        Initializes the ObjectDetection prediction and ground truth,
+        and converts the data to the required format.
+
+        Args:
+            prediction_file_path (Path): path to json file with predictions dump from OD model
+            ground_truth_file_path (Path): path to json file with OD ground truth data
+        """
+        # TODO: Test after https://unstructured-ai.atlassian.net/browse/ML-92
+        # is done.
+        with open(prediction_file_path) as f:
+            predictions_data = json.load(f)
+        with open(ground_truth_file_path) as f:
+            ground_truth_data = json.load(f)
+
+        assert sorted(predictions_data["object_detection_classes"]) == sorted(
+            ground_truth_data["object_detection_classes"]
+        ), "Classes in predictions and ground truth do not match."
+        assert len(predictions_data["pages"]) == len(ground_truth_data["pages"]), (
+            "Pages number in predictions and ground truth do not match."
+        )
+        for pred_page, gt_page in zip(
+            sorted(predictions_data["pages"], key=lambda p: p["number"]),
+            sorted(ground_truth_data["pages"], key=lambda p: p["number"]),
+        ):
+            assert pred_page["number"] == gt_page["number"], (
+                f"Page numbers in predictions {prediction_file_path.name} "
+                f"({pred_page['number']}) and ground truth {ground_truth_file_path.name} "
+                f"({gt_page['number']}) do not match."
+            )
+            page_num = pred_page["number"]
+
+            # TODO: translate the bboxes instead of raising error
+            assert pred_page["size"] == gt_page["size"], (
+                f"Page sizes in predictions {prediction_file_path.name} "
+                f"({pred_page['size'][0]} x {pred_page['size'][1]}) "
+                f"and ground truth {ground_truth_file_path.name} ({gt_page['size'][0]} x "
+                f"{gt_page['size'][1]}) do not match for page {page_num}."
+            )
+
+        class_labels = predictions_data["object_detection_classes"]
+        document_preds = cls._process_data(predictions_data, class_labels, prediction=True)
+        document_targets = cls._process_data(ground_truth_data, class_labels)
+        pages_height, pages_width = cls._parse_page_dimensions(predictions_data)
+
+        return cls(document_preds, document_targets, pages_height, pages_width, class_labels)
+
+    def get_metrics(
+        self,
+    ) -> tuple[ObjectDetectionAggregatedEvaluation, ObjectDetectionPerClassEvaluation]:
+        """Get per document OD metrics.
+
+        Returns:
+            tuple: Tuple of ObjectDetectionAggregatedEvaluation and
+                ObjectDetectionPerClassEvaluation
+        """
+        document_matchings = []
+        for preds, targets, height, width in zip(
+            self.document_preds, self.document_targets, self.pages_height, self.pages_width
+        ):
+            # iterate over each page
+            page_matching_tensors = self._compute_page_detection_matching(
+                preds=preds,
+                targets=targets,
+                height=height,
+                width=width,
+            )
+            document_matchings.append(page_matching_tensors)
+
+        # compute metrics for all detections and targets
+        mean_ap, mean_precision, mean_recall, mean_f1 = (
+            -1.0,
+            -1.0,
+            -1.0,
+            -1.0,
+        )
+
+        num_cls = len(self.class_labels)
+        mean_ap_per_class = np.full(num_cls, np.nan)
+        mean_precision_per_class = np.full(num_cls, np.nan)
+        mean_recall_per_class = np.full(num_cls, np.nan)
+        mean_f1_per_class = np.full(num_cls, np.nan)
+
+        if len(document_matchings):
+            matching_info_tensors = [torch.cat(x, 0) for x in list(zip(*document_matchings))]
+
+            # shape (n_class, nb_iou_thresh)
+            (
+                ap_per_present_classes,
+                precision_per_present_classes,
+                recall_per_present_classes,
+                f1_per_present_classes,
+                present_classes,
+            ) = self._compute_detection_metrics(
+                *matching_info_tensors,
+            )
+
+            # Precision, recall and f1 are computed for IoU threshold range, averaged over classes
+            # results before version 3.0.4 (Dec 11 2022) were computed only for smallest value
+            # (i.e IoU 0.5 if metric is @0.5:0.95)
+            mean_precision, mean_recall, mean_f1 = (
+                precision_per_present_classes.mean(),
+                recall_per_present_classes.mean(),
+                f1_per_present_classes.mean(),
+            )
+
+            # MaP is averaged over IoU thresholds and over classes
+            mean_ap = ap_per_present_classes.mean()
+
+            # Fill array of per-class AP scores with values for classes that were present in the
+            # dataset
+            ap_per_class = ap_per_present_classes.mean(1)
+            precision_per_class = precision_per_present_classes.mean(1)
+            recall_per_class = recall_per_present_classes.mean(1)
+            f1_per_class = f1_per_present_classes.mean(1)
+            for i, class_index in enumerate(present_classes):
+                mean_ap_per_class[class_index] = float(ap_per_class[i])
+
+                mean_precision_per_class[class_index] = float(precision_per_class[i])
+                mean_recall_per_class[class_index] = float(recall_per_class[i])
+                mean_f1_per_class[class_index] = float(f1_per_class[i])
+
+        od_per_class_evaluation = ObjectDetectionPerClassEvaluation.from_tensors(
+            ap=mean_ap_per_class,
+            precision=mean_precision_per_class,
+            recall=mean_recall_per_class,
+            f1=mean_f1_per_class,
+            class_labels=self.class_labels,
+        )
+
+        od_evaluation = ObjectDetectionAggregatedEvaluation(
+            f1_score=float(mean_f1),
+            precision=float(mean_precision),
+            recall=float(mean_recall),
+            m_ap=float(mean_ap),
+        )
+
+        return od_evaluation, od_per_class_evaluation
+
+    @staticmethod
+    def _parse_page_dimensions(data: dict) -> tuple[list, list]:
+        """
+        Process the page dimensions from the json file to the required format.
+        """
+        pages_height = []
+        pages_width = []
+        for page in data["pages"]:
+            pages_height.append(page["size"]["height"])
+            pages_width.append(page["size"]["width"])
+        return pages_height, pages_width
+
+    @staticmethod
+    def _process_data(data: dict, class_labels, prediction: bool = False) -> list[dict]:
+        """
+        Process the elements from the json file to the required format.
+        """
+        pages_list = []
+        for page in data["pages"]:
+            page_elements = []
+            for element in page["elements"]:
+                # Extract coordinates, confidence, and class label from each prediction
+                class_label = element["type"]
+                class_idx = class_labels.index(class_label)
+                x1, y1, x2, y2 = element["bbox"]
+                if prediction:
+                    confidence = element["prob"]
+                    page_elements.append([x1, y1, x2, y2, confidence, class_idx])
+                else:
+                    page_elements.append([class_idx, x1, y1, x2, y2])
+            page_tensor = torch.tensor(page_elements)
+            pages_list.append(page_tensor)
+
+        return pages_list
+
+    @staticmethod
+    def _get_top_k_idx_per_cls(
+        preds_scores: torch.Tensor, preds_cls: torch.Tensor, top_k: int
+    ) -> torch.Tensor:
+        # From: https://github.com/Deci-AI/super-gradients/blob/master/src/super_gradients/training/utils/detection_utils.py  # noqa E501
+        """
+        Get the indexes of all the top k predictions for every class
+
+        Args:
+            preds_scores:   The confidence scores, vector of shape (n_pred)
+            preds_cls:      The predicted class, vector of shape (n_pred)
+            top_k:          Number of predictions to keep per class, ordered by confidence score
+
+        Returns:
+            top_k_idx:     Indexes of the top k predictions. length <= (k * n_unique_class)
+        """
+        n_unique_cls = torch.max(preds_cls)
+        mask = preds_cls.view(-1, 1) == torch.arange(
+            n_unique_cls + 1, device=preds_scores.device
+        ).view(1, -1)
+        preds_scores_per_cls = preds_scores.view(-1, 1) * mask
+
+        sorted_scores_per_cls, sorting_idx = preds_scores_per_cls.sort(0, descending=True)
+        idx_with_satisfying_scores = sorted_scores_per_cls[:top_k, :].nonzero(as_tuple=False)
+        top_k_idx = sorting_idx[idx_with_satisfying_scores.split(1, dim=1)]
+        return top_k_idx.view(-1)
+
+    @staticmethod
+    def _change_bbox_bounds_for_image_size(
+        boxes: np.ndarray, img_shape: tuple[int, int]
+    ) -> np.ndarray:
+        # From: https://github.com/Deci-AI/super-gradients/blob/master/src/super_gradients/training/utils/detection_utils.py  # noqa E501
+        """
+        Clips bboxes to image boundaries.
+
+        Args:
+            bboxes:         Input bounding boxes in XYXY format of [..., 4] shape
+            img_shape:      Image shape (height, width).
+        Returns:
+            clipped_boxes:  Clipped bboxes in XYXY format of [..., 4] shape
+        """
+        boxes[..., [0, 2]] = boxes[..., [0, 2]].clip(min=0, max=img_shape[1])
+        boxes[..., [1, 3]] = boxes[..., [1, 3]].clip(min=0, max=img_shape[0])
+        return boxes
+
+    @staticmethod
+    def _box_iou(box1: torch.Tensor, box2: torch.Tensor) -> torch.Tensor:
+        # From: https://github.com/Deci-AI/super-gradients/blob/master/src/super_gradients/training/utils/detection_utils.py  # noqa E501
+        """
+        Return intersection-over-union (Jaccard index) of boxes.
+        Both sets of boxes are expected to be in (x1, y1, x2, y2) format.
+
+        Args:
+            box1: Tensor of shape [N, 4]
+            box2: Tensor of shape [M, 4]
+
+        Returns:
+            iou:    Tensor of shape [N, M]: the NxM matrix containing the pairwise IoU values
+                    for every element in boxes1 and boxes2
+        """
+
+        def box_area(box):
+            # box = 4xn
+            return (box[2] - box[0]) * (box[3] - box[1])
+
+        area1 = box_area(box1.T)
+        area2 = box_area(box2.T)
+
+        # inter(N,M) = (rb(N,M,2) - lt(N,M,2)).clamp(0).prod(2)
+        inter = (
+            (torch.min(box1[:, None, 2:], box2[:, 2:]) - torch.max(box1[:, None, :2], box2[:, :2]))
+            .clamp(0)
+            .prod(2)
+        )
+        return inter / (area1[:, None] + area2 - inter)  # iou = inter / (area1 + area2 - inter)
+
+    def _compute_targets(
+        self,
+        preds_box_xyxy: torch.Tensor,
+        preds_cls: torch.Tensor,
+        targets_box_xyxy: torch.Tensor,
+        targets_cls: torch.Tensor,
+        preds_matched: torch.Tensor,
+        targets_matched: torch.Tensor,
+        preds_idx_to_use: torch.Tensor,
+        iou_thresholds: torch.Tensor,
+    ) -> torch.Tensor:
+        # From: https://github.com/Deci-AI/super-gradients/blob/master/src/super_gradients/training/utils/detection_utils.py  # noqa E501
+        """
+        Computes the matching targets based on IoU for regular scenarios.
+
+        Args:
+            preds_box_xyxy: (torch.Tensor) Predicted bounding boxes in XYXY format.
+            preds_cls: (torch.Tensor) Predicted classes.
+            targets_box_xyxy: (torch.Tensor) Target bounding boxes in XYXY format.
+            targets_cls: (torch.Tensor) Target classes.
+            preds_matched: (torch.Tensor) Tensor indicating which predictions are matched.
+            targets_matched: (torch.Tensor) Tensor indicating which targets are matched.
+            preds_idx_to_use: (torch.Tensor) Indices of predictions to use.
+
+        Returns:
+            targets: Computed matching targets.
+        """
+        # shape = (n_preds x n_targets)
+        iou = self._box_iou(preds_box_xyxy[preds_idx_to_use], targets_box_xyxy)
+
+        # Fill IoU values at index (i, j) with 0 when the prediction (i) and target(j)
+        # are of different class
+        # Filling with 0 is equivalent to ignore these values
+        # since with want IoU > iou_threshold > 0
+        cls_mismatch = preds_cls[preds_idx_to_use].view(-1, 1) != targets_cls.view(1, -1)
+        iou[cls_mismatch] = 0
+
+        # The matching priority is first detection confidence and then IoU value.
+        # The detection is already sorted by confidence in NMS,
+        # so here for each prediction we order the targets by iou.
+        sorted_iou, target_sorted = iou.sort(descending=True, stable=True)
+
+        # Only iterate over IoU values higher than min threshold to speed up the process
+        for pred_selected_i, target_sorted_i in (sorted_iou > iou_thresholds[0]).nonzero(
+            as_tuple=False
+        ):
+            # pred_selected_i and target_sorted_i are relative to filters/sorting,
+            # so we extract their absolute indexes
+            pred_i = preds_idx_to_use[pred_selected_i]
+            target_i = target_sorted[pred_selected_i, target_sorted_i]
+
+            # Vector[j], True when IoU(pred_i, target_i) is above the (j)th threshold
+            is_iou_above_threshold = sorted_iou[pred_selected_i, target_sorted_i] > iou_thresholds
+
+            # Vector[j], True when both pred_i and target_i are not matched yet
+            # for the (j)th threshold
+            are_candidates_free = torch.logical_and(
+                ~preds_matched[pred_i, :], ~targets_matched[target_i, :]
+            )
+
+            # Vector[j], True when (pred_i, target_i) can be matched for the (j)th threshold
+            are_candidates_good = torch.logical_and(is_iou_above_threshold, are_candidates_free)
+
+            # For every threshold (j) where target_i and pred_i can be matched together
+            # ( are_candidates_good[j]==True )
+            # fill the matching placeholders with True
+            targets_matched[target_i, are_candidates_good] = True
+            preds_matched[pred_i, are_candidates_good] = True
+
+            # When all the targets are matched with a prediction for every IoU Threshold, stop.
+            if targets_matched.all():
+                break
+
+        return preds_matched
+
+    def _compute_page_detection_matching(
+        self,
+        preds: torch.Tensor,
+        targets: torch.Tensor,
+        height: int,
+        width: int,
+        top_k: int = 100,
+        return_on_cpu: bool = True,
+    ) -> tuple:
+        # Adapted from: https://github.com/Deci-AI/super-gradients/blob/master/src/super_gradients/training/utils/detection_utils.py  # noqa E501
+        """
+        Match predictions (NMS output) and the targets (ground truth) with respect to metric
+        and confidence score for a given image.
+
+        Args:
+            preds:          Tensor of shape (num_img_predictions, 6)
+                            format: (x1, y1, x2, y2, confidence, class_label)
+                            where x1,y1,x2,y2 are according to image size
+            targets:        targets for this image of shape (num_img_targets, 5)
+                            format:     (label, x1, y1, x2, y2)
+                            where x1,y1,x2,y2 are according to image size
+            height:         dimensions of the image
+            width:          dimensions of the image
+            top_k:          Number of predictions to keep per class, ordered by confidence score
+            return_on_cpu:  If True, the output will be returned on "CPU", otherwise it will be
+                            returned on "device"
+
+        Returns:
+            preds_matched:      Tensor of shape (num_img_predictions, n_thresholds)
+                                True when prediction (i) is matched with a target with respect to
+                                the (j)th threshold
+            preds_to_ignore:    Tensor of shape (num_img_predictions, n_thresholds)
+                                True when prediction (i) is matched with a crowd target with
+                                respect to the (j)th threshold
+            preds_scores:       Tensor of shape (num_img_predictions),
+                                confidence score for every prediction
+            preds_cls:          Tensor of shape (num_img_predictions),
+                                predicted class for every prediction
+            targets_cls:        Tensor of shape (num_img_targets),
+                                ground truth class for every target
+        """
+        thresholds = self.iou_thresholds.to(device=self.device)
+        num_thresholds = len(thresholds)
+
+        if preds is None or len(preds) == 0:
+            preds_matched = torch.zeros((0, num_thresholds), dtype=torch.bool, device=self.device)
+            preds_to_ignore = torch.zeros((0, num_thresholds), dtype=torch.bool, device=self.device)
+            preds_scores = torch.tensor([], dtype=torch.float32, device=self.device)
+            preds_cls = torch.tensor([], dtype=torch.float32, device=self.device)
+            targets_cls = targets[:, 0].to(device=self.device)
+            return preds_matched, preds_to_ignore, preds_scores, preds_cls, targets_cls
+
+        preds_matched = torch.zeros(
+            len(preds), num_thresholds, dtype=torch.bool, device=self.device
+        )
+        targets_matched = torch.zeros(
+            len(targets), num_thresholds, dtype=torch.bool, device=self.device
+        )
+        preds_to_ignore = torch.zeros(
+            len(preds), num_thresholds, dtype=torch.bool, device=self.device
+        )
+
+        preds_cls, preds_box, preds_scores = preds[:, -1], preds[:, 0:4], preds[:, 4]
+        targets_cls, targets_box = targets[:, 0], targets[:, 1:5]
+
+        # Ignore all but the predictions that were top_k for their class
+        preds_idx_to_use = self._get_top_k_idx_per_cls(preds_scores, preds_cls, top_k)
+        preds_to_ignore[:, :] = True
+        preds_to_ignore[preds_idx_to_use] = False
+
+        if len(targets) > 0:  # or len(crowd_targets) > 0:
+            self._change_bbox_bounds_for_image_size(preds, (height, width))
+
+            preds_matched = self._compute_targets(
+                preds_box,
+                preds_cls,
+                targets_box,
+                targets_cls,
+                preds_matched,
+                targets_matched,
+                preds_idx_to_use,
+                thresholds,
+            )
+
+        return preds_matched, preds_to_ignore, preds_scores, preds_cls, targets_cls
+
+    def _compute_detection_metrics(
+        self,
+        preds_matched: torch.Tensor,
+        preds_to_ignore: torch.Tensor,
+        preds_scores: torch.Tensor,
+        preds_cls: torch.Tensor,
+        targets_cls: torch.Tensor,
+    ) -> tuple:
+        # Adapted from: https://github.com/Deci-AI/super-gradients/blob/master/src/super_gradients/training/utils/detection_utils.py  # noqa E501
+        """
+        Compute the list of precision, recall, MaP and f1 for every class.
+
+        Args:
+            preds_matched:      Tensor of shape (num_predictions, n_iou_thresholds)
+                                True when prediction (i) is matched with a target with respect
+                                to the (j)th IoU threshold
+            preds_to_ignore     Tensor of shape (num_predictions, n_iou_thresholds)
+                                True when prediction (i) is matched with a crowd target with
+                                respect to the (j)th IoU threshold
+            preds_scores:       Tensor of shape (num_predictions),
+                                confidence score for every prediction
+            preds_cls:          Tensor of shape (num_predictions),
+                                predicted class for every prediction
+            targets_cls:        Tensor of shape (num_targets),
+                                ground truth class for every target box to be detected
+
+        Returns:
+            ap, precision, recall, f1:  Tensors of shape (n_class, nb_iou_thrs)
+            unique_classes:             Vector with all unique target classes
+        """
+
+        preds_matched, preds_to_ignore = (
+            preds_matched.to(self.device),
+            preds_to_ignore.to(self.device),
+        )
+        preds_scores, preds_cls, targets_cls = (
+            preds_scores.to(self.device),
+            preds_cls.to(self.device),
+            targets_cls.to(self.device),
+        )
+
+        recall_thresholds = self.recall_thresholds.to(self.device)
+        score_threshold = self.score_threshold
+
+        unique_classes = torch.unique(targets_cls).long()
+
+        n_class, nb_iou_thrs = len(unique_classes), preds_matched.shape[-1]
+
+        ap = torch.zeros((n_class, nb_iou_thrs), device=self.device)
+        precision = torch.zeros((n_class, nb_iou_thrs), device=self.device)
+        recall = torch.zeros((n_class, nb_iou_thrs), device=self.device)
+
+        for cls_i, class_value in enumerate(unique_classes):
+            cls_preds_idx, cls_targets_idx = (
+                (preds_cls == class_value),
+                (targets_cls == class_value),
+            )
+            (
+                cls_ap,
+                cls_precision,
+                cls_recall,
+            ) = self._compute_detection_metrics_per_cls(
+                preds_matched=preds_matched[cls_preds_idx],
+                preds_to_ignore=preds_to_ignore[cls_preds_idx],
+                preds_scores=preds_scores[cls_preds_idx],
+                n_targets=cls_targets_idx.sum(),
+                recall_thresholds=recall_thresholds,
+                score_threshold=score_threshold,
+            )
+            ap[cls_i, :] = cls_ap
+            precision[cls_i, :] = cls_precision
+            recall[cls_i, :] = cls_recall
+
+        f1 = 2 * precision * recall / (precision + recall + 1e-16)
+        return ap, precision, recall, f1, unique_classes
+
+    def _compute_detection_metrics_per_cls(
+        self,
+        preds_matched: torch.Tensor,
+        preds_to_ignore: torch.Tensor,
+        preds_scores: torch.Tensor,
+        n_targets: int,
+        recall_thresholds: torch.Tensor,
+        score_threshold: float,
+    ):
+        # Adapted from: https://github.com/Deci-AI/super-gradients/blob/master/src/super_gradients/training/utils/detection_utils.py  # noqa E501
+        """
+        Compute the list of precision, recall and MaP of a given class for every recall threshold.
+
+        Args:
+            preds_matched:      Tensor of shape (num_predictions, n_thresholds)
+                                True when prediction (i) is matched with a target
+                                with respect to the(j)th threshold
+            preds_to_ignore     Tensor of shape (num_predictions, n_thresholds)
+                                True when prediction (i) is matched with a crowd target
+                                with respect to the (j)th threshold
+            preds_scores:       Tensor of shape (num_predictions),
+                                confidence score for every prediction
+            n_targets:          Number of target boxes of this class
+            recall_thresholds:  Tensor of shape (max_n_rec_thresh)
+                                list of recall thresholds used to compute MaP
+            score_threshold:    Minimum confidence score to consider a prediction
+                                for the computation of precision and recall (not MaP)
+
+        Returns:
+            ap, precision, recall:     Tensors of shape (nb_thrs)
+        """
+
+        nb_iou_thrs = preds_matched.shape[-1]
+
+        tps = preds_matched
+        fps = torch.logical_and(
+            torch.logical_not(preds_matched), torch.logical_not(preds_to_ignore)
+        )
+
+        if len(tps) == 0:
+            return (
+                torch.zeros(nb_iou_thrs, device=self.device),
+                torch.zeros(nb_iou_thrs, device=self.device),
+                torch.zeros(nb_iou_thrs, device=self.device),
+            )
+
+        # Sort by decreasing score
+        dtype = (
+            torch.uint8
+            if preds_scores.is_cuda and preds_scores.dtype is torch.bool
+            else preds_scores.dtype
+        )
+        sort_ind = torch.argsort(preds_scores.to(dtype), descending=True)
+        tps = tps[sort_ind, :]
+        fps = fps[sort_ind, :]
+        preds_scores = preds_scores[sort_ind].contiguous()
+
+        # Rolling sum over the predictions
+        rolling_tps = torch.cumsum(tps, axis=0, dtype=torch.float)
+        rolling_fps = torch.cumsum(fps, axis=0, dtype=torch.float)
+
+        rolling_recalls = rolling_tps / n_targets
+        rolling_precisions = rolling_tps / (
+            rolling_tps + rolling_fps + torch.finfo(torch.float64).eps
+        )
+
+        # Reversed cummax to only have decreasing values
+        rolling_precisions = rolling_precisions.flip(0).cummax(0).values.flip(0)
+
+        # ==================
+        # RECALL & PRECISION
+
+        # We want the rolling precision/recall at index i so that:
+        # preds_scores[i-1] >= score_threshold > preds_scores[i]
+        # Note: torch.searchsorted works on increasing sequence and preds_scores is decreasing,
+        # so we work with "-"
+        # Note2: right=True due to negation
+        lowest_score_above_threshold = torch.searchsorted(
+            -preds_scores, -score_threshold, right=True
+        )
+
+        if (
+            lowest_score_above_threshold == 0
+        ):  # Here score_threshold > preds_scores[0], so no pred is above the threshold
+            recall = torch.zeros(nb_iou_thrs, device=self.device)
+            precision = torch.zeros(
+                nb_iou_thrs, device=self.device
+            )  # the precision is not really defined when no pred but we need to give it a value
+        else:
+            recall = rolling_recalls[lowest_score_above_threshold - 1]
+            precision = rolling_precisions[lowest_score_above_threshold - 1]
+
+        # ==================
+        # AVERAGE PRECISION
+
+        # shape = (nb_iou_thrs, n_recall_thresholds)
+        recall_thresholds = recall_thresholds.view(1, -1).repeat(nb_iou_thrs, 1)
+
+        # We want the index i so that:
+        # rolling_recalls[i-1] < recall_thresholds[k] <= rolling_recalls[i]
+        # Note:  when recall_thresholds[k] > max(rolling_recalls), i = len(rolling_recalls)
+        # Note2: we work with transpose (.T) to apply torch.searchsorted on first dim
+        # instead of the last one
+        recall_threshold_idx = torch.searchsorted(
+            rolling_recalls.T.contiguous(), recall_thresholds, right=False
+        ).T
+
+        # When recall_thresholds[k] > max(rolling_recalls),
+        # rolling_precisions[i] is not defined, and we want precision = 0
+        rolling_precisions = torch.cat(
+            (rolling_precisions, torch.zeros(1, nb_iou_thrs, device=self.device)), dim=0
+        )
+
+        # shape = (n_recall_thresholds, nb_iou_thrs)
+        sampled_precision_points = torch.gather(
+            input=rolling_precisions, index=recall_threshold_idx, dim=0
+        )
+
+        # Average over the recall_thresholds
+        ap = sampled_precision_points.mean(0)
+
+        return ap, precision, recall
+
+
+if __name__ == "__main__":
+    from dataclasses import asdict
+
+    # Example usage
+    prediction_file_paths = [Path("pths/to/predictions.json"), Path("pths/to/predictions2.json")]
+    ground_truth_file_paths = [
+        Path("pths/to/ground_truth.json"),
+        Path("pths/to/ground_truth2.json"),
+    ]
+
+    for prediction_file_path, ground_truth_file_path in zip(
+        prediction_file_paths, ground_truth_file_paths
+    ):
+        eval_processor = ObjectDetectionEvalProcessor.from_json_files(
+            prediction_file_path, ground_truth_file_path
+        )
+
+        metrics, per_class_metrics = eval_processor.get_metrics()
+        print(f"Metrics for {ground_truth_file_path.name}:\n{asdict(metrics)}")
+        print(f"Per class Metrics for {ground_truth_file_path.name}:\n{asdict(per_class_metrics)}")

unstructured/metrics/table/table_alignment.py

@@ -0,0 +1,180 @@
+import difflib
+from typing import Any, Dict, List
+
+import numpy as np
+import pandas as pd
+from unstructured_inference.models.eval import compare_contents_as_df
+
+
+class TableAlignment:
+    def __init__(self, cutoff: float = 0.8):
+        self.cutoff = cutoff
+
+    @staticmethod
+    def get_content_in_tables(table_data: List[List[Dict[str, Any]]]) -> List[str]:
+        # Replace below docstring with google-style docstring
+        """Extracts and concatenates the content of cells from each table in a list of tables.
+
+        Args:
+          table_data: A list of tables, each table being a list of cell data dictionaries.
+
+        Returns:
+          List of strings where each string represents the concatenated content of one table.
+        """
+        return [" ".join([d["content"] for d in td if "content" in d]) for td in table_data]
+
+    @staticmethod
+    def get_table_level_alignment(
+        predicted_table_data: List[List[Dict[str, Any]]],
+        ground_truth_table_data: List[List[Dict[str, Any]]],
+    ) -> List[int]:
+        """Compares predicted table data with ground truth data to find the best
+        matching table index for each predicted table.
+
+        Args:
+          predicted_table_data: A list of predicted tables.
+          ground_truth_table_data: A list of ground truth tables.
+
+        Returns:
+          A list of indices indicating the best match in the ground truth for
+          each predicted table.
+
+        """
+        ground_truth_texts = TableAlignment.get_content_in_tables(ground_truth_table_data)
+        matched_indices = []
+        for td in predicted_table_data:
+            reference = TableAlignment.get_content_in_tables([td])[0]
+            matches = difflib.get_close_matches(reference, ground_truth_texts, cutoff=0.1, n=1)
+            matched_indices.append(ground_truth_texts.index(matches[0]) if matches else -1)
+        return matched_indices
+
+    @staticmethod
+    def _zip_to_dataframe(table_data: List[Dict[str, Any]]) -> pd.DataFrame:
+        df = pd.DataFrame(table_data, columns=["row_index", "col_index", "content"])
+        df = df.set_index("row_index")
+        df["col_index"] = df["col_index"].astype(str)
+        return df
+
+    @staticmethod
+    def get_element_level_alignment(
+        predicted_table_data: List[List[Dict[str, Any]]],
+        ground_truth_table_data: List[List[Dict[str, Any]]],
+        matched_indices: List[int],
+        cutoff: float = 0.8,
+    ) -> Dict[str, float]:
+        """Aligns elements of the predicted tables with the ground truth tables at the cell level.
+
+        Args:
+          predicted_table_data: A list of predicted tables.
+          ground_truth_table_data: A list of ground truth tables.
+          matched_indices: Indices of the best matching ground truth table for each predicted table.
+          cutoff: The cutoff value for the close matches.
+
+        Returns:
+          A dictionary with column and row alignment accuracies.
+
+        """
+        content_diff_cols = []
+        content_diff_rows = []
+        col_index_acc = []
+        row_index_acc = []
+
+        for idx, td in zip(matched_indices, predicted_table_data):
+            if idx == -1:
+                content_diff_cols.append(0)
+                content_diff_rows.append(0)
+                col_index_acc.append(0)
+                row_index_acc.append(0)
+                continue
+            ground_truth_td = ground_truth_table_data[idx]
+
+            # Get row and col content accuracy
+            predict_table_df = TableAlignment._zip_to_dataframe(td)
+            ground_truth_table_df = TableAlignment._zip_to_dataframe(ground_truth_td)
+
+            table_content_diff = compare_contents_as_df(
+                ground_truth_table_df.fillna(""),
+                predict_table_df.fillna(""),
+            )
+            content_diff_cols.append(table_content_diff["by_col_token_ratio"])
+            content_diff_rows.append(table_content_diff["by_row_token_ratio"])
+
+            aligned_element_col_count = 0
+            aligned_element_row_count = 0
+            total_element_count = 0
+            # Get row and col index accuracy
+            ground_truth_td_contents_list = [gtd["content"].lower() for gtd in ground_truth_td]
+            used_indices = set()
+            indices_tuple_pairs = []
+            for td_ele in td:
+                content = td_ele["content"].lower()
+                row_index = td_ele["row_index"]
+                col_idx = td_ele["col_index"]
+
+                matches = difflib.get_close_matches(
+                    content,
+                    ground_truth_td_contents_list,
+                    cutoff=cutoff,
+                    n=1,
+                )
+                # BUG FIX: the previous matched_idx will only output the first matched index if
+                # the match has duplicates in the
+                # ground_truth_td_contents_list, the current fix will output its correspondence idx
+                # once matching is exhausted, it will go back search again the same fashion
+                matching_indices = []
+                if matches != []:
+                    b_indices = [
+                        i
+                        for i, b_string in enumerate(ground_truth_td_contents_list)
+                        if b_string == matches[0] and i not in used_indices
+                    ]
+                    if not b_indices:
+                        # If all indices are used, reset used_indices and use the first index
+                        used_indices.clear()
+                        b_indices = [
+                            i
+                            for i, b_string in enumerate(ground_truth_td_contents_list)
+                            if b_string == matches[0] and i not in used_indices
+                        ]
+                    matching_index = b_indices[0]
+                    matching_indices.append(matching_index)
+                    used_indices.add(matching_index)
+                else:
+                    matching_indices = [-1]
+                matched_idx = matching_indices[0]
+                if matched_idx >= 0:
+                    gt_row_index = ground_truth_td[matched_idx]["row_index"]
+                    gt_col_index = ground_truth_td[matched_idx]["col_index"]
+                    indices_tuple_pairs.append(((row_index, col_idx), (gt_row_index, gt_col_index)))
+
+            for indices_tuple_pair in indices_tuple_pairs:
+                if indices_tuple_pair[0][0] == indices_tuple_pair[1][0]:
+                    aligned_element_row_count += 1
+                if indices_tuple_pair[0][1] == indices_tuple_pair[1][1]:
+                    aligned_element_col_count += 1
+                total_element_count += 1
+
+            table_col_index_acc = 0
+            table_row_index_acc = 0
+            if total_element_count > 0:
+                table_col_index_acc = round(aligned_element_col_count / total_element_count, 2)
+                table_row_index_acc = round(aligned_element_row_count / total_element_count, 2)
+
+            col_index_acc.append(table_col_index_acc)
+            row_index_acc.append(table_row_index_acc)
+
+        not_found_gt_table_indexes = [
+            id for id in range(len(ground_truth_table_data)) if id not in matched_indices
+        ]
+        for _ in not_found_gt_table_indexes:
+            content_diff_cols.append(0)
+            content_diff_rows.append(0)
+            col_index_acc.append(0)
+            row_index_acc.append(0)
+
+        return {
+            "col_index_acc": round(np.mean(col_index_acc), 2),
+            "row_index_acc": round(np.mean(row_index_acc), 2),
+            "col_content_acc": round(np.mean(content_diff_cols) / 100.0, 2),
+            "row_content_acc": round(np.mean(content_diff_rows) / 100.0, 2),
+        }

unstructured/metrics/table/table_eval.py

@@ -0,0 +1,342 @@
+"""
+The purpose of this script is to create a comprehensive metric for table evaluation
+1. Verify table identification.
+    a. Concatenate all text in the table and ground truth.
+    b. Calculate the difference to find the closest matches.
+    c. If contents are too different, mark as a failure.
+
+2. For each identified table:
+    a. Align elements at the level of individual elements.
+    b. Match elements by text.
+    c. Determine indexes for both predicted and actual data.
+    d. Compare index tuples at column and row levels to assess content shifts.
+    e. Compare the token orders by flattened along column and row levels
+    f. Note: Imperfect HTML is acceptable unless it impedes parsing,
+       in which case the table is considered failed.
+
+Example
+python table_eval.py  \
+    --prediction_file "model_output.pdf.json" \
+    --ground_truth_file "ground_truth.pdf.json"
+"""
+
+import difflib
+import json
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+import click
+import numpy as np
+
+from unstructured.metrics.table.table_alignment import TableAlignment
+from unstructured.metrics.table.table_extraction import (
+    extract_and_convert_tables_from_ground_truth,
+    extract_and_convert_tables_from_prediction,
+)
+
+
+@dataclass
+class TableEvaluation:
+    """Class representing a gathered table metrics."""
+
+    total_tables: int
+    total_predicted_tables: int
+    table_level_acc: float
+    table_detection_recall: float
+    table_detection_precision: float
+    table_detection_f1: float
+    element_col_level_index_acc: float
+    element_row_level_index_acc: float
+    element_col_level_content_acc: float
+    element_row_level_content_acc: float
+
+    @property
+    def composite_structure_acc(self) -> float:
+        return (
+            self.element_col_level_index_acc
+            + self.element_row_level_index_acc
+            + (self.element_col_level_content_acc + self.element_row_level_content_acc) / 2
+        ) / 3
+
+
+def table_level_acc(predicted_table_data, ground_truth_table_data, matched_indices):
+    """computes for each predicted table its accurary compared to ground truth.
+
+    The accuracy is defined as the SequenceMatcher.ratio() between those two strings. If a
+    prediction does not have a matched ground truth its accuracy is 0
+    """
+    score = np.zeros((len(matched_indices),))
+    ground_truth_text = TableAlignment.get_content_in_tables(ground_truth_table_data)
+    for idx, predicted in enumerate(predicted_table_data):
+        matched_idx = matched_indices[idx]
+        if matched_idx == -1:
+            # false positive; default score 0
+            continue
+        score[idx] = difflib.SequenceMatcher(
+            None,
+            TableAlignment.get_content_in_tables([predicted])[0],
+            ground_truth_text[matched_idx],
+        ).ratio()
+    return score
+
+
+def _count_predicted_tables(matched_indices: List[int]) -> int:
+    """Counts the number of predicted tables that have a corresponding match in the ground truth.
+
+    Args:
+      matched_indices: List of indices indicating matches between predicted
+        and ground truth tables.
+
+    Returns:
+      The count of matched predicted tables.
+
+    """
+    return sum(1 for idx in matched_indices if idx >= 0)
+
+
+def calculate_table_detection_metrics(
+    matched_indices: list[int], ground_truth_tables_number: int
+) -> tuple[float, float, float]:
+    """
+    Calculate the table detection metrics: recall, precision, and f1 score.
+    Args:
+        matched_indices:
+            List of indices indicating matches between predicted and ground truth tables
+            For example: matched_indices[i] = j means that the
+            i-th predicted table is matched with the j-th ground truth table.
+        ground_truth_tables_number: the number of ground truth tables.
+
+    Returns:
+        Tuple of recall, precision, and f1 scores
+    """
+    predicted_tables_number = len(matched_indices)
+
+    matched_set = set(matched_indices)
+    if -1 in matched_set:
+        matched_set.remove(-1)
+
+    true_positive = len(matched_set)
+    false_positive = predicted_tables_number - true_positive
+    positive = ground_truth_tables_number
+
+    recall = true_positive / positive if positive > 0 else 0
+    precision = (
+        true_positive / (true_positive + false_positive)
+        if true_positive + false_positive > 0
+        else 0
+    )
+    f1 = 2 * precision * recall / (precision + recall) if precision + recall > 0 else 0
+
+    return recall, precision, f1
+
+
+class TableEvalProcessor:
+    def __init__(
+        self,
+        prediction: List[Dict[str, Any]],
+        ground_truth: List[Dict[str, Any]],
+        cutoff: float = 0.8,
+        source_type: str = "html",
+    ):
+        """
+        Initializes the TableEvalProcessor prediction and ground truth.
+
+        Args:
+            ground_truth: Ground truth table data. The tables text should be in the deckerd format.
+            prediction: Predicted table data.
+            cutoff: The cutoff value for the element level alignment. Default is 0.8.
+
+        Examples:
+            ground_truth: [
+                {
+                    "type": "Table",
+                    "text": [
+                        {
+                            "id": "f4c35dae-105b-46f5-a77a-7fbc199d6aca",
+                            "x": 0,
+                            "y": 0,
+                            "w": 1,
+                            "h": 1,
+                            "content": "Cell text"
+                        },
+                        ...
+                }
+            ]
+            prediction: [
+                {
+                    "element_id": <id_string>,
+                    ...
+                    "metadata": {
+                        ...
+                        "text_as_html": "<table><thead><tr><th rowspan=\"2\">June....
+                                                                </tr></td></table>",
+                        "table_as_cells":
+                        [
+                            {
+                                "x": 0,
+                                "y": 0,
+                                "w": 1,
+                                "h": 2,
+                                "content": "June"
+                            },
+                            ...
+                        ]
+                    }
+                },
+            ]
+
+        """
+        self.prediction = prediction
+        self.ground_truth = ground_truth
+        self.cutoff = cutoff
+        self.source_type = source_type
+
+    @classmethod
+    def from_json_files(
+        cls,
+        prediction_file: Path,
+        ground_truth_file: Path,
+        cutoff: Optional[float] = None,
+        source_type: str = "html",
+    ) -> "TableEvalProcessor":
+        """Factory classmethod to initialize the object with path to json files instead of dicts
+
+        Args:
+          prediction_file: Path to the json file containing the predicted table data.
+          ground_truth_file: Path to the json file containing the ground truth table data.
+          source_type: 'cells' or 'html'. 'cells' refers to reading 'table_as_cells' field while
+            'html' is extracted from 'text_as_html'
+          cutoff: The cutoff value for the element level alignment.
+            If not set, class default value is used (=0.8).
+
+        Returns:
+          TableEvalProcessor: An instance of the class initialized with the provided data.
+        """
+        with open(prediction_file) as f:
+            prediction = json.load(f)
+        with open(ground_truth_file) as f:
+            ground_truth = json.load(f)
+        if cutoff is not None:
+            return cls(
+                prediction=prediction,
+                ground_truth=ground_truth,
+                cutoff=cutoff,
+                source_type=source_type,
+            )
+        else:
+            return cls(prediction=prediction, ground_truth=ground_truth, source_type=source_type)
+
+    def process_file(self) -> TableEvaluation:
+        """Processes the files and computes table-level and element-level accuracy.
+
+        Returns:
+            TableEvaluation: A dataclass object containing the computed metrics.
+        """
+        ground_truth_table_data = extract_and_convert_tables_from_ground_truth(
+            self.ground_truth,
+        )
+
+        predicted_table_data = extract_and_convert_tables_from_prediction(
+            file_elements=self.prediction, source_type=self.source_type
+        )
+        is_table_in_gt = bool(ground_truth_table_data)
+        is_table_predicted = bool(predicted_table_data)
+        if not is_table_in_gt:
+            # There is no table data in ground truth, you either got perfect score or 0
+            score = 0 if is_table_predicted else np.nan
+            table_acc = 1 if not is_table_predicted else 0
+            return TableEvaluation(
+                total_tables=0,
+                total_predicted_tables=len(predicted_table_data),
+                table_level_acc=table_acc,
+                table_detection_recall=score,
+                table_detection_precision=score,
+                table_detection_f1=score,
+                element_col_level_index_acc=score,
+                element_row_level_index_acc=score,
+                element_col_level_content_acc=score,
+                element_row_level_content_acc=score,
+            )
+        if is_table_in_gt and not is_table_predicted:
+            return TableEvaluation(
+                total_tables=len(ground_truth_table_data),
+                total_predicted_tables=0,
+                table_level_acc=0,
+                table_detection_recall=0,
+                table_detection_precision=0,
+                table_detection_f1=0,
+                element_col_level_index_acc=0,
+                element_row_level_index_acc=0,
+                element_col_level_content_acc=0,
+                element_row_level_content_acc=0,
+            )
+        else:
+            # We have both ground truth tables and predicted tables
+            matched_indices = TableAlignment.get_table_level_alignment(
+                predicted_table_data,
+                ground_truth_table_data,
+            )
+            predicted_table_acc = np.mean(
+                table_level_acc(predicted_table_data, ground_truth_table_data, matched_indices)
+            )
+
+            metrics = TableAlignment.get_element_level_alignment(
+                predicted_table_data,
+                ground_truth_table_data,
+                matched_indices,
+                cutoff=self.cutoff,
+            )
+
+            (
+                table_detection_recall,
+                table_detection_precision,
+                table_detection_f1,
+            ) = calculate_table_detection_metrics(
+                matched_indices=matched_indices,
+                ground_truth_tables_number=len(ground_truth_table_data),
+            )
+
+            evaluation = TableEvaluation(
+                total_tables=len(ground_truth_table_data),
+                total_predicted_tables=len(predicted_table_data),
+                table_level_acc=predicted_table_acc,
+                table_detection_recall=table_detection_recall,
+                table_detection_precision=table_detection_precision,
+                table_detection_f1=table_detection_f1,
+                element_col_level_index_acc=metrics.get("col_index_acc", 0),
+                element_row_level_index_acc=metrics.get("row_index_acc", 0),
+                element_col_level_content_acc=metrics.get("col_content_acc", 0),
+                element_row_level_content_acc=metrics.get("row_content_acc", 0),
+            )
+            return evaluation
+
+
+@click.command()
+@click.option(
+    "--prediction_file", help="Path to the model prediction JSON file", type=click.Path(exists=True)
+)
+@click.option(
+    "--ground_truth_file", help="Path to the ground truth JSON file", type=click.Path(exists=True)
+)
+@click.option(
+    "--cutoff",
+    type=float,
+    show_default=True,
+    default=0.8,
+    help="The cutoff value for the element level alignment. \
+        If not set, a default value is used",
+)
+def run(prediction_file: str, ground_truth_file: str, cutoff: Optional[float]):
+    """Runs the table evaluation process and prints the computed metrics."""
+    processor = TableEvalProcessor.from_json_files(
+        Path(prediction_file),
+        Path(ground_truth_file),
+        cutoff=cutoff,
+    )
+    report = processor.process_file()
+    print(report)
+
+
+if __name__ == "__main__":
+    run()

unstructured/metrics/table/table_extraction.py

@@ -0,0 +1,288 @@
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+from bs4 import BeautifulSoup
+from unstructured_inference.models.tables import cells_to_html
+
+EMPTY_CELL = {
+    "row_index": "",
+    "col_index": "",
+    "content": "",
+}
+
+
+def _move_cells_for_spanned_cells(cells: List[Dict[str, Any]]):
+    """Move cells to the right if spanned cells have an influence on the rendering.
+
+    Args:
+        cells: List of cells in the table in Deckerd format.
+
+    Returns:
+        List of cells in the table in Deckerd format with cells moved to the right if spanned.
+    """
+    sorted_cells = sorted(cells, key=lambda x: (x["y"], x["x"]))
+    cells_occupied_by_spanned = set()
+    for cell in sorted_cells:
+        if cell["w"] > 1 or cell["h"] > 1:
+            for i in range(cell["y"], cell["y"] + cell["h"]):
+                for j in range(cell["x"], cell["x"] + cell["w"]):
+                    if (i, j) != (cell["y"], cell["x"]):
+                        cells_occupied_by_spanned.add((i, j))
+        while (cell["y"], cell["x"]) in cells_occupied_by_spanned:
+            cell_y, cell_x = cell["y"], cell["x"]
+            cells_to_the_right = [c for c in sorted_cells if c["y"] == cell_y and c["x"] >= cell_x]
+            for cell_to_move in cells_to_the_right:
+                cell_to_move["x"] += 1
+            cells_occupied_by_spanned.remove((cell_y, cell_x))
+    return sorted_cells
+
+
+def html_table_to_deckerd(content: str) -> List[Dict[str, Any]]:
+    """Convert html format to Deckerd table structure.
+
+    Args:
+        content: The html content with a table to extract.
+
+    Returns:
+        A list of dictionaries where each dictionary represents a cell in the table.
+    """
+
+    soup = BeautifulSoup(content, "html.parser")
+    table = soup.find("table")
+    rows = table.find_all(["tr"])
+    table_data = []
+
+    for i, row in enumerate(rows):
+        cells = row.find_all(["th", "td"])
+        for j, cell_data in enumerate(cells):
+            cell = {
+                "y": i,
+                "x": j,
+                "w": int(cell_data.attrs.get("colspan", 1)),
+                "h": int(cell_data.attrs.get("rowspan", 1)),
+                "content": cell_data.text,
+            }
+            table_data.append(cell)
+    return _move_cells_for_spanned_cells(table_data)
+
+
+def deckerd_table_to_html(cells: List[Dict[str, Any]]) -> str:
+    """Convert Deckerd table structure to html format.
+
+    Args:
+        cells: List of dictionaries where each dictionary represents a cell in the table.
+
+    Returns:
+        A string with the html content of the table.
+    """
+    transformer_cells = []
+    # determine which cells are in header. Consider row 0 as header
+    # but spans may make it larger
+    first_row_cells = [cell for cell in cells if cell["y"] == 0]
+    header_length = max(cell["w"] for cell in first_row_cells)
+    header_rows = set(range(header_length))
+    for cell in cells:
+        cell_data = {
+            "row_nums": list(range(cell["y"], cell["y"] + cell["h"])),
+            "column_nums": list(range(cell["x"], cell["x"] + cell["w"])),
+            "w": cell["w"],
+            "h": cell["h"],
+            "cell text": cell["content"],
+            "column header": cell["y"] in header_rows,
+        }
+        transformer_cells.append(cell_data)
+    # reuse the existing function to convert to HTML
+    table = cells_to_html(transformer_cells)
+    return table
+
+
+def _convert_table_from_html(content: str) -> List[Dict[str, Any]]:
+    """Convert html format to table structure. As a middle step it converts
+    html to the Deckerd format as it's more convenient to work with.
+
+    Args:
+        content: The html content with a table to extract.
+
+    Returns:
+        A list of dictionaries where each dictionary represents a cell in the table.
+    """
+    deckerd_cells = html_table_to_deckerd(content)
+    return _convert_table_from_deckerd(deckerd_cells)
+
+
+def _convert_table_from_deckerd(content: List[Dict[str, Any]]) -> List[Dict[str, Any]]:
+    """Convert deckerd format to table structure.
+
+    Args:
+      content: The deckerd formatted content with a table to extract.
+
+    Returns:
+      A list of dictionaries where each dictionary represents a cell in the table.
+    """
+    table_data = []
+    for table in content:
+        try:
+            cell_data = {
+                "row_index": table["y"],
+                "col_index": table["x"],
+                "content": table["content"],
+            }
+        except KeyError:
+            cell_data = EMPTY_CELL
+        except TypeError:
+            cell_data = EMPTY_CELL
+        table_data.append(cell_data)
+    return table_data
+
+
+def _sort_table_cells(table_data: List[List[Dict[str, Any]]]) -> List[List[Dict[str, Any]]]:
+    return sorted(table_data, key=lambda cell: (cell["row_index"], cell["col_index"]))
+
+
+def extract_and_convert_tables_from_ground_truth(
+    file_elements: List[Dict[str, Any]],
+) -> List[List[Dict[str, Any]]]:
+    """Extracts and converts tables data to a structured format based on the specified table type.
+
+    Args:
+        file_elements: List of elements from the ground truth file.
+
+    Returns:
+        A list of tables with each table represented as a list of cell data dictionaries.
+
+    """
+    ground_truth_table_data = []
+    for element in file_elements:
+        if "type" in element and element["type"] == "Table" and "text" in element:
+            try:
+                converted_data = _convert_table_from_deckerd(
+                    element["text"],
+                )
+                ground_truth_table_data.append(_sort_table_cells(converted_data))
+            except Exception as e:
+                print(f"Error converting ground truth data: {e}")
+                ground_truth_table_data.append({})
+
+    return ground_truth_table_data
+
+
+def extract_and_convert_tables_from_prediction(
+    file_elements: List[Dict[str, Any]], source_type: str = "html"
+) -> List[List[Dict[str, Any]]]:
+    """Extracts and converts table data to a structured format
+
+    Args:
+      file_elements: List of elements from the file.
+      source_type: 'cells' or 'html'. 'cells' refers to reading 'table_as_cells' field while
+        'html' is extracted from 'text_as_html'
+
+    Returns:
+      A list of tables with each table represented as a list of cell data dictionaries.
+
+    """
+    source_type_to_extraction_strategies = {
+        "html": extract_cells_from_text_as_html,
+        "cells": extract_cells_from_table_as_cells,
+    }
+    if source_type not in source_type_to_extraction_strategies:
+        raise ValueError(
+            f'source_type {source_type} is not valid. Allowed source_types are "html" and "cells"'
+        )
+
+    extract_cells_fn = source_type_to_extraction_strategies[source_type]
+    fallback_extract_cells_fn = (
+        extract_cells_from_table_as_cells
+        if source_type == "cells"
+        else extract_cells_from_text_as_html
+    )
+
+    predicted_table_data = []
+    for element in file_elements:
+        if element.get("type") == "Table":
+            extracted_cells = extract_cells_fn(element)
+            if not extracted_cells:
+                extracted_cells = fallback_extract_cells_fn(element)
+            if extracted_cells:
+                sorted_cells = _sort_table_cells(extracted_cells)
+                predicted_table_data.append(sorted_cells)
+
+    return predicted_table_data
+
+
+def extract_cells_from_text_as_html(element: Dict[str, Any]) -> List[Dict[str, Any]] | None:
+    """Extracts and parse cells from "text_as_html" field in Element structure
+
+    Args:
+        element: Example element:
+        {
+            "type": "Table",
+            "metadata": {
+                "text_as_html": "<table>
+                                    <thead>
+                                        <tr>
+                                            <th>Month A.</th>
+                                        </tr>
+                                    </thead>
+                                    </tbody>
+                                        <tr>
+                                            <td>22</td><
+                                        </tr>
+                                    </tbody>
+                                </table>"
+            }
+        }
+
+    Returns:
+        List of extracted cells in a format:
+        [
+            {
+                "row_index": 0,
+                "col_index": 0,
+                "content": "Month A.",
+            },
+            ...,
+        ]
+    """
+    val = element["metadata"].get("text_as_html")
+    if not val or "<table>" not in val:
+        return None
+
+    predicted_cells = None
+    try:
+        predicted_cells = _convert_table_from_html(val)
+    except Exception as e:
+        print(f"Error converting Unstructured table data: {e}")
+
+    return predicted_cells
+
+
+def extract_cells_from_table_as_cells(element: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """Extracts and parse cells from "table_as_cells" field in Element structure
+
+    Args:
+        element: Example element:
+        {
+            "type": "Table",
+            "metadata": {
+                "table_as_cells": [{"x": 0, "y": 0, "w": 1, "h": 1, "content": "Month A."},
+                                   {"x": 0, "y": 1, "w": 1, "h": 1, "content": "22"}]
+            }
+        }
+
+    Returns:
+        List of extracted cells in a format:
+        [
+            {
+                "row_index": 0,
+                "col_index": 0,
+                "content": "Month A.",
+            },
+            ...,
+        ]
+    """
+    predicted_cells = element["metadata"].get("table_as_cells")
+    converted_cells = None
+    if predicted_cells:
+        converted_cells = _convert_table_from_deckerd(predicted_cells)
+    return converted_cells

unstructured/metrics/table/table_formats.py

@@ -0,0 +1,49 @@
+from dataclasses import dataclass
+from typing import Union
+
+
+@dataclass
+class SimpleTableCell:
+    x: int
+    y: int
+    w: int
+    h: int
+    content: str = ""
+
+    def to_dict(self):
+        return {
+            "x": self.x,
+            "y": self.y,
+            "w": self.w,
+            "h": self.h,
+            "content": self.content,
+        }
+
+    @classmethod
+    def from_table_transformer_cell(cls, tatr_table_cell: dict[str, Union[list[int], str]]):
+        """
+        Args:
+            tatr_table_cell (dict):
+                Cell in a format returned by Table Transformer model, for example:
+                    {
+                        "row_nums": [1,2,3],
+                        "column_nums": [2],
+                        "cell text": "Text inside cell"
+                    }
+        """
+
+        row_nums = tatr_table_cell.get("row_nums", [])
+        column_nums = tatr_table_cell.get("column_nums", [])
+
+        if not row_nums:
+            raise ValueError(f'Cell {tatr_table_cell} has missing values under "row_nums" key')
+        if not column_nums:
+            raise ValueError(f'Cell {tatr_table_cell} has missing values under "column_nums" key')
+
+        return cls(
+            x=min(column_nums),
+            y=min(row_nums),
+            w=len(column_nums),
+            h=len(row_nums),
+            content=tatr_table_cell.get("cell text", ""),
+        )

unstructured/metrics/table_structure.py

@@ -0,0 +1,48 @@
+import numpy as np
+import pandas as pd
+from PIL import Image
+
+from unstructured.partition.pdf_image.ocr import get_table_tokens
+from unstructured.partition.pdf_image.pdf_image_utils import convert_pdf_to_images
+from unstructured.partition.utils.ocr_models.ocr_interface import OCRAgent
+from unstructured.utils import requires_dependencies
+
+
+@requires_dependencies("unstructured_inference")
+def image_or_pdf_to_dataframe(filename: str) -> pd.DataFrame:
+    """helper to JUST run table transformer on the input image/pdf file. It assumes the input is
+    JUST a table. This is intended to facilitate metric tracking on table structure detection ALONE
+    without mixing metric of element detection model"""
+    from unstructured_inference.models.tables import load_agent, tables_agent
+
+    load_agent()
+
+    if filename.endswith(".pdf"):
+        image = list(convert_pdf_to_images(filename))[0].convert("RGB")
+    else:
+        image = Image.open(filename).convert("RGB")
+
+    ocr_agent = OCRAgent.get_agent(language="eng")
+
+    return tables_agent.run_prediction(
+        image, ocr_tokens=get_table_tokens(image, ocr_agent), result_format="dataframe"
+    )
+
+
+@requires_dependencies("unstructured_inference")
+def eval_table_transformer_for_file(
+    filename: str,
+    true_table_filename: str,
+    eval_func: str = "token_ratio",
+) -> float:
+    """evaluate the predicted table structure vs. actual table structure by column and row as a
+    number between 0 and 1"""
+    from unstructured_inference.models.eval import compare_contents_as_df
+
+    pred_table = image_or_pdf_to_dataframe(filename).fillna("").replace(np.nan, "")
+    actual_table = pd.read_csv(true_table_filename).astype(str).fillna("").replace(np.nan, "")
+
+    results = np.array(
+        list(compare_contents_as_df(actual_table, pred_table, eval_func=eval_func).values()),
+    )
+    return results.mean() / 100.0

unstructured/metrics/text_extraction.py

@@ -0,0 +1,224 @@
+from typing import Dict, Optional, Tuple
+
+from rapidfuzz.distance import Levenshtein
+
+from unstructured.cleaners.core import clean_bullets, remove_sentence_punctuation
+
+_DOUBLE_QUOTE_CODEPOINTS = (
+    "\u0022",  # U+0022 Standard typewriter/programmer's quote
+    "\u201c",  # U+201C Left double quotation mark
+    "\u201d",  # U+201D Right double quotation mark
+    "\u201e",  # U+201E Double low-9 quotation mark
+    "\u201f",  # U+201F Double high-reversed-9 quotation mark
+    "\u00ab",  # U+00AB Left-pointing double angle quotation mark
+    "\u00bb",  # U+00BB Right-pointing double angle quotation mark
+    "\u275d",  # U+275D Heavy double turned comma quotation mark ornament
+    "\u275e",  # U+275E Heavy double comma quotation mark ornament
+    "\u2e42",  # U+2E42 Double low-reversed-9 quotation mark
+    "\U0001f676",  # U+1F676 SANS-SERIF HEAVY DOUBLE TURNED COMMA QUOTATION MARK ORNAMENT
+    "\U0001f677",  # U+1F677 SANS-SERIF HEAVY DOUBLE COMMA QUOTATION MARK ORNAMENT
+    "\U0001f678",  # U+1F678 SANS-SERIF HEAVY LOW DOUBLE COMMA QUOTATION MARK ORNAMENT
+    "\u2826",  # U+2826 Braille double closing quotation mark
+    "\u2834",  # U+2834 Braille double opening quotation mark
+    "\u301d",  # U+301D REVERSED DOUBLE PRIME QUOTATION MARK
+    "\u301e",  # U+301E DOUBLE PRIME QUOTATION MARK
+    "\u301f",  # U+301F LOW DOUBLE PRIME QUOTATION MARK
+    "\uff02",  # U+FF02 FULLWIDTH QUOTATION MARK
+)
+
+_SINGLE_QUOTE_CODEPOINTS = (
+    "\u0027",  # U+0027 Standard typewriter/programmer's quote
+    "\u2018",  # U+2018 Left single quotation mark
+    "\u2019",  # U+2019 Right single quotation mark
+    "\u201a",  # U+201A Single low-9 quotation mark
+    "\u201b",  # U+201B Single high-reversed-9 quotation mark
+    "\u2039",  # U+2039 Single left-pointing angle quotation mark
+    "\u203a",  # U+203A Single right-pointing angle quotation mark
+    "\u275b",  # U+275B Heavy single turned comma quotation mark ornament
+    "\u275c",  # U+275C Heavy single comma quotation mark ornament
+    "\u300c",  # U+300C Left corner bracket
+    "\u300d",  # U+300D Right corner bracket
+    "\u300e",  # U+300E Left white corner bracket
+    "\u300f",  # U+300F Right white corner bracket
+    "\ufe41",  # U+FE41 PRESENTATION FORM FOR VERTICAL LEFT CORNER BRACKET
+    "\ufe42",  # U+FE42 PRESENTATION FORM FOR VERTICAL RIGHT CORNER BRACKET
+    "\ufe43",  # U+FE43 PRESENTATION FORM FOR VERTICAL LEFT WHITE CORNER BRACKET
+    "\ufe44",  # U+FE44 PRESENTATION FORM FOR VERTICAL RIGHT WHITE CORNER BRACKET
+    "\uff07",  # U+FF07 FULLWIDTH APOSTROPHE
+    "\uff62",  # U+FF62 HALFWIDTH LEFT CORNER BRACKET
+    "\uff63",  # U+FF63 HALFWIDTH RIGHT CORNER BRACKET
+)
+
+_TRANSLATION_TABLE = str.maketrans(
+    dict.fromkeys(_DOUBLE_QUOTE_CODEPOINTS, '"') | dict.fromkeys(_SINGLE_QUOTE_CODEPOINTS, "'")
+)
+
+
+def calculate_accuracy(
+    output: Optional[str],
+    source: Optional[str],
+    weights: Tuple[int, int, int] = (2, 1, 1),
+) -> float:
+    """
+    Calculates accuracy by calling calculate_edit_distance function using `return_as=score`.
+    The function will return complement of the edit distance instead.
+    """
+    return calculate_edit_distance(output, source, weights, return_as="score")
+
+
+def calculate_edit_distance(
+    output: Optional[str],
+    source: Optional[str],
+    weights: Tuple[int, int, int] = (2, 1, 1),
+    return_as: str = "distance",
+    standardize_whitespaces: bool = True,
+) -> float:
+    """
+    Calculates edit distance using Levenshtein distance between two strings.
+
+    Args:
+        output (str): The target string to be compared.
+        source (str): The reference string against which 'output' is compared.
+        weights (Tuple[int, int, int], optional): A tuple containing weights
+            for insertion, deletion, and substitution operations in the edit
+            distance calculation. Default is (2, 1, 1).
+        return_as (str, optional): The type of result to return, one of
+            ["score", "distance"].
+            Default is "distance".
+
+    Returns:
+        float: The calculated edit distance or similarity score between
+            the 'output' and 'source' strings.
+
+    Raises:
+        ValueError: If 'return_as' is not one of the valid return types
+        ["score", "distance"].
+
+    Note:
+        This function calculates the edit distance (or similarity score) between
+        two strings using the Levenshtein distance algorithm. The 'weights' parameter
+        allows customizing the cost of insertion, deletion, and substitution
+        operations. The 'return_as' parameter determines the type of result to return:
+        - "score": Returns the similarity score, where 1.0 indicates a perfect match.
+        - "distance": Returns the raw edit distance value.
+
+    """
+    return_types = ["score", "distance"]
+    if return_as not in return_types:
+        raise ValueError("Invalid return value type. Expected one of: %s" % return_types)
+    output = standardize_quotes(prepare_str(output, standardize_whitespaces))
+    source = standardize_quotes(prepare_str(source, standardize_whitespaces))
+    distance = Levenshtein.distance(output, source, weights=weights)  # type: ignore
+    # lower bounded the char length for source string at 1.0 because to avoid division by zero
+    # in the case where source string is empty, the distance should be at 100%
+    source_char_len = max(len(source), 1.0)  # type: ignore
+    bounded_percentage_distance = min(max(distance / source_char_len, 0.0), 1.0)
+    if return_as == "score":
+        return 1 - bounded_percentage_distance
+    elif return_as == "distance":
+        return distance
+    return 0.0
+
+
+def bag_of_words(text: str) -> Dict[str, int]:
+    """
+    Outputs the bag of words (BOW) found in the input text and their frequencies.
+
+    Takes "clean, concatenated text" (CCT) from a document as input.
+
+    Removes sentence punctuation, but not punctuation within a word (ex. apostrophes).
+    """
+    bow: Dict[str, int] = {}
+    incorrect_word: str = ""
+    words = clean_bullets(remove_sentence_punctuation(text.lower(), ["-", "'"])).split()
+
+    i = 0
+    while i < len(words):
+        if len(words[i]) > 1:
+            if words[i] in bow:
+                bow[words[i]] += 1
+            else:
+                bow[words[i]] = 1
+            i += 1
+        else:
+            j = i
+            incorrect_word = ""
+
+            while j < len(words) and len(words[j]) == 1:
+                incorrect_word += words[j]
+                j += 1
+
+            if len(incorrect_word) == 1 and words[i].isalnum():
+                if incorrect_word in bow:
+                    bow[incorrect_word] += 1
+                else:
+                    bow[incorrect_word] = 1
+            i = j
+    return bow
+
+
+def calculate_percent_missing_text(
+    output: Optional[str],
+    source: Optional[str],
+) -> float:
+    """
+    Creates the bag of words (BOW) found in each input text and their frequencies, then compares the
+    output BOW against the source BOW to calculate the % of text from the source text missing from
+    the output text.
+
+    Takes "clean, concatenated text" (CCT) from a document output and the ground truth source text
+    as inputs.
+
+    If the output text contains all words from the source text and then some extra, result will be
+    0% missing text - this calculation does not penalize duplication.
+
+    A spaced-out word (ex. h e l l o) is considered missing; individual characters of a word
+    will not be counted as separate words.
+
+    Returns the percentage of missing text represented as a decimal between 0 and 1.
+    """
+    output = prepare_str(output)
+    source = prepare_str(source)
+    output_bow = bag_of_words(output)
+    source_bow = bag_of_words(source)
+
+    # get total words in source bow while counting missing words
+    total_source_word_count = 0
+    total_missing_word_count = 0
+
+    for source_word, source_count in source_bow.items():
+        total_source_word_count += source_count
+        if source_word not in output_bow:
+            # entire count is missing
+            total_missing_word_count += source_count
+        else:
+            output_count = output_bow[source_word]
+            total_missing_word_count += max(source_count - output_count, 0)
+
+    # calculate percent missing text
+    if total_source_word_count == 0:
+        return 0  # nothing missing because nothing in source document
+
+    fraction_missing = round(total_missing_word_count / total_source_word_count, 3)
+    return min(fraction_missing, 1)  # limit to 100%
+
+
+def prepare_str(string: Optional[str], standardize_whitespaces: bool = False) -> str:
+    if not string:
+        return ""
+    if standardize_whitespaces:
+        return " ".join(string.split())
+    return str(string)  # type: ignore
+
+
+def standardize_quotes(text: str) -> str:
+    """
+    Converts all unicode quotes to standard ASCII quotes with comprehensive coverage.
+
+    Args:
+        text (str): The input text to be standardized.
+
+    Returns:
+        str: The text with standardized quotes.
+    """
+    return text.translate(_TRANSLATION_TABLE)

unstructured/metrics/utils.py

@@ -0,0 +1,248 @@
+import logging
+import os
+import re
+import statistics
+from pathlib import Path
+from typing import List, Optional, Union
+
+import click
+import pandas as pd
+
+from unstructured.staging.base import elements_from_json, elements_to_text
+
+logger = logging.getLogger("unstructured.eval")
+
+
+def _prepare_output_cct(docpath: str, output_type: str) -> str:
+    """
+    Convert given input document (path) into cct-ready. The function only support conversion
+    from `json` or `txt` file.
+    """
+    try:
+        if output_type == "json":
+            output_cct = elements_to_text(elements_from_json(docpath))
+        elif output_type == "txt":
+            output_cct = _read_text_file(docpath)
+        else:
+            raise ValueError(
+                f"File type not supported. Expects one of `json` or `txt`, \
+                    but received {output_type} instead."
+            )
+    except ValueError as e:
+        logger.error(f"Could not read the file {docpath}")
+        raise e
+    return output_cct
+
+
+def _listdir_recursive(dir: str) -> List[str]:
+    """
+    Recursively lists all files in the given directory and its subdirectories.
+    Returns a list of all files found, with each file's path relative to the
+    initial directory.
+    """
+    listdir = []
+    for dirpath, _, filenames in os.walk(dir):
+        for filename in filenames:
+            # Remove the starting directory from the path to show the relative path
+            relative_path = os.path.relpath(dirpath, dir)
+            if relative_path == ".":
+                listdir.append(filename)
+            else:
+                listdir.append(os.path.join(relative_path, filename))
+    return listdir
+
+
+def _rename_aggregated_columns(df):
+    """
+    Renames aggregated columns in a DataFrame based on a predefined mapping.
+
+    Parameters:
+    df (pandas.DataFrame): The DataFrame with aggregated columns to rename.
+
+    Returns:
+    pandas.DataFrame: A new DataFrame with renamed aggregated columns.
+    """
+    rename_map = {"_mean": "mean", "_stdev": "stdev", "_pstdev": "pstdev", "_count": "count"}
+    return df.rename(columns=rename_map)
+
+
+def _format_grouping_output(*df):
+    """
+    Concatenates multiple pandas DataFrame objects along the columns (side-by-side)
+    and resets the index.
+    """
+    return pd.concat(df, axis=1).reset_index()
+
+
+def _display(df):
+    """
+    Displays the evaluation metrics in a formatted text table.
+    """
+    if len(df) == 0:
+        return
+    headers = df.columns.tolist()
+    col_widths = [
+        max(len(header), max(len(str(item)) for item in df[header])) for header in headers
+    ]
+    click.echo(" ".join(header.ljust(col_widths[i]) for i, header in enumerate(headers)))
+    click.echo("-" * sum(col_widths) + "-" * (len(headers) - 1))
+    for _, row in df.iterrows():
+        formatted_row = []
+        for item in row:
+            if isinstance(item, float):
+                formatted_row.append(f"{item:.3f}")
+            else:
+                formatted_row.append(str(item))
+        click.echo(
+            " ".join(formatted_row[i].ljust(col_widths[i]) for i in range(len(formatted_row))),
+        )
+
+
+def _write_to_file(
+    directory: str, filename: str, df: pd.DataFrame, mode: str = "w", overwrite: bool = True
+):
+    """
+    Save the metrics report to tsv file. The function allows an option 1) to choose `mode`
+    as `w` (write) or `a` (append) and 2) to `overwrite` the file if filename existed or not.
+    """
+    if mode not in ["w", "a"]:
+        raise ValueError("Mode not supported. Mode must be one of [w, a].")
+    if directory:
+        Path(directory).mkdir(exist_ok=True)
+    # Operate on a copy so callers can pass a view/slice without chained-assignment warnings.
+    df = df.copy()
+    if "count" in df.columns:
+        df["count"] = df["count"].astype(int)
+    if "filename" in df.columns and "connector" in df.columns:
+        df = df.sort_values(by=["connector", "filename"])
+    if not overwrite:
+        filename = _get_non_duplicated_filename(directory, filename)
+    df.to_csv(
+        os.path.join(directory, filename), sep="\t", mode=mode, index=False, header=(mode == "w")
+    )
+
+
+def _sorting_key(filename):
+    """
+    A function that defines the sorting method for duplicated file names. For example,
+    with filename.ext filename (1).ext filename (2).ext filename (10).ext - this function
+    extracts the integer in the bracket and sort those numbers ascendingly.
+    """
+    # Regular expression to find the number in the filename
+    numbers = re.findall(r"(\d+)", filename)
+    if numbers:
+        # If there's a number, return it as an integer for sorting
+        return int(numbers[-1])
+    else:
+        # If no number, return 0 so these files come first
+        return 0
+
+
+def _uniquity_file(file_list, target_filename) -> str:
+    """
+    Checks the duplicity of the file name from the list and run the numerical check
+    of the minimum number needed as extension to not overwrite the exising file.
+    Returns a string of file name in the format of `filename (<min number>).ext`.
+    """
+    original_filename, extension = target_filename.rsplit(".", 1)
+    pattern = rf"^{re.escape(original_filename)}(?: \((\d+)\))?\.{re.escape(extension)}$"
+    duplicated_files = sorted([f for f in file_list if re.match(pattern, f)], key=_sorting_key)
+
+    numbers = []
+    for file in duplicated_files:
+        match = re.search(r"\((\d+)\)", file)
+        if match:
+            numbers.append(int(match.group(1)))
+
+    numbers.sort()
+
+    counter = 1
+    for number in numbers:
+        if number == counter:
+            counter += 1
+        else:
+            break
+
+    return original_filename + " (" + str(counter) + ")." + extension
+
+
+def _get_non_duplicated_filename(dir, filename) -> str:
+    """
+    Helper function to calls the `_uniquity_file` function. Takes in directory and file name
+    to check on.
+    """
+    filename = _uniquity_file(os.listdir(dir), filename)
+    return filename
+
+
+def _mean(scores: Union[pd.Series, List[float]], rounding: Optional[int] = 3) -> Union[float, None]:
+    """
+    Find mean from the list. Returns None if no element in the list.
+
+    Args:
+        rounding (int): optional argument that allows user to define decimal points. Default at 3.
+    """
+    if len(scores) == 0:
+        return None
+    mean = statistics.mean(scores)
+    if not rounding:
+        return mean
+    return round(mean, rounding)
+
+
+def _stdev(scores: List[Optional[float]], rounding: Optional[int] = 3) -> Union[float, None]:
+    """
+    Find standard deviation from the list.
+    Returns None if only 0 or 1 element in the list.
+
+    Args:
+        rounding (int): optional argument that allows user to define decimal points. Default at 3.
+    """
+    # Filter out None values
+    scores = [score for score in scores if score is not None]
+    # Proceed only if there are more than one value
+    if len(scores) <= 1:
+        return None
+    if not rounding:
+        return statistics.stdev(scores)
+    return round(statistics.stdev(scores), rounding)
+
+
+def _pstdev(scores: List[Optional[float]], rounding: Optional[int] = 3) -> Union[float, None]:
+    """
+    Find population standard deviation from the list.
+    Returns None if only 0 or 1 element in the list.
+
+    Args:
+        rounding (int): optional argument that allows user to define decimal points. Default at 3.
+    """
+    scores = [score for score in scores if score is not None]
+    if len(scores) <= 1:
+        return None
+    if not rounding:
+        return statistics.pstdev(scores)
+    return round(statistics.pstdev(scores), rounding)
+
+
+def _count(scores: List[Optional[float]]) -> float:
+    """
+    Returns the row count of the list.
+    """
+    return len(scores)
+
+
+def _read_text_file(path):
+    """
+    Reads the contents of a text file and returns it as a string.
+    """
+    # Check if the file exists
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"The file at {path} does not exist.")
+
+    try:
+        with open(path, errors="ignore") as f:
+            text = f.read()
+        return text
+    except OSError as e:
+        # Handle other I/O related errors
+        raise IOError(f"An error occurred when reading the file at {path}: {e}")

unstructured/nlp/english_words.py

@@ -0,0 +1,17 @@
+import os
+import pathlib
+from typing import List, Set
+
+DIRECTORY = pathlib.Path(__file__).parent.resolve()
+# NOTE(robinson) - the list of English words is based on the nlkt.corpus.words corpus
+# and the list of English words found here at the link below. Add more words to the text
+# file if needed.
+# ref: https://github.com/jeremy-rifkin/Wordlist
+ENGLISH_WORDS_FILE = os.path.join(DIRECTORY, "english-words.txt")
+
+with open(ENGLISH_WORDS_FILE) as f:
+    BASE_ENGLISH_WORDS = f.read().split("\n")
+
+# NOTE(robinson) - add new words that we want to pass for the English check in here
+ADDITIONAL_ENGLISH_WORDS: List[str] = []
+ENGLISH_WORDS: Set[str] = set(BASE_ENGLISH_WORDS + ADDITIONAL_ENGLISH_WORDS)

unstructured/nlp/partition.py

@@ -0,0 +1,7 @@
+# flake8: noqa
+from unstructured.partition.pdf import partition_pdf  # noqa
+from unstructured.partition.text_type import (  # noqa
+    is_bulleted_text,
+    is_possible_narrative_text,
+    is_possible_title,
+)

unstructured/nlp/patterns.py

@@ -0,0 +1,143 @@
+import re
+from typing import Final, List
+
+# NOTE(robinson) - Modified from answers found on this stackoverflow post
+# ref: https://stackoverflow.com/questions/16699007/
+# regular-expression-to-match-standard-10-digit-phone-number
+US_PHONE_NUMBERS_PATTERN = (
+    r"(?:\+?(\d{1,3}))?[-. (]*(\d{3})?[-. )]*(\d{3})[-. ]*(\d{4})(?: *x(\d+))?\s*$"
+)
+US_PHONE_NUMBERS_RE = re.compile(US_PHONE_NUMBERS_PATTERN)
+
+# NOTE(robinson) - Based on this regex from regex101. Regex was updated to run fast
+# and avoid catastrophic backtracking
+# ref: https://regex101.com/library/oR3jU1?page=673
+US_CITY_STATE_ZIP_PATTERN = (
+    r"(?i)\b(?:[A-Z][a-z.-]{1,15}[ ]?){1,5},\s?"
+    r"(?:{Alabama|Alaska|Arizona|Arkansas|California|Colorado|Connecticut|Delaware|Florida"
+    r"|Georgia|Hawaii|Idaho|Illinois|Indiana|Iowa|Kansas|Kentucky|Louisiana|Maine|Maryland"
+    r"|Massachusetts|Michigan|Minnesota|Mississippi|Missouri|Montana|Nebraska|Nevada|"
+    r"New[ ]Hampshire|New[ ]Jersey|New[ ]Mexico|New[ ]York|North[ ]Carolina|North[ ]Dakota"
+    r"|Ohio|Oklahoma|Oregon|Pennsylvania|Rhode[ ]Island|South[ ]Carolina|South[ ]Dakota"
+    r"|Tennessee|Texas|Utah|Vermont|Virginia|Washington|West[ ]Virginia|Wisconsin|Wyoming}"
+    r"|{AL|AK|AS|AZ|AR|CA|CO|CT|DE|DC|FM|FL|GA|GU|HI|ID|IL|IN|IA|KS|KY|LA|ME|MH|MD|MA|MI|MN"
+    r"|MS|MO|MT|NE|NV|NH|NJ|NM|NY|NC|ND|MP|OH|OK|OR|PW|PA|PR|RI|SC|SD|TN|TX|UT|VT|VI|VA|"
+    r"WA|WV|WI|WY})(, |\s)?(?:\b\d{5}(?:-\d{4})?\b)"
+)
+US_CITY_STATE_ZIP_RE = re.compile(US_CITY_STATE_ZIP_PATTERN)
+
+UNICODE_BULLETS: Final[List[str]] = [
+    "\u0095",
+    "\u2022",
+    "\u2023",
+    "\u2043",
+    "\u3164",
+    "\u204c",
+    "\u204d",
+    "\u2219",
+    "\u25cb",
+    "\u25cf",
+    "\u25d8",
+    "\u25e6",
+    "\u2619",
+    "\u2765",
+    "\u2767",
+    "\u29be",
+    "\u29bf",
+    "\u002d",
+    "\u2013",
+    "",
+    r"\*",
+    "\x95",
+    "·",
+]
+BULLETS_PATTERN = "|".join(UNICODE_BULLETS)
+UNICODE_BULLETS_RE = re.compile(f"(?:{BULLETS_PATTERN})(?!{BULLETS_PATTERN})")
+# zero-width positive lookahead so bullet characters will not be removed when using .split()
+UNICODE_BULLETS_RE_0W = re.compile(f"(?={BULLETS_PATTERN})(?<!{BULLETS_PATTERN})")
+E_BULLET_PATTERN = re.compile(r"^e(?=\s)", re.MULTILINE)
+
+# NOTE(klaijan) - Captures reference of format [1] or [i] or [a] at any point in the line.
+REFERENCE_PATTERN = r"\[(?:[\d]+|[a-z]|[ivxlcdm])\]"
+REFERENCE_PATTERN_RE = re.compile(REFERENCE_PATTERN)
+
+ENUMERATED_BULLETS_RE = re.compile(r"(?:(?:\d{1,3}|[a-z][A-Z])\.?){1,3}")
+
+EMAIL_HEAD_PATTERN = (
+    r"(MIME-Version: 1.0(.*)?\n)?Date:.*\nMessage-ID:.*\nSubject:.*\nFrom:.*\nTo:.*"
+)
+EMAIL_HEAD_RE = re.compile(EMAIL_HEAD_PATTERN)
+
+# Helps split text by paragraphs. There must be one newline, with potential whitespace
+# (incluing \r and \n chars) on either side
+PARAGRAPH_PATTERN = r"\s*\n\s*"
+
+PARAGRAPH_PATTERN_RE = re.compile(
+    f"((?:{BULLETS_PATTERN})|{PARAGRAPH_PATTERN})(?!{BULLETS_PATTERN}|$)",
+)
+DOUBLE_PARAGRAPH_PATTERN_RE = re.compile("(" + PARAGRAPH_PATTERN + "){2}")
+
+# Captures all new line \n and keeps the \n as its own element,
+# considers \n\n as two separate elements
+LINE_BREAK = r"(?<=\n)"
+LINE_BREAK_RE = re.compile(LINE_BREAK)
+
+# NOTE(klaijan) - captures a line that does not ends with period (.)
+ONE_LINE_BREAK_PARAGRAPH_PATTERN = r"^(?:(?!\.\s*$).)*$"
+ONE_LINE_BREAK_PARAGRAPH_PATTERN_RE = re.compile(ONE_LINE_BREAK_PARAGRAPH_PATTERN)
+
+# IP Address examples: ba23::58b5:2236:45g2:88h2, 10.0.2.01 or 68.183.71.12
+IP_ADDRESS_PATTERN = (
+    r"(?:25[0-5]|2[0-4]\d|1\d{2}|[1-9]?\d)(?:\.(?:25[0-5]|2[0-4]\d|1\d{2}|[1-9]?\d)){3}",
+    "[a-z0-9]{4}::[a-z0-9]{4}:[a-z0-9]{4}:[a-z0-9]{4}:[a-z0-9]{4}%?[0-9]*",
+)
+IP_ADDRESS_PATTERN_RE = re.compile(f"({'|'.join(IP_ADDRESS_PATTERN)})")
+
+IP_ADDRESS_NAME_PATTERN = r"[a-zA-Z0-9-]*\.[a-zA-Z]*\.[a-zA-Z]*"
+
+# Mapi ID example: 32.88.5467.123
+MAPI_ID_PATTERN = r"[0-9]*\.[0-9]*\.[0-9]*\.[0-9]*;"
+
+# Date, time, timezone example: Fri, 26 Mar 2021 11:04:09 +1200
+EMAIL_DATETIMETZ_PATTERN = (
+    r"[A-Za-z]{3},\s\d{1,2}\s[A-Za-z]{3}\s\d{4}\s\d{2}:\d{2}:\d{2}\s[+-]\d{4}"
+)
+EMAIL_DATETIMETZ_PATTERN_RE = re.compile(EMAIL_DATETIMETZ_PATTERN)
+
+EMAIL_ADDRESS_PATTERN = r"[a-z0-9\.\-+_]+@[a-z0-9\.\-+_]+\.[a-z]+"
+EMAIL_ADDRESS_PATTERN_RE = re.compile(EMAIL_ADDRESS_PATTERN)
+
+ENDS_IN_PUNCT_PATTERN = r"[^\w\s]\Z"
+ENDS_IN_PUNCT_RE = re.compile(ENDS_IN_PUNCT_PATTERN)
+
+# NOTE(robinson) - Used to detect if text is in the expected "list of dicts"
+# format for document elements
+LIST_OF_DICTS_PATTERN = r"\A\s*\[\s*{?"
+
+
+# (?s) dot all (including newline characters)
+# \{(?=.*:) opening brace and at least one colon
+# .*? any characters (non-greedy)
+# (?:\}|$) non-capturing group that matches either the closing brace } or the end of
+# the string to handle cases where the JSON is cut off
+# | or
+# \[(?s:.*?)\] matches the opening bracket [ in a JSON array and any characters inside the array
+# (?:$|,|\]) non-capturing group that matches either the end of the string, a comma,
+# or the closing bracket to handle cases where the JSON array is cut off
+JSON_PATTERN = r"(?s)\{(?=.*:).*?(?:\}|$)|\[(?s:.*?)\](?:$|,|\])"
+
+
+# taken from https://stackoverflow.com/a/3845829/12406158
+VALID_JSON_CHARACTERS = r"[,:{}\[\]0-9.\-+Eaeflnr-u \n\r\t]"
+
+IMAGE_URL_PATTERN = (
+    r"(?i)https?://"
+    r"(?:[a-z0-9$_@.&+!*\\(\\),%-])+"
+    r"(?:/[a-z0-9$_@.&+!*\\(\\),%-]*)*"
+    r"\.(?:jpg|jpeg|png|gif|bmp|heic)"
+)
+
+# NOTE(klaijan) - only supports one level numbered list for now
+# e.g. 1. 2. 3. or 1) 2) 3), not 1.1 1.2 1.3
+NUMBERED_LIST_PATTERN = r"^\d+(\.|\))\s(.+)"
+NUMBERED_LIST_RE = re.compile(NUMBERED_LIST_PATTERN)

unstructured/nlp/tokenize.py

@@ -0,0 +1,192 @@
+from __future__ import annotations
+
+import hashlib
+import importlib
+import logging
+import os
+import shutil
+import sys
+import sysconfig
+import tempfile
+import urllib.error
+import urllib.request
+from functools import lru_cache
+from typing import Final, List, Tuple
+
+import spacy
+from filelock import FileLock
+
+logger = logging.getLogger(__name__)
+
+CACHE_MAX_SIZE: Final[int] = 128
+
+_SPACY_MODEL_NAME: Final[str] = "en_core_web_sm"
+_SPACY_MODEL_VERSION: Final[str] = "3.8.0"
+_SPACY_MODEL_URL: Final[str] = (
+    f"https://github.com/explosion/spacy-models/releases/download/"
+    f"{_SPACY_MODEL_NAME}-{_SPACY_MODEL_VERSION}/"
+    f"{_SPACY_MODEL_NAME}-{_SPACY_MODEL_VERSION}-py3-none-any.whl"
+)
+_SPACY_MODEL_SHA256: Final[str] = "1932429db727d4bff3deed6b34cfc05df17794f4a52eeb26cf8928f7c1a0fb85"
+
+
+_DOWNLOAD_TIMEOUT_SECONDS: Final[int] = 120
+_INSTALL_LOCK_PATH: Final[str] = os.path.join(
+    tempfile.gettempdir(), f"{_SPACY_MODEL_NAME}.install.lock"
+)
+
+
+def _download_with_timeout(url: str, dest: str) -> None:
+    """Download a URL to a local file with a socket-level timeout."""
+    try:
+        with urllib.request.urlopen(url, timeout=_DOWNLOAD_TIMEOUT_SECONDS) as resp:
+            with open(dest, "wb") as out:
+                shutil.copyfileobj(resp, out)
+    except urllib.error.URLError as exc:
+        raise RuntimeError(
+            f"Failed to download spaCy model from {url}: {exc}. "
+            "Check your network connection and try again."
+        ) from exc
+
+
+def _install_spacy_model() -> None:
+    """Download and install the pinned spaCy model wheel using the `installer` library."""
+    from installer import install
+    from installer.destinations import SchemeDictionaryDestination
+    from installer.sources import WheelFile
+    from installer.utils import get_launcher_kind
+
+    with tempfile.TemporaryDirectory() as tmp:
+        whl_path = os.path.join(tmp, f"{_SPACY_MODEL_NAME}-{_SPACY_MODEL_VERSION}-py3-none-any.whl")
+        logger.info("Downloading spaCy model %s %s …", _SPACY_MODEL_NAME, _SPACY_MODEL_VERSION)
+        _download_with_timeout(_SPACY_MODEL_URL, whl_path)
+
+        with open(whl_path, "rb") as f:
+            sha256 = hashlib.sha256(f.read()).hexdigest()
+        if sha256 != _SPACY_MODEL_SHA256:
+            raise RuntimeError(
+                f"Hash mismatch for {_SPACY_MODEL_NAME}: "
+                f"expected {_SPACY_MODEL_SHA256}, got {sha256}"
+            )
+
+        # Install into a staging directory to avoid races with other processes
+        staging = os.path.join(tmp, "staging")
+        paths = sysconfig.get_paths()
+        staged_paths = paths.copy()
+        staged_paths["purelib"] = staging
+        staged_paths["platlib"] = staging
+
+        destination = SchemeDictionaryDestination(
+            staged_paths,
+            interpreter=sys.executable,
+            script_kind=get_launcher_kind(),
+        )
+        with WheelFile.open(whl_path) as source:
+            install(source=source, destination=destination, additional_metadata={})
+
+        # Move installed packages from staging into real site-packages.
+        # The caller holds _INSTALL_LOCK_PATH so no other process races here.
+        # Any dst that already exists is a remnant of a previous failed install
+        # (spacy.load() just failed), so remove it before moving to avoid
+        # shutil.move placing src *inside* an existing directory.
+        site_packages = paths["purelib"]
+        for item in os.listdir(staging):
+            src = os.path.join(staging, item)
+            dst = os.path.join(site_packages, item)
+            try:
+                if os.path.isdir(dst):
+                    shutil.rmtree(dst)
+                elif os.path.exists(dst):
+                    os.remove(dst)
+                shutil.move(src, dst)
+            except OSError as exc:
+                raise RuntimeError(
+                    f"Failed to install {_SPACY_MODEL_NAME} to {site_packages}: {exc}. "
+                    "Ensure the site-packages directory is writable, or pre-install the model "
+                    f"with: python -m spacy download {_SPACY_MODEL_NAME}"
+                ) from exc
+
+    logger.info("Installed %s %s", _SPACY_MODEL_NAME, _SPACY_MODEL_VERSION)
+
+
+# Only tok2vec, tagger, parser (sentence boundaries), and sentencizer are used
+# (pos_tag and sent_tokenize). Excluding the remaining components saves ~7 MiB
+# of model weights per process.
+_SPACY_EXCLUDE = ["ner", "lemmatizer", "attribute_ruler"]
+
+
+def _load_spacy_model() -> spacy.language.Language:
+    try:
+        return spacy.load(_SPACY_MODEL_NAME, exclude=_SPACY_EXCLUDE)
+    except OSError:
+        pass
+
+    # Serialize model installation across processes with an exclusive file lock.
+    # A well-known path in the system temp dir is visible to all processes
+    # regardless of their working directory.
+    with FileLock(_INSTALL_LOCK_PATH, timeout=-1):
+        # Double-check: another process may have installed while we waited.
+        importlib.invalidate_caches()
+        try:
+            return spacy.load(_SPACY_MODEL_NAME, exclude=_SPACY_EXCLUDE)
+        except OSError:
+            pass
+        _install_spacy_model()
+        importlib.invalidate_caches()
+        try:
+            return spacy.load(_SPACY_MODEL_NAME, exclude=_SPACY_EXCLUDE)
+        except OSError as exc:
+            raise RuntimeError(
+                f"Installed {_SPACY_MODEL_NAME} but spacy.load() still failed. "
+                "Check site-packages permissions and installation integrity."
+            ) from exc
+
+
+@lru_cache(maxsize=1)
+def _get_nlp() -> spacy.language.Language:
+    """Load the spaCy model on first use and cache it for the lifetime of the process."""
+    return _load_spacy_model()
+
+
+def _process(text: str) -> spacy.tokens.Doc:
+    """Run the spaCy pipeline once. All public functions extract what they need from the Doc."""
+    # -- str() handles numpy.str_ from OCR pipelines --
+    text = str(text)
+    nlp = _get_nlp()
+    if len(text) > nlp.max_length:
+        logger.warning(
+            "Input text of length %d exceeds spaCy max_length=%d; "
+            "truncating for partition heuristics.",
+            len(text),
+            nlp.max_length,
+        )
+        # Prefer to cut at the last whitespace within the budget so we don't split a token.
+        cut = text.rfind(" ", max(0, nlp.max_length - 256), nlp.max_length)
+        truncated = text[: cut if cut != -1 else nlp.max_length]
+        return nlp(truncated)
+    return nlp(text)
+
+
+def sent_tokenize(text: str) -> List[str]:
+    """A wrapper so that we can cache the result of sentence tokenization as an
+    immutable, while returning the expected return type (list)."""
+    return list(_tokenize_for_cache(text))
+
+
+@lru_cache(maxsize=CACHE_MAX_SIZE)
+def word_tokenize(text: str) -> List[str]:
+    """A wrapper around the spaCy word tokenizer with LRU caching enabled."""
+    return [token.text for token in _process(text)]
+
+
+@lru_cache(maxsize=CACHE_MAX_SIZE)
+def pos_tag(text: str) -> List[Tuple[str, str]]:
+    """A wrapper around the spaCy POS tagger with LRU caching enabled."""
+    doc = _process(text)
+    return [(token.text, token.tag_) for token in doc]
+
+
+@lru_cache(maxsize=CACHE_MAX_SIZE)
+def _tokenize_for_cache(text: str) -> Tuple[str, ...]:
+    """A wrapper around the spaCy sentence tokenizer with LRU caching enabled."""
+    return tuple(sent.text for sent in _process(text).sents)

unstructured/partition/api.py

@@ -0,0 +1,341 @@
+from __future__ import annotations
+
+import contextlib
+from typing import IO, Any, Optional, Sequence
+
+import requests
+from unstructured_client import UnstructuredClient
+from unstructured_client.models import operations, shared
+from unstructured_client.utils import retries
+
+from unstructured.documents.elements import Element
+from unstructured.logger import logger
+from unstructured.partition.common.common import exactly_one
+from unstructured.staging.base import elements_from_dicts, elements_from_json
+
+# Default retry configuration taken from the client code
+DEFAULT_RETRIES_INITIAL_INTERVAL_SEC = 3000
+DEFAULT_RETRIES_MAX_INTERVAL_SEC = 720000
+DEFAULT_RETRIES_EXPONENT = 1.5
+DEFAULT_RETRIES_MAX_ELAPSED_TIME_SEC = 1800000
+DEFAULT_RETRIES_CONNECTION_ERRORS = True
+
+
+def partition_via_api(
+    filename: Optional[str] = None,
+    content_type: Optional[str] = None,
+    file: Optional[IO[bytes]] = None,
+    file_filename: Optional[str] = None,
+    api_url: str = "https://api.unstructured.io/general/v0/general",
+    api_key: str = "",
+    metadata_filename: Optional[str] = None,
+    retries_initial_interval: [int] = None,
+    retries_max_interval: Optional[int] = None,
+    retries_exponent: Optional[float] = None,
+    retries_max_elapsed_time: Optional[int] = None,
+    retries_connection_errors: Optional[bool] = None,
+    **request_kwargs: Any,
+) -> list[Element]:
+    """Partitions a document using the Unstructured REST API. This is equivalent to
+    running the document through partition.
+
+    See https://api.unstructured.io/general/docs for the hosted API documentation or
+    https://github.com/Unstructured-IO/unstructured-api for instructions on how to run
+    the API locally as a container.
+
+    Parameters
+    ----------
+    filename
+        A string defining the target filename path.
+    content_type
+        A string defining the file content in MIME type
+    file
+        A file-like object using "rb" mode --> open(filename, "rb").
+    metadata_filename
+        When file is not None, the filename (string) to store in element metadata. E.g. "foo.txt"
+    api_url
+        The URL for the Unstructured API. Defaults to the hosted Unstructured API.
+    api_key
+        The API key to pass to the Unstructured API.
+    retries_initial_interval
+        Defines the time interval (in seconds) to wait before the first retry in case of a request
+        failure. Defaults to 3000. If set should be > 0.
+    retries_max_interval
+        Defines the maximum time interval (in seconds) to wait between retries (the interval
+        between retries is increased as using exponential increase algorithm
+        - this setting limits it). Defaults to 720000. If set should be > 0.
+    retries_exponent
+        Defines the exponential factor to increase the interval between retries. Defaults to 1.5.
+        If set should be > 0.0.
+    retries_max_elapsed_time
+        Defines the maximum time (in seconds) to wait for retries. If exceeded, the original
+        exception is raised. Defaults to 1800000. If set should be > 0.
+    retries_connection_errors
+        Defines whether to retry on connection errors. Defaults to True.
+    request_kwargs
+        Additional parameters to pass to the data field of the request to the Unstructured API.
+        For example the `strategy` parameter.
+    """
+    exactly_one(filename=filename, file=file)
+
+    if metadata_filename and file_filename:
+        raise ValueError(
+            "Only one of metadata_filename and file_filename is specified. "
+            "metadata_filename is preferred. file_filename is marked for deprecation.",
+        )
+
+    if file_filename is not None:
+        metadata_filename = file_filename
+        logger.warning(
+            "The file_filename kwarg will be deprecated in a future version of unstructured. "
+            "Please use metadata_filename instead.",
+        )
+
+    # Note(austin) - the sdk takes the base url, but we have the full api_url
+    # For consistency, just strip off the path when it's given
+    base_url = api_url[:-19] if "/general/v0/general" in api_url else api_url
+    sdk = UnstructuredClient(api_key_auth=api_key, server_url=base_url)
+
+    if filename is not None:
+        with open(filename, "rb") as f:
+            files = shared.Files(
+                content=f.read(),
+                file_name=filename,
+            )
+
+    elif file is not None:
+        if metadata_filename is None:
+            raise ValueError(
+                "If file is specified in partition_via_api, "
+                "metadata_filename must be specified as well.",
+            )
+        files = shared.Files(content=file, file_name=metadata_filename)
+
+    req = operations.PartitionRequest(
+        partition_parameters=shared.PartitionParameters(files=files, **request_kwargs)
+    )
+
+    retries_config = get_retries_config(
+        retries_connection_errors=retries_connection_errors,
+        retries_exponent=retries_exponent,
+        retries_initial_interval=retries_initial_interval,
+        retries_max_elapsed_time=retries_max_elapsed_time,
+        retries_max_interval=retries_max_interval,
+        sdk=sdk,
+    )
+
+    response = sdk.general.partition(
+        request=req,
+        retries=retries_config,
+    )
+
+    if response.status_code == 200:
+        return elements_from_json(text=response.raw_response.text)
+    else:
+        raise ValueError(
+            f"Receive unexpected status code {response.status_code} from the API.",
+        )
+
+
+def get_retries_config(
+    retries_connection_errors: Optional[bool],
+    retries_exponent: Optional[float],
+    retries_initial_interval: Optional[int],
+    retries_max_elapsed_time: Optional[int],
+    retries_max_interval: Optional[int],
+    sdk: UnstructuredClient,
+) -> Optional[retries.RetryConfig]:
+    """Constructs a RetryConfig object from the provided parameters. If any of the parameters
+    are None, the default values are taken from the SDK configuration or the default constants.
+
+    If all parameters are None, returns None (and the SDK-managed defaults are used within the
+    client)
+
+    The solution is not perfect as the RetryConfig object does not include the defaults by
+    itself so we might need to construct it basing on our defaults.
+
+    Parameters
+    ----------
+    retries_connection_errors
+        Defines whether to retry on connection errors. If not set the
+        DEFAULT_RETRIES_CONNECTION_ERRORS constant is used.
+    retries_exponent
+        Defines the exponential factor to increase the interval between retries.
+        If set, should be > 0.0 (otherwise the DEFAULT_RETRIES_EXPONENT constant is used)
+    retries_initial_interval
+        Defines the time interval to wait before the first retry in case of a request failure.
+        If set, should be > 0 (otherwise the DEFAULT_RETRIES_INITIAL_INTERVAL_SEC constant is used)
+    retries_max_elapsed_time
+        Defines the maximum time to wait for retries. If exceeded, the original exception is raised.
+        If set, should be > 0 (otherwise the DEFAULT_RETRIES_MAX_ELAPSED_TIME_SEC constant is used)
+    retries_max_interval
+        Defines the maximum time interval to wait between retries. If set, should be > 0
+        (otherwise the DEFAULT_RETRIES_MAX_INTERVAL_SEC constant is used)
+    sdk
+        The UnstructuredClient object to take the default values from.
+    """
+    retries_config = None
+    sdk_default_retries_config = sdk.sdk_configuration.retry_config
+    if any(
+        setting is not None
+        for setting in (
+            retries_initial_interval,
+            retries_max_interval,
+            retries_exponent,
+            retries_max_elapsed_time,
+            retries_connection_errors,
+        )
+    ):
+
+        def get_backoff_default(setting_name: str, default_value: Any) -> Any:
+            if sdk_default_retries_config:  # noqa: SIM102
+                if setting_value := getattr(sdk_default_retries_config.backoff, setting_name):
+                    return setting_value
+            return default_value
+
+        default_retries_connneciton_errors = (
+            sdk_default_retries_config.retry_connection_errors
+            if sdk_default_retries_config
+            and sdk_default_retries_config.retry_connection_errors is not None
+            else DEFAULT_RETRIES_CONNECTION_ERRORS
+        )
+
+        backoff_strategy = retries.BackoffStrategy(
+            initial_interval=(
+                retries_initial_interval
+                or get_backoff_default("initial_interval", DEFAULT_RETRIES_INITIAL_INTERVAL_SEC)
+            ),
+            max_interval=(
+                retries_max_interval
+                or get_backoff_default("max_interval", DEFAULT_RETRIES_MAX_INTERVAL_SEC)
+            ),
+            exponent=(
+                retries_exponent or get_backoff_default("exponent", DEFAULT_RETRIES_EXPONENT)
+            ),
+            max_elapsed_time=(
+                retries_max_elapsed_time
+                or get_backoff_default("max_elapsed_time", DEFAULT_RETRIES_MAX_ELAPSED_TIME_SEC)
+            ),
+        )
+        retries_config = retries.RetryConfig(
+            strategy="backoff",
+            backoff=backoff_strategy,
+            retry_connection_errors=(
+                retries_connection_errors
+                if retries_connection_errors is not None
+                else default_retries_connneciton_errors
+            ),
+        )
+    return retries_config
+
+
+def partition_multiple_via_api(
+    filenames: Optional[list[str]] = None,
+    content_types: Optional[list[str]] = None,
+    files: Optional[Sequence[IO[bytes]]] = None,
+    file_filenames: Optional[list[str]] = None,
+    api_url: str = "https://api.unstructured.io/general/v0/general",
+    api_key: str = "",
+    metadata_filenames: Optional[list[str]] = None,
+    **request_kwargs: Any,
+) -> list[list[Element]]:
+    """Partitions multiple documents using the Unstructured REST API by batching
+    the documents into a single HTTP request.
+
+    See https://api.unstructured.io/general/docs for the hosted API documentation or
+    https://github.com/Unstructured-IO/unstructured-api for instructions on how to run
+    the API locally as a container.
+
+    Parameters
+    ----------
+    filenames
+        A list of strings defining the target filename paths.
+    content_types
+        A list of strings defining the file contents in MIME types.
+    files
+        A list of file-like object using "rb" mode --> open(filename, "rb").
+    metadata_filename
+        When file is not None, the filename (string) to store in element metadata. E.g. "foo.txt"
+    api_url
+        The URL for the Unstructured API. Defaults to the hosted Unstructured API.
+    api_key
+        The API key to pass to the Unstructured API.
+    request_kwargs
+        Additional parameters to pass to the data field of the request to the Unstructured API.
+        For example the `strategy` parameter.
+    """
+    headers = {
+        "ACCEPT": "application/json",
+        "UNSTRUCTURED-API-KEY": api_key,
+    }
+
+    if metadata_filenames and file_filenames:
+        raise ValueError(
+            "Only one of metadata_filenames and file_filenames is specified. "
+            "metadata_filenames is preferred. file_filenames is marked for deprecation.",
+        )
+
+    if file_filenames is not None:
+        metadata_filenames = file_filenames
+        logger.warning(
+            "The file_filenames kwarg will be deprecated in a future version of unstructured. "
+            "Please use metadata_filenames instead.",
+        )
+
+    if filenames is not None:
+        if content_types and len(content_types) != len(filenames):
+            raise ValueError("content_types and filenames must have the same length.")
+
+        with contextlib.ExitStack() as stack:
+            files = [stack.enter_context(open(f, "rb")) for f in filenames]  # type: ignore
+
+            _files = []
+            for i, file in enumerate(files):
+                filename = filenames[i]
+                content_type = content_types[i] if content_types is not None else None
+                _files.append(("files", (filename, file, content_type)))
+
+            response = requests.post(
+                api_url,
+                headers=headers,
+                data=request_kwargs,
+                files=_files,  # type: ignore
+            )
+
+    elif files is not None:
+        if content_types and len(content_types) != len(files):
+            raise ValueError("content_types and files must have the same length.")
+
+        if not metadata_filenames:
+            raise ValueError("metadata_filenames must be specified if files are passed")
+        elif len(metadata_filenames) != len(files):
+            raise ValueError("metadata_filenames and files must have the same length.")
+
+        _files = []
+        for i, _file in enumerate(files):  # type: ignore
+            content_type = content_types[i] if content_types is not None else None
+            filename = metadata_filenames[i]
+            _files.append(("files", (filename, _file, content_type)))
+
+        response = requests.post(
+            api_url,
+            headers=headers,
+            data=request_kwargs,
+            files=_files,  # type: ignore
+        )
+
+    if response.status_code == 200:
+        documents = []
+        response_list = response.json()
+        # NOTE(robinson) - this check is because if only one filename is passed, the return
+        # type from the API is a list of objects instead of a list of lists
+        if not isinstance(response_list[0], list):
+            response_list = [response_list]
+
+        for document in response_list:
+            documents.append(elements_from_dicts(document))
+        return documents
+    else:
+        raise ValueError(
+            f"Receive unexpected status code {response.status_code} from the API.",
+        )

unstructured/partition/audio.py

@@ -0,0 +1,145 @@
+"""Partition audio files into elements using speech-to-text transcription."""
+
+from __future__ import annotations
+
+import shutil
+import tempfile
+from pathlib import Path
+from typing import IO, Any
+
+from unstructured.chunking import add_chunking_strategy
+from unstructured.documents.elements import Element, ElementMetadata, NarrativeText
+from unstructured.file_utils.filetype import detect_filetype
+from unstructured.file_utils.model import FileType
+from unstructured.partition.common.common import exactly_one
+from unstructured.partition.common.metadata import apply_metadata, get_last_modified_date
+from unstructured.partition.utils.speech_to_text.speech_to_text_interface import (
+    SpeechToTextAgent,
+)
+from unstructured.utils import is_temp_file_path
+
+_AUDIO_MIME_TYPE_FALLBACK = FileType.WAV.mime_type
+
+
+@apply_metadata()
+@add_chunking_strategy
+def partition_audio(
+    filename: str | None = None,
+    *,
+    file: IO[bytes] | None = None,
+    language: str | None = None,
+    stt_agent: str | None = None,
+    metadata_filename: str | None = None,
+    metadata_last_modified: str | None = None,
+    **kwargs: Any,
+) -> list[Element]:
+    """Partition an audio file into elements using speech-to-text transcription.
+
+    Supports any audio format handled by the configured STT agent. The default Whisper
+    agent accepts any format that ffmpeg supports (WAV, MP3, FLAC, M4A, OGG, OPUS, WEBM,
+    and more). Transcription produces one NarrativeText element per segment, preserving
+    segment-level timestamps in ``metadata.segment_start_seconds`` and
+    ``metadata.segment_end_seconds``. Requires the optional ``audio`` extra:
+    ``pip install "unstructured[audio]"``.
+
+    .. note::
+        **Chunking drops segment timestamps.** When elements are merged by a chunking
+        strategy, ``segment_start_seconds`` and ``segment_end_seconds`` are currently
+        discarded (consolidation strategy ``DROP``). If audio timeline alignment matters
+        for your use-case, consume the un-chunked elements directly.
+
+    Parameters
+    ----------
+    filename
+        Path to the audio file.
+    file
+        File-like object opened in binary mode (e.g. ``open("audio.mp3", "rb")``).
+    language
+        Optional ISO 639-1 language code for the spoken language (e.g. ``"en"``).
+        When ``None``, the speech-to-text agent may auto-detect.
+    stt_agent
+        Optional fully-qualified class name of the SpeechToTextAgent implementation.
+        Defaults to the Whisper agent when the ``audio`` extra is installed.
+    metadata_filename
+        Filename to store in element metadata when partitioning from a file object.
+    metadata_last_modified
+        Last modified date to store in element metadata.
+    """
+    exactly_one(filename=filename, file=file)
+
+    # Metadata from original input only (not the temp path); compute before any temp file lifecycle.
+    last_modified = get_last_modified_date(filename) if filename else None
+    mime_type = _audio_mime_type(filename, file, metadata_filename)
+
+    audio_path: str
+    if filename is not None:
+        audio_path = filename
+    else:
+        assert file is not None  # guaranteed by exactly_one()
+        file.seek(0)
+        suffix = _audio_suffix(file, metadata_filename)
+        with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
+            shutil.copyfileobj(file, tmp)
+            audio_path = tmp.name
+
+    try:
+        agent = SpeechToTextAgent.get_agent(stt_agent)
+        segments = agent.transcribe_segments(audio_path, language=language)
+    finally:
+        if filename is None and is_temp_file_path(audio_path):
+            Path(audio_path).unlink(missing_ok=True)
+
+    if not segments:
+        return []
+
+    elements: list[Element] = []
+    for seg in segments:
+        text = seg["text"].strip()
+        if not text:
+            continue
+        element = NarrativeText(text=text)
+        element.metadata = ElementMetadata(
+            filetype=mime_type,
+            last_modified=last_modified,
+            segment_start_seconds=seg["start"],
+            segment_end_seconds=seg["end"],
+        )
+        element.metadata.detection_origin = "speech_to_text"
+        elements.append(element)
+
+    return elements
+
+
+def _audio_mime_type(
+    filename: str | None,
+    file: IO[bytes] | None,
+    metadata_filename: str | None,
+) -> str:
+    """Return the MIME type for the audio source, falling back to WAV when undetectable."""
+    detected: FileType | None = None
+    if filename is not None:
+        detected = detect_filetype(file_path=filename)
+    elif file is not None:
+        detected = detect_filetype(
+            file=file,
+            metadata_file_path=metadata_filename,
+        )
+    if detected is not None and detected not in (FileType.UNK, FileType.EMPTY):
+        return detected.mime_type
+    return _AUDIO_MIME_TYPE_FALLBACK
+
+
+def _audio_suffix(file: IO[bytes], metadata_filename: str | None) -> str:
+    """Return the file-extension suffix for a temp file that wraps `file`.
+
+    Preference order:
+    1. Extension of `metadata_filename` when provided (e.g. ".mp3" from "recording.mp3").
+    2. Extension of `file.name` when the file object exposes a name (e.g. a real opened file).
+    3. ".wav" as a safe fallback recognised by all STT backends.
+    """
+    for name in (metadata_filename, getattr(file, "name", None)):
+        if name:
+            suffix = Path(name).suffix
+            if suffix:
+                return suffix
+    return ".wav"

unstructured/partition/auto.py

@@ -0,0 +1,390 @@
+"""Provides partitioning with automatic file-type detection."""
+
+from __future__ import annotations
+
+import copy
+import importlib
+import io
+from typing import IO, Any, Callable, Optional
+
+from typing_extensions import TypeAlias
+
+from unstructured.documents.elements import DataSourceMetadata, Element
+from unstructured.file_utils.filetype import (
+    detect_filetype,
+    is_json_processable,
+    is_ndjson_processable,
+)
+from unstructured.file_utils.model import FileType
+from unstructured.logger import logger
+from unstructured.partition.common import UnsupportedFileFormatError
+from unstructured.partition.common.common import exactly_one
+from unstructured.partition.common.lang import check_language_args
+from unstructured.partition.utils.constants import PartitionStrategy
+from unstructured.safe_http import safe_get
+from unstructured.utils import dependency_exists
+
+Partitioner: TypeAlias = Callable[..., list[Element]]
+
+
+def partition(
+    filename: Optional[str] = None,
+    *,
+    file: Optional[IO[bytes]] = None,
+    encoding: Optional[str] = None,
+    content_type: Optional[str] = None,
+    url: Optional[str] = None,
+    headers: dict[str, str] = {},
+    ssl_verify: bool = True,
+    request_timeout: Optional[int] = None,
+    strategy: str = PartitionStrategy.AUTO,
+    skip_infer_table_types: list[str] = ["pdf", "jpg", "png", "heic"],
+    ocr_languages: Optional[str] = None,  # changing to optional for deprecation
+    languages: Optional[list[str]] = None,
+    detect_language_per_element: bool = False,
+    language_fallback: Optional[Callable[[str], Optional[list[str]]]] = None,
+    pdf_infer_table_structure: bool = False,
+    extract_images_in_pdf: bool = False,
+    extract_image_block_types: Optional[list[str]] = None,
+    extract_image_block_output_dir: Optional[str] = None,
+    extract_image_block_to_payload: bool = False,
+    data_source_metadata: Optional[DataSourceMetadata] = None,
+    metadata_filename: Optional[str] = None,
+    hi_res_model_name: Optional[str] = None,
+    model_name: Optional[str] = None,  # to be deprecated
+    starting_page_number: int = 1,
+    **kwargs: Any,
+) -> list[Element]:
+    """Partitions a document into its constituent elements.
+
+    Uses libmagic to determine the file's type and route it to the appropriate partitioning
+    function. Applies the default parameters for each partitioning function. Use the document-type
+    specific partitioning functions if you need access to additional kwarg options.
+
+    Parameters
+    ----------
+    filename
+        A string defining the target filename path.
+    file
+        A file-like object using "rb" mode --> open(filename, "rb").
+    encoding
+        The character-encoding used to decode the input bytes when drawn from `filename` or `file`.
+        Defaults to "utf-8".
+    url
+        The url for a remote document. Pass in content_type if you want partition to treat
+        the document as a specific content_type.
+    headers
+        The headers to be used in conjunction with the HTTP request if URL is set.
+    ssl_verify
+        If the URL parameter is set, determines whether or not partition uses SSL verification
+        in the HTTP request.
+    request_timeout
+        The timeout for the HTTP request if URL is set. Defaults to None meaning no timeout and
+        requests will block indefinitely.
+    content_type
+        A string defining the file content in MIME type
+    metadata_filename
+        When file is not None, the filename (string) to store in element metadata. E.g. "foo.txt"
+    strategy
+        The strategy to use for partitioning PDF/image. Uses a layout detection model if set
+        to 'hi_res', otherwise partition simply extracts the text from the document
+        and processes it.
+    skip_infer_table_types
+        The document types that you want to skip table extraction with.
+    languages
+        The languages present in the document, for use in partitioning and/or OCR. For partitioning
+        image or pdf documents with Tesseract, you'll first need to install the appropriate
+        Tesseract language pack. For other partitions, language is detected using naive Bayesian
+        filter via `langdetect`. Multiple languages indicates text could be in either language.
+    detect_language_per_element
+        Detect language per element instead of at the document level.
+    language_fallback
+        Optional callable for short text (e.g. when detection defaults to English).
+        Called with the text; return a list of ISO 639-3 codes or None to leave
+        language unspecified.
+    pdf_infer_table_structure
+        Deprecated! Use `skip_infer_table_types` to opt out of table extraction for any document
+        type.
+        If True and strategy=hi_res, any Table Elements extracted from a PDF will include an
+        additional metadata field, "text_as_html," where the value (string) is a just a
+        transformation of the data into an HTML <table>.
+        The "text" field for a partitioned Table Element is always present, whether True or False.
+    extract_images_in_pdf
+        Only applicable if `strategy=hi_res`.
+        If True, any detected images will be saved in the path specified by
+        'extract_image_block_output_dir' or stored as base64 encoded data within metadata fields.
+        Deprecation Note: This parameter is marked for deprecation. Future versions will use
+        'extract_image_block_types' for broader extraction capabilities.
+    extract_image_block_types
+        Only applicable if `strategy=hi_res`.
+        Images of the element type(s) specified in this list (e.g., ["Image", "Table"]) will be
+        saved in the path specified by 'extract_image_block_output_dir' or stored as base64
+        encoded data within metadata fields.
+    extract_image_block_to_payload
+        Only applicable if `strategy=hi_res`.
+        If True, images of the element type(s) defined in 'extract_image_block_types' will be
+        encoded as base64 data and stored in two metadata fields: 'image_base64' and
+        'image_mime_type'.
+        This parameter facilitates the inclusion of element data directly within the payload,
+        especially for web-based applications or APIs.
+    extract_image_block_output_dir
+        Only applicable if `strategy=hi_res` and `extract_image_block_to_payload=False`.
+        The filesystem path for saving images of the element type(s)
+        specified in 'extract_image_block_types'.
+    hi_res_model_name
+        The layout detection model used when partitioning strategy is set to `hi_res`.
+    model_name
+        The layout detection model used when partitioning strategy is set to `hi_res`. To be
+        deprecated in favor of `hi_res_model_name`.
+    starting_page_number
+        Indicates what page number should be assigned to the first page in the document.
+        This information will be reflected in elements' metadata and can be be especially
+        useful when partitioning a document that is part of a larger document.
+    """
+    exactly_one(file=file, filename=filename, url=url)
+
+    kwargs.setdefault("metadata_filename", metadata_filename)
+
+    if pdf_infer_table_structure:
+        logger.warning(
+            "The pdf_infer_table_structure kwarg is deprecated. Please use skip_infer_table_types "
+            "instead."
+        )
+
+    languages = check_language_args(languages or [], ocr_languages)
+
+    if url is not None:
+        file, file_type = file_and_type_from_url(
+            url=url,
+            content_type=content_type,
+            headers=headers,
+            ssl_verify=ssl_verify,
+            request_timeout=request_timeout,
+        )
+    else:
+        if headers != {}:
+            logger.warning(
+                "The headers kwarg is set but the url kwarg is not. "
+                "The headers kwarg will be ignored.",
+            )
+        file_type = detect_filetype(
+            file_path=filename,
+            file=file,
+            encoding=encoding,
+            content_type=content_type,
+            metadata_file_path=metadata_filename,
+        )
+
+    if file is not None:
+        file.seek(0)
+
+    # avoid double specification of infer_table_structure; this can happen when the kwarg passed
+    # into a partition function, e.g., partition_email is reused to partition sub-elements, e.g.,
+    # partition an image attachment buy calling partition with the kwargs. In that case here kwargs
+    # would have a infer_table_structure already
+    kwargs_infer_table_structure = kwargs.pop("infer_table_structure", None)
+    infer_table_structure = (
+        kwargs_infer_table_structure
+        if kwargs_infer_table_structure is not None
+        else decide_table_extraction(
+            file_type,
+            skip_infer_table_types,
+            pdf_infer_table_structure,
+        )
+    )
+
+    partitioner_loader = _PartitionerLoader()
+
+    # -- extracting this post-processing to allow multiple exit-points from function --
+    def augment_metadata(elements: list[Element]) -> list[Element]:
+        """Add some metadata fields to each element."""
+        for element in elements:
+            element.metadata.url = url
+            element.metadata.data_source = data_source_metadata
+            if content_type is not None:
+                out_filetype = FileType.from_mime_type(content_type)
+                element.metadata.filetype = out_filetype.mime_type if out_filetype else None
+            else:
+                element.metadata.filetype = file_type.mime_type
+
+        return elements
+
+    # -- handle PDF/Image partitioning separately because they have a lot of special-case
+    # -- parameters. We'll come back to this after sorting out the other file types.
+    if file_type == FileType.PDF:
+        partition_pdf = partitioner_loader.get(file_type)
+        elements = partition_pdf(
+            filename=filename,
+            file=file,
+            url=None,
+            infer_table_structure=infer_table_structure,
+            strategy=strategy,
+            languages=languages,
+            detect_language_per_element=detect_language_per_element,
+            language_fallback=language_fallback,
+            hi_res_model_name=hi_res_model_name or model_name,
+            extract_images_in_pdf=extract_images_in_pdf,
+            extract_image_block_types=extract_image_block_types,
+            extract_image_block_output_dir=extract_image_block_output_dir,
+            extract_image_block_to_payload=extract_image_block_to_payload,
+            starting_page_number=starting_page_number,
+            **kwargs,
+        )
+        return augment_metadata(elements)
+
+    if file_type.partitioner_shortname and file_type.partitioner_shortname == "image":
+        partition_image = partitioner_loader.get(file_type)
+        elements = partition_image(
+            filename=filename,
+            file=file,
+            url=None,
+            infer_table_structure=infer_table_structure,
+            strategy=strategy,
+            languages=languages,
+            detect_language_per_element=detect_language_per_element,
+            language_fallback=language_fallback,
+            hi_res_model_name=hi_res_model_name or model_name,
+            extract_images_in_pdf=extract_images_in_pdf,
+            extract_image_block_types=extract_image_block_types,
+            extract_image_block_output_dir=extract_image_block_output_dir,
+            extract_image_block_to_payload=extract_image_block_to_payload,
+            starting_page_number=starting_page_number,
+            **kwargs,
+        )
+        return augment_metadata(elements)
+
+    # -- JSON is a special case because it's not a document format per se and is insensitive to
+    # -- most of the parameters that apply to other file types.
+    if file_type == FileType.JSON:
+        if not is_json_processable(filename=filename, file=file):
+            raise ValueError(
+                "Detected a JSON file that does not conform to the Unstructured schema. "
+                "partition_json currently only processes serialized Unstructured output.",
+            )
+        partition_json = partitioner_loader.get(file_type)
+        elements = partition_json(filename=filename, file=file, **kwargs)
+        return augment_metadata(elements)
+
+    if file_type == FileType.NDJSON:
+        if not is_ndjson_processable(filename=filename, file=file):
+            raise ValueError(
+                "Detected an NDJSON file that does not conform to the Unstructured schema. "
+                "partition_json currently only processes serialized Unstructured output.",
+            )
+        partition_ndjson = partitioner_loader.get(file_type)
+        elements = partition_ndjson(filename=filename, file=file, **kwargs)
+        return augment_metadata(elements)
+
+    # -- EMPTY is also a special case because while we can't determine the file type, we can be
+    # -- sure it doesn't contain any elements.
+    if file_type == FileType.EMPTY:
+        return []
+
+    # ============================================================================================
+    #  ALL OTHER FILE TYPES
+    # ============================================================================================
+
+    partitioning_kwargs = copy.deepcopy(kwargs)
+    partitioning_kwargs["detect_language_per_element"] = detect_language_per_element
+    partitioning_kwargs["language_fallback"] = language_fallback
+    partitioning_kwargs["encoding"] = encoding
+    partitioning_kwargs["infer_table_structure"] = infer_table_structure
+    partitioning_kwargs["languages"] = languages
+    partitioning_kwargs["starting_page_number"] = starting_page_number
+    partitioning_kwargs["strategy"] = strategy
+    partitioning_kwargs["extract_image_block_types"] = extract_image_block_types
+    partitioning_kwargs["extract_image_block_to_payload"] = extract_image_block_to_payload
+
+    partition = partitioner_loader.get(file_type)
+    elements = partition(filename=filename, file=file, **partitioning_kwargs)
+    return augment_metadata(elements)
+
+
+def file_and_type_from_url(
+    url: str,
+    content_type: Optional[str] = None,
+    headers: dict[str, str] = {},
+    ssl_verify: bool = True,
+    request_timeout: Optional[int] = None,
+) -> tuple[io.BytesIO, FileType]:
+    response = safe_get(url, headers=headers, verify=ssl_verify, timeout=request_timeout)
+    file = io.BytesIO(response.content)
+
+    if content_type := content_type or response.headers.get("Content-Type", None):
+        content_type = content_type.split(";")[0].strip().lower()
+
+    # -- non-None when response is textual --
+    encoding = response.encoding
+
+    filetype = detect_filetype(file=file, encoding=encoding, content_type=content_type)
+    return file, filetype
+
+
+def decide_table_extraction(
+    filetype: Optional[FileType],
+    skip_infer_table_types: list[str],
+    pdf_infer_table_structure: bool,
+) -> bool:
+    doc_type = filetype.name.lower() if filetype else None
+
+    if doc_type == "pdf":
+        # For backwards compatibility. Ultimately we want to remove pdf_infer_table_structure
+        # completely and rely exclusively on `skip_infer_table_types` for all file types.
+        # Until then for pdf files we first check pdf_infer_table_structure and then update
+        # based on skip_infer_tables.
+        return pdf_infer_table_structure or doc_type not in skip_infer_table_types
+
+    return doc_type not in skip_infer_table_types
+
+
+class _PartitionerLoader:
+    """Provides uniform helpful error when a partitioner dependency is not installed.
+
+    Used by `partition()` to encapsulate coping with the possibility the Python environment it is
+    executing in may not have all dependencies installed for a particular partitioner.
+
+    Provides `.get()` to access partitioners by file-type, which raises when one or more
+    dependencies for that partitioner are not installed.
+
+    The error message indicates what extra needs to be installed to enable that partitioner. This
+    avoids an inconsistent variety of possibly puzzling exceptions arising from much deeper in the
+    partitioner when access to the missing dependency is first attempted.
+    """
+
+    # -- module-lifetime cache for partitioners once loaded --
+    _partitioners: dict[FileType, Partitioner] = {}
+
+    def get(self, file_type: FileType) -> Partitioner:
+        """Return partitioner for `file_type`.
+
+        Raises when one or more package dependencies for that file-type have not been
+        installed. Also raises when the file-type is not partitionable.
+        """
+        if not file_type.is_partitionable:
+            raise UnsupportedFileFormatError(
+                f"Partitioning is not supported for the {file_type} file type."
+            )
+
+        # -- if the partitioner is not in the cache, load it; note this raises if one or more of
+        # -- the partitioner's dependencies is not installed.
+        if file_type not in self._partitioners:
+            self._partitioners[file_type] = self._load_partitioner(file_type)
+
+        return self._partitioners[file_type]
+
+    def _load_partitioner(self, file_type: FileType) -> Partitioner:
+        """Load the partitioner for `file_type` after verifying dependencies."""
+        # -- verify all package dependencies are installed --
+        for pkg_name in file_type.importable_package_dependencies:
+            if not dependency_exists(pkg_name):
+                raise ImportError(
+                    f"{file_type.partitioner_function_name}() is not available because one or"
+                    f" more dependencies are not installed. Use:"
+                    f' pip install "unstructured[{file_type.extra_name}]" (including quotes)'
+                    f" to install the required dependencies",
+                )
+
+        # -- load the partitioner and return it --
+        assert file_type.is_partitionable  # -- would be a programming error if this failed --
+        partitioner_module = importlib.import_module(file_type.partitioner_module_qname)
+        return getattr(partitioner_module, file_type.partitioner_function_name)

unstructured/partition/common/__init__.py

@@ -0,0 +1,19 @@
+from typing import Final
+
+
+class UnsupportedFileFormatError(Exception):
+    """File-type is not supported for this operation.
+
+    For example, when receiving a file for auto-partitioning where its file-formatt cannot be
+    identified or there is no partitioner available for that file-format.
+    """
+
+
+# Exceptions that email/msg partitioners treat as "unsupported attachment" and skip with a
+# warning (no data loss). Intentionally narrow: we do not catch RuntimeError (OOM, broken pipe,
+# parser failures in e.g. pdfminer would otherwise be silently skipped).
+EXPECTED_ATTACHMENT_ERRORS: Final[tuple[type[BaseException], ...]] = (
+    UnsupportedFileFormatError,
+    ImportError,
+    FileNotFoundError,
+)

unstructured/partition/common/common.py

@@ -0,0 +1,460 @@
+from __future__ import annotations
+
+import numbers
+import subprocess
+from enum import Enum
+from io import BufferedReader, BytesIO, TextIOWrapper
+from tempfile import SpooledTemporaryFile
+from time import sleep
+from typing import IO, TYPE_CHECKING, Any, Optional, TypeVar, cast
+
+import emoji
+import psutil
+
+from unstructured.documents.coordinates import CoordinateSystem, PixelSpace
+from unstructured.documents.elements import (
+    TYPE_TO_TEXT_ELEMENT_MAP,
+    CheckBox,
+    CoordinatesMetadata,
+    Element,
+    ElementMetadata,
+    ElementType,
+    ListItem,
+    PageBreak,
+    Text,
+)
+from unstructured.logger import logger
+from unstructured.nlp.patterns import ENUMERATED_BULLETS_RE, UNICODE_BULLETS_RE
+
+if TYPE_CHECKING:
+    from unstructured_inference.inference.layout import PageLayout
+    from unstructured_inference.inference.layoutelement import LayoutElement
+
+
+def normalize_layout_element(
+    layout_element: LayoutElement | Element | dict[str, Any],
+    coordinate_system: Optional[CoordinateSystem] = None,
+    infer_list_items: bool = True,
+    source_format: Optional[str] = "html",
+) -> Element | list[Element]:
+    """Converts an unstructured_inference LayoutElement object to an unstructured Element."""
+
+    if isinstance(layout_element, Element) and source_format == "html":
+        return layout_element
+
+    # NOTE(alan): Won't the lines above ensure this never runs (PageBreak is a subclass of Element)?
+    if isinstance(layout_element, PageBreak):
+        return PageBreak(text="")
+
+    if not isinstance(layout_element, dict):
+        layout_dict = layout_element.to_dict()
+    else:
+        layout_dict = layout_element
+
+    text = layout_dict.get("text", "")
+    # Both `coordinates` and `coordinate_system` must be present
+    # in order to add coordinates metadata to the element.
+    coordinates = layout_dict.get("coordinates") if coordinate_system else None
+    element_type = layout_dict.get("type")
+    prob = layout_dict.get("prob")
+    aux_origin = layout_dict.get("source", None)
+    origin = None
+    if isinstance(layout_dict.get("is_extracted"), Enum):
+        is_extracted = layout_dict["is_extracted"].value
+    else:
+        is_extracted = None
+    if aux_origin:
+        origin = aux_origin.value
+    if prob and isinstance(prob, (int, str, float, numbers.Number)):
+        class_prob_metadata = ElementMetadata(detection_class_prob=float(prob))  # type: ignore
+    else:
+        class_prob_metadata = ElementMetadata()
+    class_prob_metadata.is_extracted = is_extracted
+    common_kwargs = {
+        "coordinates": coordinates,
+        "coordinate_system": coordinate_system,
+        "metadata": class_prob_metadata,
+        "detection_origin": origin,
+    }
+    if element_type == ElementType.LIST:
+        if infer_list_items:
+            return layout_list_to_list_items(
+                text,
+                **common_kwargs,
+            )
+        else:
+            return ListItem(
+                text=text,
+                **common_kwargs,
+            )
+
+    elif element_type in TYPE_TO_TEXT_ELEMENT_MAP:
+        assert isinstance(element_type, str)  # Added to resolve type-error
+        _element_class = TYPE_TO_TEXT_ELEMENT_MAP[element_type]
+        _element_class = _element_class(
+            text=text,
+            **common_kwargs,
+        )
+        if element_type == ElementType.HEADLINE:
+            _element_class.metadata.category_depth = 1
+        elif element_type == ElementType.SUB_HEADLINE:
+            _element_class.metadata.category_depth = 2
+        return _element_class
+    elif element_type in [
+        ElementType.CHECK_BOX_CHECKED,
+        ElementType.CHECK_BOX_UNCHECKED,
+        ElementType.RADIO_BUTTON_CHECKED,
+        ElementType.RADIO_BUTTON_UNCHECKED,
+        ElementType.CHECKED,
+        ElementType.UNCHECKED,
+    ]:
+        checked = element_type in [
+            ElementType.CHECK_BOX_CHECKED,
+            ElementType.RADIO_BUTTON_CHECKED,
+            ElementType.CHECKED,
+        ]
+        return CheckBox(
+            checked=checked,
+            **common_kwargs,
+        )
+    else:
+        return Text(
+            text=text,
+            **common_kwargs,
+        )
+
+
+def layout_list_to_list_items(
+    text: Optional[str],
+    coordinates: Optional[tuple[tuple[float, float], ...]],
+    coordinate_system: Optional[CoordinateSystem],
+    metadata: Optional[ElementMetadata],
+    detection_origin: Optional[str],
+) -> list[Element]:
+    """Converts a list LayoutElement to a list of ListItem elements."""
+    split_items = ENUMERATED_BULLETS_RE.split(text) if text else []
+    # NOTE(robinson) - this means there wasn't a match for the enumerated bullets
+    if len(split_items) == 1:
+        split_items = UNICODE_BULLETS_RE.split(text) if text else []
+
+    list_items: list[Element] = []
+    for text_segment in split_items:
+        if len(text_segment.strip()) > 0:
+            # Both `coordinates` and `coordinate_system` must be present
+            # in order to add coordinates metadata to the element.
+            item = ListItem(
+                text=text_segment.strip(),
+                coordinates=coordinates,
+                coordinate_system=coordinate_system,
+                metadata=metadata,
+                detection_origin=detection_origin,
+            )
+            list_items.append(item)
+
+    return list_items
+
+
+def add_element_metadata(
+    element: Element,
+    filename: Optional[str] = None,
+    filetype: Optional[str] = None,
+    page_number: Optional[int] = None,
+    url: Optional[str] = None,
+    text_as_html: Optional[str] = None,
+    coordinates: Optional[tuple[tuple[float, float], ...]] = None,
+    coordinate_system: Optional[CoordinateSystem] = None,
+    image_path: Optional[str] = None,
+    detection_origin: Optional[str] = None,
+    languages: Optional[list[str]] = None,
+    **kwargs: Any,
+) -> Element:
+    """Adds document metadata to the document element.
+
+    Document metadata includes information like the filename, source url, and page number.
+    """
+
+    coordinates_metadata = (
+        CoordinatesMetadata(
+            points=coordinates,
+            system=coordinate_system,
+        )
+        if coordinates is not None and coordinate_system is not None
+        else None
+    )
+    links = element.links if hasattr(element, "links") and len(element.links) > 0 else None
+    link_urls = [link.get("url") for link in links] if links else None
+    link_texts = [link.get("text") for link in links] if links else None
+    link_start_indexes = [link.get("start_index") for link in links] if links else None
+    emphasized_texts = (
+        element.emphasized_texts
+        if hasattr(element, "emphasized_texts") and len(element.emphasized_texts) > 0
+        else None
+    )
+    emphasized_text_contents = (
+        [emphasized_text.get("text") for emphasized_text in emphasized_texts]
+        if emphasized_texts
+        else None
+    )
+    emphasized_text_tags = (
+        [emphasized_text.get("tag") for emphasized_text in emphasized_texts]
+        if emphasized_texts
+        else None
+    )
+    depth = element.metadata.category_depth if element.metadata.category_depth else None
+
+    metadata = ElementMetadata(
+        coordinates=coordinates_metadata,
+        filename=filename,
+        filetype=filetype,
+        page_number=page_number,
+        url=url,
+        text_as_html=text_as_html,
+        link_urls=link_urls,
+        link_texts=link_texts,
+        link_start_indexes=link_start_indexes,
+        emphasized_text_contents=emphasized_text_contents,
+        emphasized_text_tags=emphasized_text_tags,
+        category_depth=depth,
+        image_path=image_path,
+        languages=languages,
+    )
+    element.metadata.update(metadata)
+    if detection_origin is not None:
+        element.metadata.detection_origin = detection_origin
+    return element
+
+
+def remove_element_metadata(layout_elements: list[Element]) -> list[Element]:
+    """Removes document metadata from the document element.
+
+    Document metadata includes information like the filename, source url, and page number.
+    """
+    elements: list[Element] = []
+    metadata = ElementMetadata()
+    for layout_element in layout_elements:
+        element = normalize_layout_element(layout_element)
+        if isinstance(element, list):
+            for _element in element:
+                _element.metadata = metadata
+            elements.extend(element)
+        else:
+            element.metadata = metadata
+            elements.append(element)
+    return elements
+
+
+def _is_soffice_running():
+    for proc in psutil.process_iter():
+        try:
+            if "soffice" in proc.name().lower():
+                return True
+        except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
+            pass
+    return False
+
+
+def convert_office_doc(
+    input_filename: str,
+    output_directory: str,
+    target_format: str = "docx",
+    target_filter: Optional[str] = None,
+    wait_for_soffice_ready_time_out: int = 10,
+):
+    """Converts a .doc/.ppt file to a .docx/.pptx file using the libreoffice CLI.
+
+    Parameters
+    ----------
+    input_filename: str
+        The name of the .doc file to convert to .docx
+    output_directory: str
+        The output directory for the convert .docx file
+    target_format: str
+        The desired output format
+    target_filter: str
+        The output filter name to use when converting. See references below
+        for details.
+    wait_for_soffice_ready_time_out: int
+        The max wait time in seconds for soffice to become available to run
+
+    References
+    ----------
+    https://stackoverflow.com/questions/52277264/convert-doc-to-docx-using-soffice-not-working
+    https://git.libreoffice.org/core/+/refs/heads/master/filter/source/config/fragments/filters
+
+    """
+    if target_filter is not None:
+        target_format = f"{target_format}:{target_filter}"
+    # NOTE(robinson) - In the future can also include win32com client as a fallback for windows
+    # users who do not have LibreOffice installed
+    # ref: https://stackoverflow.com/questions/38468442/
+    #       multiple-doc-to-docx-file-conversion-using-python
+    command = [
+        "soffice",
+        "--headless",
+        "--convert-to",
+        target_format,
+        "--outdir",
+        output_directory,
+        input_filename,
+    ]
+    try:
+        # only one soffice process can be ran
+        wait_time = 0
+        sleep_time = 0.1
+        output = subprocess.run(command, capture_output=True)
+        message = output.stdout.decode().strip()
+        # we can't rely on returncode unfortunately because on macOS it would return 0 even when the
+        # command failed to run; instead we have to rely on the stdout being empty as a sign of the
+        # process failed
+        while (wait_time < wait_for_soffice_ready_time_out) and (message == ""):
+            wait_time += sleep_time
+            if _is_soffice_running():
+                sleep(sleep_time)
+            else:
+                output = subprocess.run(command, capture_output=True)
+                message = output.stdout.decode().strip()
+    except FileNotFoundError:
+        raise FileNotFoundError(
+            """soffice command was not found. Please install libreoffice
+on your system and try again.
+
+- Install instructions: https://www.libreoffice.org/get-help/install-howto/
+- Mac: https://formulae.brew.sh/cask/libreoffice
+- Debian: https://wiki.debian.org/LibreOffice""",
+        )
+
+    logger.info(message)
+    if output.returncode != 0 or message == "":
+        logger.error(
+            "soffice failed to convert to format %s with code %i", target_format, output.returncode
+        )
+        logger.error(output.stderr.decode().strip())
+
+
+def exactly_one(**kwargs: Any) -> None:
+    """
+    Verify arguments; exactly one of all keyword arguments must not be None.
+
+    Example:
+        >>> exactly_one(filename=filename, file=file, text=text, url=url)
+    """
+    if sum([(arg is not None and arg != "") for arg in kwargs.values()]) != 1:
+        names = list(kwargs.keys())
+        if len(names) > 1:
+            message = f"Exactly one of {', '.join(names[:-1])} and {names[-1]} must be specified."
+        else:
+            message = f"{names[0]} must be specified."
+        raise ValueError(message)
+
+
+_T = TypeVar("_T")
+
+
+def spooled_to_bytes_io_if_needed(file: _T | SpooledTemporaryFile[bytes]) -> _T | BytesIO:
+    """Convert `file` to `BytesIO` when it is a `SpooledTemporaryFile`.
+
+    Note that `file` does not need to be IO[bytes]. It can be `None` or `bytes` and this function
+    will not complain.
+
+    In Python <3.11, `SpooledTemporaryFile` does not implement `.readable()` or `.seekable()` which
+    triggers an exception when the file is loaded by certain packages. In particular, the stdlib
+    `zipfile.Zipfile` raises on opening a `SpooledTemporaryFile` as does `Pandas.read_csv()`.
+    """
+    if isinstance(file, SpooledTemporaryFile):
+        file.seek(0)
+        return BytesIO(cast(bytes, file.read()))
+
+    # -- return `file` unchanged otherwise --
+    return file
+
+
+def convert_to_bytes(file: bytes | IO[bytes]) -> bytes:
+    """Extract the bytes from `file` without preventing it from being read again later.
+
+    As a convenience to simplify client code, also returns `file` unchanged if it is already bytes.
+    """
+    if isinstance(file, bytes):
+        return file
+
+    if isinstance(file, SpooledTemporaryFile):
+        file.seek(0)
+        f_bytes = file.read()
+        file.seek(0)
+        return f_bytes
+
+    if isinstance(file, BytesIO):
+        return file.getvalue()
+
+    if isinstance(file, (TextIOWrapper, BufferedReader)):
+        with open(file.name, "rb") as f:
+            return f.read()
+
+    raise ValueError("Invalid file-like object type")
+
+
+def contains_emoji(s: str) -> bool:
+    """
+    Check if the input string contains any emoji characters.
+
+    Parameters:
+    - s (str): The input string to check.
+
+    Returns:
+    - bool: True if the string contains any emoji, False otherwise.
+    """
+
+    return bool(emoji.emoji_count(s))
+
+
+def get_page_image_metadata(page: PageLayout) -> dict[str, Any]:
+    """Retrieve image metadata and coordinate system from a page."""
+
+    image = getattr(page, "image", None)
+    image_metadata = getattr(page, "image_metadata", None)
+
+    if image:
+        image_format = image.format
+        image_width = image.width
+        image_height = image.height
+    elif image_metadata:
+        image_format = image_metadata.get("format")
+        image_width = image_metadata.get("width")
+        image_height = image_metadata.get("height")
+    else:
+        image_format = None
+        image_width = None
+        image_height = None
+
+    return {
+        "format": image_format,
+        "width": image_width,
+        "height": image_height,
+    }
+
+
+def ocr_data_to_elements(
+    ocr_data: list["LayoutElement"],
+    image_size: tuple[int | float, int | float],
+    common_metadata: Optional[ElementMetadata] = None,
+    infer_list_items: bool = True,
+    source_format: Optional[str] = None,
+) -> list[Element]:
+    """Convert OCR layout data into `unstructured` elements with associated metadata."""
+
+    image_width, image_height = image_size
+    coordinate_system = PixelSpace(width=image_width, height=image_height)
+    elements: list[Element] = []
+    for layout_element in ocr_data:
+        element = normalize_layout_element(
+            layout_element,
+            coordinate_system=coordinate_system,
+            infer_list_items=infer_list_items,
+            source_format=source_format if source_format else "html",
+        )
+
+        if common_metadata:
+            element.metadata.update(common_metadata)
+
+        elements.append(element)
+
+    return elements

unstructured/partition/common/lang.py

@@ -0,0 +1,583 @@
+from __future__ import annotations
+
+import re
+from functools import lru_cache
+from typing import Callable, Iterable, Iterator, Optional
+
+import iso639  # pyright: ignore[reportMissingTypeStubs]
+from langdetect import (  # pyright: ignore[reportMissingTypeStubs]
+    DetectorFactory,
+    detect_langs,  # pyright: ignore[reportUnknownVariableType]
+    lang_detect_exception,
+)
+
+from unstructured.documents.elements import Element
+from unstructured.logger import logger
+from unstructured.partition.utils.constants import (
+    TESSERACT_LANGUAGES_AND_CODES,
+    TESSERACT_LANGUAGES_SPLITTER,
+)
+
+_ASCII_RE = re.compile(r"^[\x00-\x7F]+$")
+
+# pytesseract.get_languages(config="") only shows user installed language packs,
+# so manually include the list of all currently supported Tesseract languages
+PYTESSERACT_LANG_CODES = [
+    "afr",
+    "amh",
+    "ara",
+    "asm",
+    "aze",
+    "aze_cyrl",
+    "bel",
+    "ben",
+    "bod",
+    "bos",
+    "bre",
+    "bul",
+    "cat",
+    "ceb",
+    "ces",
+    "chi_sim",
+    "chi_sim_vert",
+    "chi_tra",
+    "chi_tra_vert",
+    "chr",
+    "cos",
+    "cym",
+    "dan",
+    "deu",
+    "div",
+    "dzo",
+    "ell",
+    "eng",
+    "enm",
+    "epo",
+    "equ",
+    "est",
+    "eus",
+    "fao",
+    "fas",
+    "fil",
+    "fin",
+    "fra",
+    "frk",
+    "frm",
+    "fry",
+    "gla",
+    "gle",
+    "glg",
+    "grc",
+    "guj",
+    "hat",
+    "heb",
+    "hin",
+    "hrv",
+    "hun",
+    "hye",
+    "iku",
+    "ind",
+    "isl",
+    "ita",
+    "ita_old",
+    "jav",
+    "jpn",
+    "jpn_vert",
+    "kan",
+    "kat",
+    "kat_old",
+    "kaz",
+    "khm",
+    "kir",
+    "kmr",
+    "kor",
+    "kor_vert",
+    "lao",
+    "lat",
+    "lav",
+    "lit",
+    "ltz",
+    "mal",
+    "mar",
+    "mkd",
+    "mlt",
+    "mon",
+    "mri",
+    "msa",
+    "mya",
+    "nep",
+    "nld",
+    "nor",
+    "oci",
+    "ori",
+    "osd",
+    "pan",
+    "pol",
+    "por",
+    "pus",
+    "que",
+    "ron",
+    "rus",
+    "san",
+    "sin",
+    "slk",
+    "slv",
+    "snd",
+    "snum",
+    "spa",
+    "spa_old",
+    "sqi",
+    "srp",
+    "srp_latn",
+    "sun",
+    "swa",
+    "swe",
+    "syr",
+    "tam",
+    "tat",
+    "tel",
+    "tgk",
+    "tha",
+    "tir",
+    "ton",
+    "tur",
+    "uig",
+    "ukr",
+    "urd",
+    "uzb",
+    "uzb_cyrl",
+    "vie",
+    "yid",
+    "yor",
+]
+
+PYTESSERACT_TO_PADDLE_LANG_CODE_MAP = {
+    "afr": "af",  # Afrikaans
+    "ara": "ar",  # Arabic
+    "aze": "az",  # Azerbaijani
+    "bel": "be",  # Belarusian
+    "bos": "bs",  # Bosnian
+    "bul": "bg",  # Bulgarian
+    "ces": "cs",  # Czech
+    "chi_sim": "ch",  # Simplified Chinese
+    "chi_tra": "chinese_cht",  # Traditional Chinese
+    "cym": "cy",  # Welsh
+    "dan": "da",  # Danish
+    "deu": "german",  # German
+    "eng": "en",  # English
+    "est": "et",  # Estonian
+    "fas": "fa",  # Persian
+    "fra": "fr",  # French
+    "gle": "ga",  # Irish
+    "hin": "hi",  # Hindi
+    "hrv": "hr",  # Croatian
+    "hun": "hu",  # Hungarian
+    "ind": "id",  # Indonesian
+    "isl": "is",  # Icelandic
+    "ita": "it",  # Italian
+    "jpn": "japan",  # Japanese
+    "kor": "korean",  # Korean
+    "kmr": "ku",  # Kurdish
+    "lat": "rs_latin",  # Latin
+    "lav": "lv",  # Latvian
+    "lit": "lt",  # Lithuanian
+    "mar": "mr",  # Marathi
+    "mlt": "mt",  # Maltese
+    "msa": "ms",  # Malay
+    "nep": "ne",  # Nepali
+    "nld": "nl",  # Dutch
+    "nor": "no",  # Norwegian
+    "pol": "pl",  # Polish
+    "por": "pt",  # Portuguese
+    "ron": "ro",  # Romanian
+    "rus": "ru",  # Russian
+    "slk": "sk",  # Slovak
+    "slv": "sl",  # Slovenian
+    "spa": "es",  # Spanish
+    "sqi": "sq",  # Albanian
+    "srp": "rs_cyrillic",  # Serbian
+    "swa": "sw",  # Swahili
+    "swe": "sv",  # Swedish
+    "tam": "ta",  # Tamil
+    "tel": "te",  # Telugu
+    "tur": "tr",  # Turkish
+    "uig": "ug",  # Uyghur
+    "ukr": "uk",  # Ukrainian
+    "urd": "ur",  # Urdu
+    "uzb": "uz",  # Uzbek
+    "vie": "vi",  # Vietnamese
+}
+
+
+def prepare_languages_for_tesseract(languages: Optional[list[str]] = ["eng"]) -> str:
+    """
+    Entry point: convert languages (list of strings) into tesseract ocr langcode format (uses +)
+    """
+    if languages is None:
+        raise ValueError("`languages` can not be `None`")
+    converted_languages = [
+        lang_code
+        for lang_code in (
+            _convert_language_code_to_pytesseract_lang_code(lang) for lang in languages
+        )
+        if lang_code
+    ]
+    # Remove duplicates from the list but keep the original order
+    converted_languages = list(dict.fromkeys(converted_languages))
+    if len(converted_languages) == 0:
+        logger.warning(
+            "Failed to find any valid standard language code from "
+            f"languages: {languages}, proceed with `eng` instead.",
+        )
+        return "eng"
+
+    return TESSERACT_LANGUAGES_SPLITTER.join(converted_languages)
+
+
+def tesseract_to_paddle_language(tesseract_language: str) -> str:
+    """
+    Convert TesseractOCR language code to PaddleOCR language code.
+
+    :param tesseract_language: str, language code used in TesseractOCR
+    :return: str, corresponding language code for PaddleOCR or None if not found
+    """
+
+    lang = PYTESSERACT_TO_PADDLE_LANG_CODE_MAP.get(tesseract_language.lower())
+    if not lang:
+        logger.warning(
+            f"{tesseract_language} is not a language code supported by PaddleOCR, "
+            f"proceeding with `en` instead."
+        )
+        return "en"
+
+    return lang
+
+
+def check_language_args(
+    languages: list[str], ocr_languages: str | list[str] | None
+) -> list[str] | None:
+    """Handle users defining both `ocr_languages` and `languages`.
+
+    Give preference to `languages` and convert `ocr_languages` if needed, but default to `None`.
+
+    `ocr_languages` is only a parameter for `auto.partition`, `partition_image`, & `partition_pdf`.
+    `ocr_languages` should not be defined as 'auto' since 'auto' is intended for language detection
+    which is not supported by `partition_image` or `partition_pdf`.
+    """
+    # --- Clean and update defaults
+    if ocr_languages:
+        ocr_languages = _clean_ocr_languages_arg(ocr_languages)
+        logger.warning(
+            "The ocr_languages kwarg will be deprecated in a future version of unstructured. "
+            "Please use languages instead.",
+        )
+    assert ocr_languages is None or isinstance(ocr_languages, str)
+
+    if ocr_languages and "auto" in ocr_languages:
+        raise ValueError(
+            "`ocr_languages` is deprecated but was used to extract text from pdfs and images."
+            " The 'auto' argument is only for language *detection* when it is assigned"
+            " to `languages` and partitioning documents other than pdfs or images."
+            " Language detection is not currently supported in pdfs or images."
+        )
+
+    if not isinstance(languages, list):  # pyright: ignore[reportUnnecessaryIsInstance]
+        raise TypeError(
+            "The language parameter must be a list of language codes as strings, ex. ['eng']",
+        )
+
+    # --- If `languages` is a null/default value and `ocr_languages` is defined, use `ocr_languages`
+    if ocr_languages and (languages == ["auto"] or languages == [""] or not languages):
+        languages = ocr_languages.split(TESSERACT_LANGUAGES_SPLITTER)
+        logger.warning(
+            "Only one of languages and ocr_languages should be specified. "
+            "languages is preferred. ocr_languages is marked for deprecation.",
+        )
+
+    # --- Clean `languages`
+    # If "auto" is included in the list of inputs, language detection will be triggered downstream.
+    # The rest of the inputted languages are ignored.
+    if languages:
+        if "auto" not in languages:
+            for i, lang in enumerate(languages):
+                languages[i] = TESSERACT_LANGUAGES_AND_CODES.get(lang.lower(), lang)
+
+            str_languages = _clean_ocr_languages_arg(languages)
+            if not str_languages:
+                return None
+            languages = str_languages.split(TESSERACT_LANGUAGES_SPLITTER)
+        # else, remove the extraneous languages.
+        # NOTE (jennings): "auto" should only be used for partitioners OTHER THAN `_pdf` or `_image`
+        else:
+            # define as 'auto' for language detection when partitioning non-pdfs or -images
+            languages = ["auto"]
+        return languages
+
+    return None
+
+
+def convert_old_ocr_languages_to_languages(ocr_languages: str) -> list[str]:
+    """
+    Convert ocr_languages parameter to list of langcode strings.
+    Assumption: ocr_languages is in tesseract plus sign format
+    """
+
+    return ocr_languages.split(TESSERACT_LANGUAGES_SPLITTER)
+
+
+def _convert_language_code_to_pytesseract_lang_code(lang: str) -> str:
+    """
+    Convert a single language code to its tesseract formatted and recognized
+    langcode(s), if supported.
+    """
+    # if language is already tesseract langcode, return it immediately
+    # this will catch the tesseract special cases equ and osd
+    # NOTE(shreya): this may catch some cases of choosing between tesseract code variants for a lang
+    if lang in PYTESSERACT_LANG_CODES:
+        return lang
+
+    lang_iso639 = _get_iso639_language_object(lang)
+
+    # tesseract uses 3 digit codes (639-3, 639-2b, etc) as prefixes, with suffixes for orthography
+    # use first 3 letters of tesseract codes for matching to standard codes
+    pytesseract_langs_3 = {lang[:3] for lang in PYTESSERACT_LANG_CODES}
+
+    if lang_iso639:
+        # try to match ISO 639-3 code
+        if lang_iso639.part3 in pytesseract_langs_3:
+            matched_langcodes = _get_all_tesseract_langcodes_with_prefix(lang_iso639.part3)
+            return TESSERACT_LANGUAGES_SPLITTER.join(matched_langcodes)
+
+        # try to match ISO 639-2b
+        elif lang_iso639.part2b in pytesseract_langs_3:
+            matched_langcodes = _get_all_tesseract_langcodes_with_prefix(lang_iso639.part2b)
+            return TESSERACT_LANGUAGES_SPLITTER.join(matched_langcodes)
+
+        # try to match ISO 639-2t
+        elif lang_iso639.part2t in pytesseract_langs_3:
+            matched_langcodes = _get_all_tesseract_langcodes_with_prefix(lang_iso639.part2t)
+            return TESSERACT_LANGUAGES_SPLITTER.join(matched_langcodes)
+
+        else:
+            logger.warning(f"{lang} is not a language supported by Tesseract.")
+            return ""
+    logger.warning(f"{lang} is not a language supported by Tesseract.")
+    return ""
+
+
+def _get_iso639_language_object(lang: str) -> Optional[iso639.Language]:
+    language = _cached_iso639_language_match(lang)
+    if language is not None:
+        return language
+    logger.warning(f"{lang} is not a valid standard language code.")
+    return None
+
+
+def _get_all_tesseract_langcodes_with_prefix(prefix: str) -> list[str]:
+    """
+    Get all matching tesseract langcodes with this prefix (may be one or multiple variants).
+    """
+    return [langcode for langcode in PYTESSERACT_LANG_CODES if langcode.startswith(prefix)]
+
+
+def _validate_fallback_languages(
+    value: Optional[list[str]],
+) -> Optional[list[str]]:
+    """Validate and normalize language_fallback return value to ISO 639-3 codes.
+
+    Returns None for None, non-list, or when no valid codes remain (invalid entries
+    are logged and skipped).
+    """
+    if value is None:
+        return None
+    if not isinstance(value, list):
+        logger.warning(
+            f"language_fallback must return None or a list of strings, got {type(value).__name__}."
+        )
+        return None
+    validated: list[str] = []
+    for item in value:
+        if not isinstance(item, str) or not item.strip():
+            continue
+        lang = item.strip()
+        if lang == "zho":
+            validated.append("zho")
+        else:
+            language = _get_iso639_language_object(lang[:3])
+            if language is not None:
+                validated.append(language.part3)
+    return validated if validated else None
+
+
+def detect_languages(
+    text: str,
+    languages: Optional[list[str]] = None,
+    language_fallback: Optional[Callable[[str], Optional[list[str]]]] = None,
+) -> Optional[list[str]]:
+    """
+    Detects the list of languages present in the text (in the default "auto" mode),
+    or formats and passes through the user inputted document languages if provided.
+
+    For short ASCII text (fewer than 5 words), language detection is unreliable. By
+    default such text is assigned English (["eng"]). Use ``language_fallback`` to
+    override: pass a callable that takes the text and returns a list of ISO 639-3
+    codes or None. Return None to leave language unspecified. The caller is
+    responsible for returning valid ISO 639-3 codes (e.g. "eng", "fra"); invalid
+    entries are filtered out and a warning is logged; if none remain, this function
+    returns None.
+    """
+    if languages is None:
+        languages = ["auto"]
+    if not isinstance(languages, list):
+        raise TypeError(
+            'The language parameter must be a list of language codes as strings, ex. ["eng"]',
+        )
+
+    # Skip language detection for partitioners that use other partitioners.
+    # For example, partition_msg relies on partition_html and partition_text, but the metadata
+    # gets overwritten after elements have been returned by _html and _text,
+    # so `languages` would be detected twice.
+    # Also return None if there is no text.
+    if languages[0] == "" or text.strip() == "":
+        return None
+
+    # If text contains special characters (like ñ, å, or Korean/Mandarin/etc.) it will NOT default
+    # to English. It will default to English if text is only ascii characters and is short.
+    if _ASCII_RE.match(text) and len(text.split()) < 5:
+        if language_fallback is not None:
+            return _validate_fallback_languages(language_fallback(text))
+        logger.debug(f'short text: "{text}". Defaulting to English.')
+        return ["eng"]
+
+    # set seed for deterministic langdetect outputs
+    DetectorFactory.seed = 0
+
+    doc_languages: list[str] = []
+
+    # user inputted languages:
+    # if "auto" is included in the list of inputs, language detection will be triggered
+    # and the rest of the inputted languages will be ignored
+    if languages and "auto" not in languages:
+        for lang in languages:
+            str_lang = TESSERACT_LANGUAGES_AND_CODES.get(lang.lower(), lang)
+            language = _get_iso639_language_object(str_lang[:3])
+            if language:
+                doc_languages.append(language.part3)
+
+    # language detection:
+    else:
+        # warn if any values other than "auto" were provided
+        if len(languages) > 1:
+            logger.warning(
+                f'Since "auto" is present in the input languages provided ({languages}), '
+                "the language will be auto detected and the rest of the inputted "
+                "languages will be ignored.",
+            )
+
+        try:
+            langdetect_result = detect_langs(text)
+        except lang_detect_exception.LangDetectException as e:
+            logger.warning(e)
+            return None  # None as default
+
+        langdetect_langs: list[str] = []
+
+        # NOTE(robinson) - Chinese gets detected with codes zh-cn, zh-tw, zh-hk for various
+        # Chinese variants. We normalizes these because there is a single model for Chinese
+        # machine translation
+        # TODO(shreya): decide how to maintain nonstandard chinese script information
+        for langobj in langdetect_result:
+            lang_val = str(langobj.lang)
+            if lang_val.startswith("zh"):  # pyright: ignore
+                langdetect_langs.append("zho")
+            else:
+                language = _get_iso639_language_object(lang_val[:3])  # pyright: ignore
+                if language:
+                    langdetect_langs.append(language.part3)
+
+        # remove duplicate chinese (if exists) without modifying order
+        seen = set(doc_languages)
+        for lang in langdetect_langs:
+            if lang not in seen:
+                doc_languages.append(lang)
+                seen.add(lang)
+
+    return doc_languages
+
+
+def apply_lang_metadata(
+    elements: Iterable[Element],
+    languages: Optional[list[str]],
+    detect_language_per_element: bool = False,
+    language_fallback: Optional[Callable[[str], Optional[list[str]]]] = None,
+) -> Iterator[Element]:
+    """Detect language and apply it to metadata.languages for each element in `elements`.
+    If languages is None, default to auto detection.
+    If languages is an empty string, skip.
+    language_fallback is used for short text when detection is unreliable; see detect_languages."""
+    # -- Note this function has a stream interface, but reads the full `elements` stream into memory
+    # -- before emitting the first updated element as output.
+
+    # The auto `partition` function uses `None` as a default because the default for
+    # `partition_pdf` and `partition_img` conflict with the other partitioners that use ["auto"]
+    if languages is None:
+        languages = ["auto"]
+
+    # Skip language detection for partitioners that use other partitioners.
+    # For example, partition_msg relies on partition_html and partition_text, but the metadata
+    # gets overwritten after elements have been returned by _html and _text,
+    # so `languages` would be detected twice.
+    if languages == [""]:
+        yield from elements
+        return
+
+    # Convert elements to a list to get the text, detect the language, and add it to the elements
+    if not isinstance(elements, list):
+        elements = list(elements)
+
+    def detect(text: str) -> Optional[list[str]]:
+        return detect_languages(text=text, languages=languages, language_fallback=language_fallback)
+
+    full_text = " ".join(str(e.text) for e in elements if hasattr(e, "text") and e.text)
+    detected_languages = detect(full_text)
+    if (
+        detected_languages is not None
+        and len(detected_languages) == 1
+        and detect_language_per_element is False
+    ):
+        # -- apply detected language to each element's metadata --
+        for e in elements:
+            e.metadata.languages = detected_languages
+            yield e
+    else:
+        for e in elements:
+            if hasattr(e, "text"):
+                text_value = str(e.text) if e.text is not None else ""
+                e.metadata.languages = detect(text_value)
+                yield e
+            else:
+                yield e
+
+
+def _clean_ocr_languages_arg(ocr_languages: list[str] | str) -> str:
+    """Fix common incorrect definitions for ocr_languages:
+    defining it as a list, adding extra quotation marks, adding brackets.
+    Returns a single string of ocr_languages"""
+    # extract from list
+    if isinstance(ocr_languages, list):
+        ocr_languages = "+".join(ocr_languages)
+
+    # remove extra quotations
+    ocr_languages = re.sub(r"[\"']", "", ocr_languages)
+    # remove brackets
+    ocr_languages = re.sub(r"[\[\]]", "", ocr_languages)
+
+    return ocr_languages
+
+
+@lru_cache(maxsize=256)
+def _cached_iso639_language_match(lang: str) -> Optional[iso639.Language]:
+    try:
+        return iso639.Language.match(lang.lower())  # pyright: ignore[reportUnknownMemberType]
+    except iso639.LanguageNotFoundError:
+        return None

unstructured/partition/common/metadata.py

@@ -0,0 +1,335 @@
+"""Helpers used across multiple partitioners to compute metadata."""
+
+from __future__ import annotations
+
+import copy
+import datetime as dt
+import functools
+import os
+from typing import Any, Callable, Iterator, Sequence
+
+from typing_extensions import ParamSpec
+
+from unstructured.documents.elements import Element, ElementMetadata, ListItem, Title
+from unstructured.file_utils.model import FileType
+from unstructured.partition.common.lang import apply_lang_metadata
+from unstructured.utils import get_call_args_applying_defaults
+
+_P = ParamSpec("_P")
+
+
+def get_last_modified_date(filename: str) -> str | None:
+    """Modification time of file at path `filename`, if it exists.
+
+    Returns `None` when `filename` is not a path to a file on the local filesystem.
+
+    Otherwise returns date and time in ISO 8601 string format (YYYY-MM-DDTHH:MM:SS) like
+    "2024-03-05T17:02:53".
+    """
+    if not os.path.isfile(filename):
+        return None
+
+    modify_date = dt.datetime.fromtimestamp(os.path.getmtime(filename))
+    return modify_date.strftime("%Y-%m-%dT%H:%M:%S%z")
+
+
+HIERARCHY_RULE_SET = {
+    "Title": [
+        "Text",
+        "UncategorizedText",
+        "NarrativeText",
+        "ListItem",
+        "BulletedText",
+        "Table",
+        "FigureCaption",
+        "CheckBox",
+        "Table",
+    ],
+    "Header": [
+        "Title",
+        "Text",
+        "UncategorizedText",
+        "NarrativeText",
+        "ListItem",
+        "BulletedText",
+        "Table",
+        "FigureCaption",
+        "CheckBox",
+        "Table",
+    ],
+}
+
+
+# Canonical HTML heading levels -> zero-indexed category_depth. The HTML spec
+# defines exactly six heading levels (there is no h7), so this closed mapping is
+# the single source of truth for both the depth value and the heading-tag set
+# (HEADING_TAGS is derived from it, not a second copy).
+_HEADING_DEPTH = {"h1": 0, "h2": 1, "h3": 2, "h4": 3, "h5": 4, "h6": 5}
+HEADING_TAGS = tuple(_HEADING_DEPTH)
+
+
+def category_depth_from_html_tag(
+    ElementCls: type[Element], tag: str | None, list_ancestor_count: int = 0
+) -> int | None:
+    """Compute `category_depth` from an element's HTML heading level (not DOM-nesting depth).
+
+    This is the canonical mapping used by both the v1 HTML parser and the v2 (ontology) HTML
+    converter so the two paths agree on what `category_depth` means:
+
+    - `Title` (which includes ontology Title/Subtitle/Heading, i.e. ``<h1>``-``<h6>``): the heading
+      level, zero-indexed -- ``h1`` -> 0, ``h2`` -> 1, ... ``h6`` -> 5. A `Title` whose tag is not a
+      heading (e.g. a styled paragraph) is treated as a top-level heading (0).
+    - `ListItem`: the number of enclosing list containers (``ol``/``ul``/``dl``), passed in by the
+      caller (the v1 HTML parser, which computes it from list nesting). The v2 converter serializes
+      a whole ``ol``/``ul``/``dl`` as one element and never emits a standalone ``ListItem``, so it
+      does not use this.
+    - Everything else: ``None`` (no meaningful depth).
+
+    `tag` is the element's HTML tag name (e.g. ``"h2"``); it may be ``None`` for derived elements.
+    """
+    if ElementCls is ListItem:
+        return list_ancestor_count
+
+    if ElementCls is Title:
+        return _HEADING_DEPTH.get(tag, 0)
+
+    return None
+
+
+def set_element_hierarchy(
+    elements: Sequence[Element], ruleset: dict[str, list[str]] = HIERARCHY_RULE_SET
+) -> list[Element]:
+    """Sets `.metadata.parent_id` for each element it applies to.
+
+    `parent_id` assignment is based on the element's category and depth. The importance of an
+    element's category is determined by a rule set. The rule set trumps category_depth. That is,
+    category_depth is only relevant when elements are of the same category.
+    """
+    stack: list[Element] = []
+    for element in elements:
+        if element.metadata.parent_id is not None:
+            continue
+        parent_id = None
+        element_category = getattr(element, "category", None)
+        element_category_depth = getattr(element.metadata, "category_depth", 0) or 0
+
+        # -- skip elements without a category --
+        if not element_category:
+            continue
+
+        while stack:
+            top_element: Element = stack[-1]
+            top_element_category = getattr(top_element, "category")
+            top_element_category_depth = (
+                getattr(
+                    top_element.metadata,
+                    "category_depth",
+                    0,
+                )
+                or 0
+            )
+
+            if (
+                top_element_category == element_category
+                and top_element_category_depth < element_category_depth
+            ) or (
+                top_element_category != element_category
+                and element_category in ruleset.get(top_element_category, [])
+            ):
+                parent_id = top_element.id
+                break
+
+            stack.pop()
+
+        element.metadata.parent_id = parent_id
+        stack.append(element)
+
+    return list(elements)
+
+
+# ================================================================================================
+# METADATA POST-PARTITIONING PROCESSING DECORATOR
+# ================================================================================================
+
+
+def apply_metadata(
+    file_type: FileType | None = None,
+) -> Callable[[Callable[_P, list[Element]]], Callable[_P, list[Element]]]:
+    """Post-process element-metadata for this document.
+
+    This decorator adds a post-processing step to a partitioner, primarily to apply metadata that
+    is common to all partitioners. It assumes the following responsibilities:
+
+      - Hash element-ids. Computes and applies SHA1 hash element.id when `unique_element_ids`
+        argument is False.
+
+      - Element Hierarchy. Computes and applies `parent_id` metadata based on `category_depth`
+        etc. added by partitioner.
+
+      - Language metadata. Computes and applies `language` metadata based on a language detection
+        model.
+
+      - Apply `filetype` (MIME-type) metadata. There are three cases; first one in this order that
+        applies is used:
+
+          - `metadata_file_type` argument is present in call, use that.
+          - `file_type` decorator argument is populated, use that.
+          - `file_type` decorator argument is omitted or None, don't apply `.metadata.filetype`
+            (assume the partitioner will do that for itself, like `partition_image()`.
+
+      - Replace `filename` with `metadata_filename` when present.
+
+      - Replace `last_modified` with `metadata_last_modified` when present.
+
+      - Apply `url` metadata when present.
+    """
+
+    def decorator(func: Callable[_P, list[Element]]) -> Callable[_P, list[Element]]:
+        """The decorator function itself.
+
+        This function is returned by the `apply_metadata()` function and is the actual decorator.
+        Think of `apply_metadata()` as a factory function that configures this decorator, in
+        particular by setting its `file_type` value.
+        """
+
+        @functools.wraps(func)
+        def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> list[Element]:
+            elements = func(*args, **kwargs)
+            call_args = get_call_args_applying_defaults(func, *args, **kwargs)
+
+            # ------------------------------------------------------------------------------------
+            # unique-ify elements
+            # ------------------------------------------------------------------------------------
+            # Do this first to ensure all following operations behave as expected. It's easy for a
+            # partitioner to re-use an element or metadata instance when its values are common to
+            # multiple elements. This can lead to very hard-to diagnose bugs downstream when
+            # mutating one element unexpectedly also mutates others (because they are the same
+            # instance).
+            # ------------------------------------------------------------------------------------
+
+            elements = _uniqueify_elements_and_metadata(elements)
+
+            # ------------------------------------------------------------------------------------
+            # apply metadata - do this first because it affects the hash computation.
+            # ------------------------------------------------------------------------------------
+
+            # -- `language` - auto-detect language (e.g. eng, spa) --
+            languages = call_args.get("languages")
+            detect_language_per_element = call_args.get("detect_language_per_element", False)
+            language_fallback = call_args.get("language_fallback")
+            elements = list(
+                apply_lang_metadata(
+                    elements=elements,
+                    languages=languages,
+                    detect_language_per_element=detect_language_per_element,
+                    language_fallback=language_fallback,
+                )
+            )
+
+            # == apply filetype, filename, last_modified, and url metadata ===================
+            metadata_kwargs: dict[str, Any] = {}
+
+            # -- `filetype` (MIME-type) metadata --
+            metadata_file_type = call_args.get("metadata_file_type") or file_type
+            if metadata_file_type is not None:
+                metadata_kwargs["filetype"] = metadata_file_type.mime_type
+
+            # -- `filename` metadata - override with metadata_filename when it's present --
+            filename = call_args.get("metadata_filename") or call_args.get("filename")
+            if filename:
+                metadata_kwargs["filename"] = filename
+
+            # -- `last_modified` metadata - override with metadata_last_modified when present --
+            metadata_last_modified = call_args.get("metadata_last_modified")
+            if metadata_last_modified:
+                metadata_kwargs["last_modified"] = metadata_last_modified
+
+            # -- `url` metadata - record url when present --
+            url = call_args.get("url")
+            if url:
+                metadata_kwargs["url"] = url
+
+            # -- update element.metadata in single pass --
+            for element in elements:
+                # NOTE(robinson) - Attached files have already run through this logic in their own
+                # partitioning function
+                if element.metadata.attached_to_filename:
+                    continue
+                element.metadata.update(ElementMetadata(**metadata_kwargs))
+
+            # ------------------------------------------------------------------------------------
+            # compute hash ids (when so requestsd)
+            # ------------------------------------------------------------------------------------
+
+            # -- Compute and apply hash-ids if the user does not want UUIDs. Note this mutates the
+            # -- elements themselves, not their metadata.
+            unique_element_ids: bool = call_args.get("unique_element_ids", False)
+            if unique_element_ids is False:
+                elements = _assign_hash_ids(elements)
+
+            # ------------------------------------------------------------------------------------
+            # assign parent-id - do this after hash computation so parent-id is stable.
+            # ------------------------------------------------------------------------------------
+
+            # -- `parent_id` - process category-level etc. to assign parent-id --
+            elements = set_element_hierarchy(elements)
+
+            return elements
+
+        return wrapper
+
+    return decorator
+
+
+def _assign_hash_ids(elements: list[Element]) -> list[Element]:
+    """Converts `.id` of each element from UUID to hash and remaps `parent_id` accordingly.
+
+    The hash is based on the `.text` of the element, but also on its page-number and sequence number
+    on that page. This provides for deterministic results even when the document is split into one
+    or more fragments for parallel processing.
+
+    After hashing, any `element.metadata.parent_id` that references a known original UUID is
+    updated to the corresponding new hash ID. Parent IDs that do not appear in the mapping (e.g.
+    because the parent element was filtered out before hashing, or the ID was set manually to an
+    external value) are left unchanged.
+    """
+    # -- generate sequence number for each element on a page --
+    page_seq_counts = {}
+    id_mapping = {}
+    for element in elements:
+        page_number = element.metadata.page_number
+        seq_on_page_counter = page_seq_counts.get(page_number, 0)
+        original_id = element.id
+        element.id_to_hash(seq_on_page_counter)
+        id_mapping[original_id] = element.id
+        page_seq_counts[page_number] = seq_on_page_counter + 1
+
+    for element in elements:
+        if element.metadata.parent_id is not None and element.metadata.parent_id in id_mapping:
+            element.metadata.parent_id = id_mapping[element.metadata.parent_id]
+
+    return elements
+
+
+def _uniqueify_elements_and_metadata(elements: list[Element]) -> list[Element]:
+    """Ensure each of `elements` and their metadata are unique instances.
+
+    This prevents hard-to-diagnose bugs downstream when mutating one element unexpectedly also
+    mutates others because they are the same instance.
+    """
+
+    def iter_unique_elements(elements: list[Element]) -> Iterator[Element]:
+        """Substitute deep-copies of any non-unique elements or metadata in `elements`."""
+        seen_elements: set[int] = set()
+        seen_metadata: set[int] = set()
+
+        for element in elements:
+            if id(element) in seen_elements:
+                element = copy.deepcopy(element)
+            if id(element.metadata) in seen_metadata:
+                element.metadata = copy.deepcopy(element.metadata)
+            seen_elements.add(id(element))
+            seen_metadata.add(id(element.metadata))
+            yield element
+
+    return list(iter_unique_elements(elements))

unstructured/partition/csv.py

@@ -0,0 +1,187 @@
+from __future__ import annotations
+
+import contextlib
+import csv
+from functools import cached_property
+from typing import IO, Any, Iterator
+
+import pandas as pd
+
+from unstructured.chunking import add_chunking_strategy
+from unstructured.common.html_table import HtmlTable
+from unstructured.documents.elements import Element, ElementMetadata, Table
+from unstructured.file_utils.model import FileType
+from unstructured.partition.common.metadata import apply_metadata, get_last_modified_date
+from unstructured.utils import is_temp_file_path
+
+DETECTION_ORIGIN: str = "csv"
+CSV_FIELD_LIMIT = 10 * 1048576  # 10MiB
+
+
+@apply_metadata(FileType.CSV)
+@add_chunking_strategy
+def partition_csv(
+    filename: str | None = None,
+    *,
+    file: IO[bytes] | None = None,
+    encoding: str | None = None,
+    include_header: bool = False,
+    infer_table_structure: bool = True,
+    **kwargs: Any,
+) -> list[Element]:
+    """Partitions Microsoft Excel Documents in .csv format into its document elements.
+
+    Parameters
+    ----------
+    filename
+        A string defining the target filename path.
+    file
+        A file-like object using "rb" mode --> open(filename, "rb").
+    encoding
+        The encoding method used to decode the text input. If None, utf-8 will be used.
+    include_header
+        Determines whether or not header info info is included in text and medatada.text_as_html.
+    infer_table_structure
+        If True, any Table elements that are extracted will also have a metadata field
+        named "text_as_html" where the table's text content is rendered into an html string.
+        I.e., rows and cells are preserved.
+        Whether True or False, the "text" field is always present in any Table element
+        and is the text content of the table (no structure).
+    """
+    ctx = _CsvPartitioningContext.load(
+        file_path=filename,
+        file=file,
+        encoding=encoding,
+        include_header=include_header,
+        infer_table_structure=infer_table_structure,
+    )
+
+    csv.field_size_limit(CSV_FIELD_LIMIT)
+    with ctx.open() as file:
+        read_kw: dict = {"header": ctx.header, "sep": ctx.delimiter, "encoding": ctx.encoding}
+        # sep=None is not supported by the C engine; use Python engine to avoid ParserWarning.
+        if ctx.delimiter is None:
+            read_kw["engine"] = "python"
+        dataframe = pd.read_csv(file, **read_kw)
+
+    html_table = HtmlTable.from_html_text(
+        dataframe.to_html(index=False, header=include_header, na_rep="")
+    )
+
+    metadata = ElementMetadata(
+        filename=filename,
+        last_modified=ctx.last_modified,
+        text_as_html=html_table.html if infer_table_structure else None,
+    )
+
+    # -- a CSV file becomes a single `Table` element --
+    return [Table(text=html_table.text, metadata=metadata, detection_origin=DETECTION_ORIGIN)]
+
+
+class _CsvPartitioningContext:
+    """Encapsulates the partitioning-run details.
+
+    Provides access to argument values and especially encapsulates computation of values derived
+    from those values so they don't obscure the core partitioning logic.
+    """
+
+    def __init__(
+        self,
+        file_path: str | None = None,
+        file: IO[bytes] | None = None,
+        encoding: str | None = None,
+        include_header: bool = False,
+        infer_table_structure: bool = True,
+    ):
+        self._file_path = file_path
+        self._file = file
+        self._encoding = encoding
+        self._include_header = include_header
+        self._infer_table_structure = infer_table_structure
+
+    @classmethod
+    def load(
+        cls,
+        file_path: str | None,
+        file: IO[bytes] | None,
+        encoding: str | None,
+        include_header: bool,
+        infer_table_structure: bool,
+    ) -> _CsvPartitioningContext:
+        return cls(
+            file_path=file_path,
+            file=file,
+            encoding=encoding,
+            include_header=include_header,
+            infer_table_structure=infer_table_structure,
+        )._validate()
+
+    @cached_property
+    def delimiter(self) -> str | None:
+        """The CSV delimiter, nominally a comma ",".
+
+        `None` for a single-column CSV file which naturally has no delimiter.
+        """
+        sniffer = csv.Sniffer()
+        num_bytes = 65536
+
+        with self.open() as file:
+            # -- read whole lines, sniffer can be confused by a trailing partial line --
+            data = "\n".join(
+                ln.decode(self._encoding or "utf-8") for ln in file.readlines(num_bytes)
+            )
+
+        try:
+            return sniffer.sniff(data, delimiters=",;|").delimiter
+        except csv.Error:
+            # -- sniffing will fail on single-column csv as no default can be assumed --
+            return None
+
+    @cached_property
+    def header(self) -> int | None:
+        """Identifies the header row, if any, to Pandas, by idx."""
+        return 0 if self._include_header else None
+
+    @cached_property
+    def encoding(self) -> str | None:
+        """The encoding to use for reading the file."""
+        return self._encoding
+
+    @cached_property
+    def last_modified(self) -> str | None:
+        """The best last-modified date available, None if no sources are available."""
+        return (
+            None
+            if not self._file_path or is_temp_file_path(self._file_path)
+            else get_last_modified_date(self._file_path)
+        )
+
+    @contextlib.contextmanager
+    def open(self) -> Iterator[IO[bytes]]:
+        """Encapsulates complexity of dealing with file-path or file-like-object.
+
+        Provides an `IO[bytes]` object as the "common-denominator" document source.
+
+        Must be used as a context manager using a `with` statement:
+
+            with self._file as file:
+                do things with file
+
+        File is guaranteed to be at read position 0 when called.
+        """
+        if self._file_path:
+            with open(self._file_path, "rb") as f:
+                yield f
+        else:
+            file = self._file
+            assert file is not None  # -- guaranteed by `._validate()` --
+            # -- Be polite on principle. Reset file-pointer both before and after use --
+            file.seek(0)
+            yield file
+            file.seek(0)
+
+    def _validate(self) -> _CsvPartitioningContext:
+        """Raise on invalid argument values."""
+        if self._file_path is None and self._file is None:
+            raise ValueError("either file-path or file-like object must be provided")
+        return self

unstructured/partition/doc.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import os
+import tempfile
+from typing import IO, Any, Optional
+
+from unstructured.documents.elements import Element
+from unstructured.file_utils.model import FileType
+from unstructured.partition.common.common import convert_office_doc, exactly_one
+from unstructured.partition.common.metadata import get_last_modified_date
+from unstructured.partition.docx import partition_docx
+
+
+def partition_doc(
+    filename: Optional[str] = None,
+    file: Optional[IO[bytes]] = None,
+    metadata_filename: Optional[str] = None,
+    metadata_last_modified: Optional[str] = None,
+    libre_office_filter: Optional[str] = "MS Word 2007 XML",
+    **kwargs: Any,
+) -> list[Element]:
+    """Partitions Microsoft Word Documents in .doc format into its document elements.
+
+    All parameters available on `partition_docx()` are also available here.
+
+    Parameters
+    ----------
+    filename
+        A string defining the target filename path.
+    file
+        A file-like object using "rb" mode --> open(filename, "rb").
+    metadata_last_modified
+        The last modified date for the document.
+    libre_office_filter
+        The filter to use when coverting to .doc. The default is the
+        filter that is required when using LibreOffice7. Pass in None
+        if you do not want to apply any filter.
+    languages
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
+        Additional Parameters:
+            detect_language_per_element
+                Detect language per element instead of at the document level.
+    starting_page_number
+        Indicates what page number should be assigned to the first page in the document.
+        This information will be reflected in elements' metadata and can be be especially
+        useful when partitioning a document that is part of a larger document.
+    """
+    exactly_one(filename=filename, file=file)
+
+    last_modified = get_last_modified_date(filename) if filename else None
+
+    # -- validate file-path when provided so we can provide a more meaningful error --
+    if filename is not None and not os.path.exists(filename):
+        raise ValueError(f"The file {filename} does not exist.")
+
+    # -- `convert_office_doc` uses a command-line program that ships with LibreOffice to convert
+    # -- from DOC -> DOCX. So both the source and the target need to be file-system files. Put
+    # -- transient files in a temporary directory that is automatically removed so they don't
+    # -- pile up.
+    with tempfile.TemporaryDirectory() as target_dir:
+        source_file_path = f"{target_dir}/document.doc" if file is not None else filename
+        assert source_file_path is not None
+
+        # -- when source is a file-like object, write it to the filesystem so the command-line
+        # -- process can access it (CLI executes in different memory-space).
+        if file is not None:
+            with open(source_file_path, "wb") as f:
+                f.write(file.read())
+
+        # -- convert the .doc file to .docx. The resulting file takes the same base-name as the
+        # -- source file and is written to `target_dir`.
+        convert_office_doc(
+            source_file_path,
+            target_dir,
+            target_format="docx",
+            target_filter=libre_office_filter,
+        )
+
+        # -- compute the path of the resulting .docx document --
+        _, filename_no_path = os.path.split(os.path.abspath(source_file_path))
+        base_filename, _ = os.path.splitext(filename_no_path)
+        target_file_path = os.path.join(target_dir, f"{base_filename}.docx")
+
+        # -- and partition it. Note that `kwargs` is not passed which is a sketchy way to partially
+        # -- disable post-partitioning processing (what the decorators do) so for example the
+        # -- resulting elements are not double-chunked.
+        elements = partition_docx(
+            filename=target_file_path,
+            metadata_filename=metadata_filename or filename,
+            metadata_file_type=FileType.DOC,
+            metadata_last_modified=metadata_last_modified or last_modified,
+            **kwargs,
+        )
+
+    # -- Remove temporary document.docx path from metadata when necessary. Note `metadata_filename`
+    # -- defaults to `None` but that's better than a meaningless temporary filename.
+    if file:
+        for element in elements:
+            element.metadata.filename = metadata_filename
+
+    return elements

unstructured/partition/docx.py

@@ -0,0 +1,990 @@
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+import io
+import itertools
+import logging
+import os
+import tempfile
+import zipfile
+from functools import cached_property
+from typing import IO, Any, Iterator, Protocol, Type
+
+import docx
+from docx.document import Document
+from docx.enum.section import WD_SECTION_START
+from docx.oxml.table import CT_Tbl
+from docx.oxml.text.paragraph import CT_P
+from docx.section import Section, _Footer, _Header
+from docx.table import Table as DocxTable
+from docx.table import _Cell, _Row
+from docx.text.hyperlink import Hyperlink
+from docx.text.pagebreak import RenderedPageBreak
+from docx.text.paragraph import Paragraph
+from docx.text.run import Run
+from typing_extensions import TypeAlias
+
+from unstructured.chunking import add_chunking_strategy
+from unstructured.cleaners.core import clean_bullets
+from unstructured.common.html_table import htmlify_matrix_of_cell_texts
+from unstructured.documents.elements import (
+    Address,
+    Element,
+    ElementMetadata,
+    EmailAddress,
+    Footer,
+    Header,
+    Image,
+    Link,
+    ListItem,
+    NarrativeText,
+    PageBreak,
+    Table,
+    Text,
+    Title,
+)
+from unstructured.file_utils.model import FileType
+from unstructured.partition.common.metadata import apply_metadata, get_last_modified_date
+from unstructured.partition.text_type import (
+    is_bulleted_text,
+    is_email_address,
+    is_possible_narrative_text,
+    is_us_city_state_zip,
+)
+from unstructured.partition.utils.constants import PartitionStrategy
+from unstructured.utils import is_temp_file_path
+
+STYLE_TO_ELEMENT_MAPPING = {
+    "Caption": Text,  # TODO(robinson) - add caption element type
+    "Heading 1": Title,
+    "Heading 2": Title,
+    "Heading 3": Title,
+    "Heading 4": Title,
+    "Heading 5": Title,
+    "Heading 6": Title,
+    "Heading 7": Title,
+    "Heading 8": Title,
+    "Heading 9": Title,
+    "Intense Quote": Text,  # TODO(robinson) - add quote element type
+    "List": ListItem,
+    "List 2": ListItem,
+    "List 3": ListItem,
+    "List Bullet": ListItem,
+    "List Bullet 2": ListItem,
+    "List Bullet 3": ListItem,
+    "List Continue": ListItem,
+    "List Continue 2": ListItem,
+    "List Continue 3": ListItem,
+    "List Number": ListItem,
+    "List Number 2": ListItem,
+    "List Number 3": ListItem,
+    "List Paragraph": ListItem,
+    "Macro Text": Text,
+    "No Spacing": Text,
+    "Quote": Text,  # TODO(robinson) - add quote element type
+    "Subtitle": Title,
+    "TOCHeading": Title,
+    "Title": Title,
+}
+
+DETECTION_ORIGIN: str = "docx"
+# -- CT_* stands for "complex-type", an XML element type in docx parlance --
+BlockElement: TypeAlias = "CT_P | CT_Tbl"
+BlockItem: TypeAlias = "Paragraph | DocxTable"
+
+
+def register_picture_partitioner(picture_partitioner: PicturePartitionerT) -> None:
+    """Specify a pluggable sub-partitioner to be used for partitioning DOCX images."""
+    DocxPartitionerOptions.register_picture_partitioner(picture_partitioner)
+
+
+# ================================================================================================
+# DOCX DOMAIN MODEL DEFINITIONS
+# ================================================================================================
+
+
+class PicturePartitionerT(Protocol):
+    """Defines the interface for a pluggable sub-partitioner for DOCX Picture objects.
+
+    In Microsoft Word parlance, an image is a "picture". We use that term here for an image in a
+    DOCX file both for domain consistency and because it conveniently avoids confusion with an
+    `unstructured` `Image` element.
+
+    A picture can be either *inline* or *floating*. An inline picture is treated like a big
+    character in the text of a paragraph, moving with the text. A floating picture can be moved
+    freely and text flows around it.
+
+    Both inline and floating pictures are defined inside a paragraph in the DOCX file. A paragraph
+    can have zero or more pictures. A DOCX picture partitioner takes a `docx` `Paragraph` object
+    and generates an `Image` element for each picture found in that paragraph.
+    """
+
+    @classmethod
+    def iter_elements(cls, paragraph: Paragraph, opts: DocxPartitionerOptions) -> Iterator[Image]:
+        """Generate an `Image` element for each picture in `paragraph`."""
+        ...
+
+
+# ================================================================================================
+# PARTITIONER
+# ================================================================================================
+
+
+@apply_metadata(FileType.DOCX)
+@add_chunking_strategy
+def partition_docx(
+    filename: str | None = None,
+    *,
+    file: IO[bytes] | None = None,
+    include_page_breaks: bool = True,
+    infer_table_structure: bool = True,
+    starting_page_number: int = 1,
+    strategy: str | None = None,
+    **kwargs: Any,
+) -> list[Element]:
+    """Partitions Microsoft Word Documents in .docx format into its document elements.
+
+    Parameters
+    ----------
+    filename
+        A string defining the target filename path.
+    file
+        A file-like object using "rb" mode --> open(filename, "rb").
+    include_page_breaks
+        When True, add a `PageBreak` element to the element-stream when a page-break is detected in
+        the document. Note that not all DOCX files include page-break information.
+    infer_table_structure
+        If True, any Table elements that are extracted will also have a metadata field
+        named "text_as_html" where the table's text content is rendered into an html string.
+        I.e., rows and cells are preserved.
+        Whether True or False, the "text" field is always present in any Table element
+        and is the text content of the table (no structure).
+    metadata_filename
+        The filename to use for the metadata. Relevant because partition_doc converts the document
+        to .docx before partition. We want the original source filename in the metadata.
+    metadata_last_modified
+        The last modified date for the document.
+    starting_page_number
+        Assign this number to the first page of this document and increment the page number from
+        there.
+    """
+    opts = DocxPartitionerOptions.load(
+        file=file,
+        file_path=filename,
+        include_page_breaks=include_page_breaks,
+        infer_table_structure=infer_table_structure,
+        starting_page_number=starting_page_number,
+        strategy=strategy,
+    )
+
+    elements = _DocxPartitioner.iter_document_elements(opts)
+
+    return list(elements)
+
+
+class DocxPartitionerOptions:
+    """Encapsulates partitioning option validation, computation, and application of defaults."""
+
+    _PicturePartitionerCls = None
+    """Sub-partitioner used to extract pictures from a paragraph as `Image` elements.
+
+    This value has module lifetime and is updated by calling the `register_picture_partitioner()`
+    function defined in this module. The value sent to `register_picture_partitioner()` must be a
+    pluggable sub-partitioner implementing the `PicturePartitionerT` interface. After
+    registration, all paragraphs in subsequently partitioned DOCX documents will be sent to this
+    sub-partitioner to extract images when so configured.
+    """
+
+    def __init__(
+        self,
+        *,
+        file: IO[bytes] | None,
+        file_path: str | None,
+        include_page_breaks: bool,
+        infer_table_structure: bool,
+        starting_page_number: int = 1,
+        strategy: str | None = None,
+    ):
+        self._file = file
+        self._file_path = file_path
+        self._include_page_breaks = include_page_breaks
+        self._infer_table_structure = infer_table_structure
+        self._strategy = strategy
+        # -- options object maintains page-number state --
+        self._page_counter = starting_page_number
+
+    @classmethod
+    def load(cls, **kwargs: Any) -> DocxPartitionerOptions:
+        """Construct and validate an instance."""
+        return cls(**kwargs)._validate()
+
+    @classmethod
+    def register_picture_partitioner(cls, picture_partitioner: PicturePartitionerT):
+        """Specify a pluggable sub-partitioner to extract images from DOCX paragraphs."""
+        cls._PicturePartitionerCls = picture_partitioner
+
+    @cached_property
+    def document(self) -> Document:
+        """The python-docx `Document` object loaded from file or filename."""
+        return docx.Document(self._docx_file)
+
+    @cached_property
+    def include_page_breaks(self) -> bool:
+        """When True, include `PageBreak` elements in element-stream.
+
+        Note that regardless of this setting, page-breaks are detected, and page-number is tracked
+        and included in element metadata. Only the presence of distinct `PageBreak` elements (which
+        contain no text) in the element stream is affected.
+        """
+        return self._include_page_breaks
+
+    def increment_page_number(self) -> Iterator[PageBreak]:
+        """Increment page-number by 1 and generate a PageBreak element if enabled."""
+        self._page_counter += 1
+        # -- only emit page-breaks when enabled --
+        if self._include_page_breaks:
+            yield PageBreak("", detection_origin=DETECTION_ORIGIN)
+
+    @cached_property
+    def infer_table_structure(self) -> bool:
+        """True when partitioner should compute and apply `text_as_html` metadata for tables."""
+        return self._infer_table_structure
+
+    @cached_property
+    def last_modified(self) -> str | None:
+        """The best last-modified date available, None if no sources are available."""
+        if not self._file_path:
+            return None
+
+        return (
+            None if is_temp_file_path(self._file_path) else get_last_modified_date(self._file_path)
+        )
+
+    @cached_property
+    def metadata_file_path(self) -> str | None:
+        """The best available file-path for this document or `None` if unavailable."""
+        return self._file_path
+
+    @property
+    def metadata_page_number(self) -> int | None:
+        """The current page number to report in metadata, or None if we can't really tell.
+
+        Page numbers are not added to element metadata if we can't find any page-breaks in the
+        document (which may be a common case).
+
+        In the DOCX format, determining page numbers is strictly a best-efforts attempt since
+        actual page-breaks are determined at rendering time (e.g. printing) based on the
+        font-metrics of the target device. Explicit (hard) page-breaks are always recorded in the
+        docx file but the rendered page-breaks are only added optionally.
+        """
+        return self._page_counter if self._document_contains_pagebreaks else None
+
+    @property
+    def page_number(self) -> int:
+        """The current page number.
+
+        Note this value may not represent the actual rendered page number when rendered page-break
+        indicators are not present in the document (not uncommon). Use `.metadata_page_number` for
+        metadata purposes, which is `None` when rendered page-breaks are not present in this
+        document.
+        """
+        return self._page_counter
+
+    @cached_property
+    def picture_partitioner(self) -> PicturePartitionerT:
+        """The sub-partitioner to use for DOCX image extraction."""
+        # -- Note this value has partitioning-run scope. An instance of this options class is
+        # -- instantiated once per partitioning run (each document can have different options).
+        # -- Because this is a lazyproperty, it is computed only on the first reference. All
+        # -- subsequent references during the same partitioning run will get the same value. This
+        # -- ensures image extraction is processed consistently within a single document.
+        return self._PicturePartitionerCls or _NullPicturePartitioner
+
+    @cached_property
+    def strategy(self) -> str:
+        """The partitioning strategy for this document.
+
+        One of "hi_res", "fast", and a few others. These are available as class attributes on
+        `unstructured.partition.utils.constants.PartitionStrategy` but resolve to str values.
+        """
+        return PartitionStrategy.HI_RES if self._strategy is None else self._strategy
+
+    @cached_property
+    def _document_contains_pagebreaks(self) -> bool:
+        """True when there is at least one page-break detected in the document.
+
+        Only `w:lastRenderedPageBreak` elements reliably indicate a page-break. These are reliably
+        inserted by Microsoft Word, but probably don't appear in documents converted into .docx
+        format from for example .odt format.
+        """
+        xpath = (
+            # NOTE(scanny) - w:lastRenderedPageBreak (lrpb) is run (w:r) inner content. `w:r` can
+            # appear in a paragraph (w:p). w:r can also appear in a hyperlink (w:hyperlink), which
+            # is w:p inner-content and both of these can occur inside a table-cell as well as the
+            # document body
+            "./w:body/w:p/w:r/w:lastRenderedPageBreak"
+            " | ./w:body/w:p/w:hyperlink/w:r/w:lastRenderedPageBreak"
+            " | ./w:body/w:tbl/w:tr/w:tc/w:p/w:r/w:lastRenderedPageBreak"
+            " | ./w:body/w:tbl/w:tr/w:tc/w:p/w:hyperlink/w:r/w:lastRenderedPageBreak"
+        )
+
+        return bool(self.document.element.xpath(xpath))
+
+    @cached_property
+    def _docx_file(self) -> str | IO[bytes]:
+        """The Word 2007+ document file to be partitioned.
+
+        This is either a `str` path or a file-like object. `python-docx` accepts either for opening
+        a document file.
+        """
+        if self._file_path:
+            return self._file_path
+
+        # -- In Python <3.11 SpooledTemporaryFile does not implement ".seekable" which triggers an
+        # -- exception when Zipfile tries to open it. The docx format is a zip archive so we need
+        # -- to work around that bug here.
+        if isinstance(self._file, tempfile.SpooledTemporaryFile):
+            self._file.seek(0)
+            return io.BytesIO(self._file.read())
+
+        assert self._file is not None  # -- assured by `._validate()` --
+        return self._file
+
+    def _validate(self) -> DocxPartitionerOptions:
+        """Raise on first invalide option, return self otherwise."""
+        # -- provide distinguished error between "file-not-found" and "not-a-DOCX-file" --
+        if self._file_path:
+            if not os.path.isfile(self._file_path):
+                raise FileNotFoundError(f"no such file or directory: {repr(self._file_path)}")
+            if not zipfile.is_zipfile(self._file_path):
+                raise ValueError(f"not a ZIP archive (so not a DOCX file): {repr(self._file_path)}")
+        elif self._file:
+            if not zipfile.is_zipfile(self._file):
+                raise ValueError(f"not a ZIP archive (so not a DOCX file): {repr(self._file)}")
+        else:
+            raise ValueError(
+                "no DOCX document specified, either `filename` or `file` argument must be provided"
+            )
+
+        return self
+
+
+class _DocxPartitioner:
+    """Provides `.partition()` for MS-Word 2007+ (.docx) files."""
+
+    def __init__(self, opts: DocxPartitionerOptions) -> None:
+        self._opts = opts
+
+    @classmethod
+    def iter_document_elements(cls, opts: DocxPartitionerOptions) -> Iterator[Element]:
+        """Partition MS Word documents (.docx format) into its document elements."""
+        self = cls(opts)
+        # NOTE(scanny): It's possible for a Word document to have no sections. In particular, a
+        # Microsoft Teams chat transcript exported to DOCX contains no sections. Such a
+        # "section-less" document has to be interated differently and has no headers or footers and
+        # therefore no page-size or margins.
+        return (
+            self._iter_document_elements()
+            if self._document_contains_sections
+            else self._iter_sectionless_document_elements()
+        )
+
+    def _iter_document_elements(self) -> Iterator[Element]:
+        """Generate each document-element in (docx) `document` in document order."""
+        # -- This implementation composes a collection of iterators into a "combined" iterator
+        # -- return value using `yield from`. You can think of the return value as an Element
+        # -- stream and each `yield from` as "add elements found by this function to the stream".
+        # -- This is functionally analogous to declaring `elements: list[Element] = []` at the top
+        # -- and using `elements.extend()` for the results of each of the function calls, but is
+        # -- more perfomant, uses less memory (avoids producing and then garbage-collecting all
+        # -- those small lists), is more flexible for later iterator operations like filter,
+        # -- chain, map, etc. and is perhaps more elegant and simpler to read once you have the
+        # -- concept of what it's doing. You can see the same pattern repeating in the "sub"
+        # -- functions like `._iter_paragraph_elements()` where the "just return when done"
+        # -- characteristic of a generator avoids repeated code to form interim results into lists.
+        for section_idx, section in enumerate(self._document.sections):
+            yield from self._iter_section_page_breaks(section_idx, section)
+            yield from self._iter_section_headers(section)
+
+            for block_item in section.iter_inner_content():
+                # -- a block-item can be a Paragraph or a Table, maybe others later so elif here.
+                # -- Paragraph is more common so check that first.
+                if isinstance(block_item, Paragraph):
+                    yield from self._iter_paragraph_elements(block_item)
+                elif isinstance(  # pyright: ignore[reportUnnecessaryIsInstance]
+                    block_item, DocxTable
+                ):
+                    yield from self._iter_table_element(block_item)
+
+            yield from self._iter_section_footers(section)
+
+    def _iter_sectionless_document_elements(self) -> Iterator[Element]:
+        """Generate each document-element in a docx `document` that has no sections.
+
+        A "section-less" DOCX must be iterated differently. Also it will have no headers or footers
+        (because those live in a section).
+        """
+        for block_item in self._document.iter_inner_content():
+            if isinstance(block_item, Paragraph):
+                yield from self._iter_paragraph_elements(block_item)
+            # -- can only be a Paragraph or Table so far but more types may come later --
+            elif isinstance(block_item, DocxTable):  # pyright: ignore[reportUnnecessaryIsInstance]
+                yield from self._iter_table_element(block_item)
+
+    def _classify_paragraph_to_element(self, paragraph: Paragraph) -> Iterator[Element]:
+        """Generate zero-or-one document element for `paragraph`.
+
+        In Word, an empty paragraph is commonly used for inter-paragraph spacing. An empty paragraph
+        does not contribute to the document-element stream and will not cause an element to be
+        emitted.
+        """
+        text = "".join(
+            e.text
+            for e in paragraph._p.xpath(
+                "w:r | w:hyperlink | w:r/descendant::wp:inline[ancestor::w:drawing][1]//w:r"
+            )
+        )
+
+        # -- blank paragraphs are commonly used for spacing between paragraphs and do not
+        # -- contribute to the document-element stream
+        if not text.strip():
+            return
+
+        metadata = self._paragraph_metadata(paragraph)
+
+        # -- a list-item gets some special treatment, mutating the text to remove a
+        # -- bullet-character if present
+        if self._is_list_item(paragraph):
+            clean_text = clean_bullets(text).strip()
+            if clean_text:
+                yield ListItem(
+                    text=clean_text,
+                    metadata=metadata,
+                    detection_origin=DETECTION_ORIGIN,
+                )
+            return
+
+        # -- determine element-type from an explicit Word paragraph-style if possible --
+        TextSubCls = self._style_based_element_type(paragraph)
+        if TextSubCls:
+            yield TextSubCls(text=text, metadata=metadata, detection_origin=DETECTION_ORIGIN)
+            return
+
+        # -- try to recognize the element type by parsing its text --
+        TextSubCls = self._parse_paragraph_text_for_element_type(paragraph)
+        if TextSubCls:
+            yield TextSubCls(text=text, metadata=metadata, detection_origin=DETECTION_ORIGIN)
+            return
+
+        # -- if all that fails we give it the default `Text` element-type --
+        yield Text(text, metadata=metadata, detection_origin=DETECTION_ORIGIN)
+
+    def _convert_table_to_html(self, table: DocxTable) -> str:
+        """HTML string version of `table`.
+
+        Example:
+
+            <table>
+            <tbody>
+            <tr><th>item  </th><th style="text-align: right;">  qty</th></tr>
+            <tr><td>spam  </td><td style="text-align: right;">   42</td></tr>
+            <tr><td>eggs  </td><td style="text-align: right;">  451</td></tr>
+            <tr><td>bacon </td><td style="text-align: right;">    0</td></tr>
+            </tbody>
+            </table>
+
+        `is_nested` is used for recursive calls when a nested table is encountered. Certain
+        behaviors are different in that case, but the caller can safely ignore that parameter and
+        allow it to take its default value.
+        """
+
+        def iter_cell_block_items(cell: _Cell) -> Iterator[str]:
+            """Generate the text of each paragraph or table in `cell` as a separate string.
+
+            A table nested in `cell` is converted to the normalized text it contains.
+            """
+            for block_item in cell.iter_inner_content():
+                if isinstance(paragraph := block_item, Paragraph):
+                    # -- all docx content is ultimately in a paragraph; a nested table contributes
+                    # -- structure only
+                    yield paragraph.text
+                elif isinstance(table := block_item, DocxTable):
+                    for row in table.rows:
+                        yield from iter_row_cells_as_text(row)
+
+        def iter_row_cells_as_text(row: _Row) -> Iterator[str]:
+            """Generate the normalized text of each cell in `row` as a separate string.
+
+            The text of each paragraph within a cell is not separated. A table nested in a cell is
+            converted to a normalized string of its contents and combined with the text of the
+            cell that contains the table.
+            """
+            # -- Each omitted cell at the start of the row (pretty rare) gets the empty string.
+            # -- This preserves column alignment when one or more initial cells are omitted.
+            for _ in range(row.grid_cols_before):
+                yield ""
+
+            try:
+                # -- row.cells may introduce `ValueError: no tc element at grid_offset=X` if the
+                # -- table has merged or malformed cells. always wrap in try/except.
+                for cell in row.cells:
+                    cell_text = " ".join(iter_cell_block_items(cell))
+                    yield " ".join(cell_text.split())
+            except Exception as e:
+                logging.warning(f"Skipping cell in _iter_row_cells_as_text due to: {e}")
+                yield ""
+
+            # -- Each omitted cell at the end of the row (also rare) gets the empty string. --
+            for _ in range(row.grid_cols_after):
+                yield ""
+
+        return htmlify_matrix_of_cell_texts([list(iter_row_cells_as_text(r)) for r in table.rows])
+
+    @cached_property
+    def _document(self) -> Document:
+        """The python-docx `Document` object loaded from file or filename."""
+        return self._opts.document
+
+    @cached_property
+    def _document_contains_sections(self) -> bool:
+        """True when there is at least one section in the document.
+
+        This is always true for a document produced by Word, but may not always be the case when the
+        document results from conversion or export. In particular, a Microsoft Teams chat-transcript
+        export will have no sections.
+        """
+        return bool(self._document.sections)
+
+    def _header_footer_text(self, hdrftr: _Header | _Footer) -> str:
+        """The text enclosed in `hdrftr` as a single string.
+
+        Each paragraph is included along with the text of each table cell. Empty text is omitted.
+        Each paragraph text-item is separated by a newline ("\n") although note that a paragraph
+        that contains a line-break will also include a newline representing that line-break, so
+        newlines do not necessarily distinguish separate paragraphs.
+
+        The entire text of a table is included as a single string with a space separating the text
+        of each cell.
+
+        A header with no text or only whitespace returns the empty string ("").
+        """
+
+        def iter_hdrftr_texts(hdrftr: _Header | _Footer) -> Iterator[str]:
+            """Generate each text item in `hdrftr` stripped of leading and trailing whitespace.
+
+            This includes paragraphs as well as table cell contents.
+            """
+            for block_item in hdrftr.iter_inner_content():
+                if isinstance(block_item, Paragraph):
+                    yield block_item.text.strip()
+                # -- can only be a Paragraph or Table so far but more types may come later --
+                elif isinstance(  # pyright: ignore[reportUnnecessaryIsInstance]
+                    block_item, DocxTable
+                ):
+                    yield " ".join(self._iter_table_texts(block_item))
+
+        return "\n".join(text for text in iter_hdrftr_texts(hdrftr) if text)
+
+    def _is_list_item(self, paragraph: Paragraph) -> bool:
+        """True when `paragraph` can be identified as a list-item."""
+        if is_bulleted_text(paragraph.text):
+            return True
+
+        return "<w:numPr>" in paragraph._p.xml
+
+    def _iter_paragraph_elements(self, paragraph: Paragraph) -> Iterator[Element]:
+        """Generate zero-or-more document elements for `paragraph`.
+
+        The generated elements can be both textual elements and PageBreak elements. An empty
+        paragraph produces no elements.
+        """
+
+        def iter_paragraph_items(paragraph: Paragraph) -> Iterator[Paragraph | RenderedPageBreak]:
+            """Generate Paragraph and RenderedPageBreak items from `paragraph`.
+
+            Each generated paragraph is the portion of the paragraph on the same page. When the
+            paragraph contains no page-breaks, it is iterated unchanged and iteration stops. When
+            there is a page-break, in general there one paragraph "fragment" before the page break,
+            the page break, and then the fragment after the page break. However many combinations
+            are possible. The first item can be either a page-break or a paragraph, but the type
+            always alternates throughout the sequence.
+            """
+            if not paragraph.contains_page_break:
+                yield paragraph
+                return
+
+            page_break = paragraph.rendered_page_breaks[0]
+
+            # -- preceding-fragment is None when first paragraph content is a page-break --
+            preceding_paragraph_fragment = page_break.preceding_paragraph_fragment
+            if preceding_paragraph_fragment:
+                yield preceding_paragraph_fragment
+
+            yield page_break
+
+            # -- following-fragment is None when page-break is last paragraph content. This is
+            # -- probably quite rare (Word moves these to the start of the next paragraph) but
+            # -- easier to check for it than prove it can't happen.
+            following_paragraph_fragment = page_break.following_paragraph_fragment
+            # -- the paragraph fragment following a page-break can itself contain another
+            # -- page-break; this would also be quite rare, but it can happen so we just recurse
+            # -- into the second fragment the same way we handled the original paragraph
+            if following_paragraph_fragment:
+                yield from iter_paragraph_items(following_paragraph_fragment)
+
+        for item in iter_paragraph_items(paragraph):
+            if isinstance(item, Paragraph):
+                yield from self._classify_paragraph_to_element(item)
+                yield from self._iter_paragraph_images(item)
+            else:
+                yield from self._opts.increment_page_number()
+
+    def _iter_paragraph_emphasis(self, paragraph: Paragraph) -> Iterator[dict[str, str]]:
+        """Generate e.g. {"text": "MUST", "tag": "b"} for each emphasis in `paragraph`."""
+        for run in paragraph.runs:
+            text = run.text.strip() if run.text else ""
+            if not text:
+                continue
+            if run.bold:
+                yield {"text": text, "tag": "b"}
+            if run.italic:
+                yield {"text": text, "tag": "i"}
+
+    def _iter_paragraph_images(self, paragraph: Paragraph) -> Iterator[Image]:
+        """Generate `Image` element for each picture shape in `paragraph` when so configured."""
+        # -- Delegate this job to the pluggable Picture partitioner. Note the default picture
+        # -- partitioner does not extract images.
+        PicturePartitionerCls = self._opts.picture_partitioner
+        yield from PicturePartitionerCls.iter_elements(paragraph, self._opts)
+
+    def _iter_section_footers(self, section: Section) -> Iterator[Footer]:
+        """Generate any `Footer` elements defined for this section.
+
+        A Word document has up to three header and footer definition pairs for each document
+        section, a primary, first-page, and even-page header and footer. The first-page pair
+        applies only to the first page of the section (perhaps a title page or chapter start). The
+        even-page pair is used in book-bound documents where there are both recto and verso pages
+        (it is applied to verso (even-numbered) pages). A page where neither more specialized
+        footer applies uses the primary footer.
+        """
+
+        def iter_footer(footer: _Footer, header_footer_type: str) -> Iterator[Footer]:
+            """Generate zero-or-one Footer elements for `footer`."""
+            if footer.is_linked_to_previous:
+                return
+            text = self._header_footer_text(footer)
+            if not text:
+                return
+            yield Footer(
+                text=text,
+                detection_origin=DETECTION_ORIGIN,
+                metadata=ElementMetadata(
+                    filename=self._opts.metadata_file_path,
+                    header_footer_type=header_footer_type,
+                    category_depth=0,
+                ),
+            )
+
+        yield from iter_footer(section.footer, "primary")
+        if section.different_first_page_header_footer:
+            yield from iter_footer(section.first_page_footer, "first_page")
+        if self._document.settings.odd_and_even_pages_header_footer:
+            yield from iter_footer(section.even_page_footer, "even_page")
+
+    def _iter_section_headers(self, section: Section) -> Iterator[Header]:
+        """Generate `Header` elements for this section if it has them.
+
+        See `._iter_section_footers()` docstring for more on docx headers and footers.
+        """
+
+        def maybe_iter_header(header: _Header, header_footer_type: str) -> Iterator[Header]:
+            """Generate zero-or-one Header elements for `header`."""
+            if header.is_linked_to_previous:
+                return
+            text = self._header_footer_text(header)
+            if not text:
+                return
+            yield Header(
+                text=text,
+                detection_origin=DETECTION_ORIGIN,
+                metadata=ElementMetadata(
+                    filename=self._opts.metadata_file_path,
+                    header_footer_type=header_footer_type,
+                    category_depth=0,  # -- headers are always at the root level}
+                ),
+            )
+
+        yield from maybe_iter_header(section.header, "primary")
+        if section.different_first_page_header_footer:
+            yield from maybe_iter_header(section.first_page_header, "first_page")
+        if self._document.settings.odd_and_even_pages_header_footer:
+            yield from maybe_iter_header(section.even_page_header, "even_page")
+
+    def _iter_section_page_breaks(self, section_idx: int, section: Section) -> Iterator[PageBreak]:
+        """Generate zero-or-one `PageBreak` document elements for `section`.
+
+        A docx section has a "start" type which can be "continuous" (no page-break), "nextPage",
+        "evenPage", or "oddPage". For the next, even, and odd varieties, a `w:renderedPageBreak`
+        element signals one page break. Here we only need to handle the case where we need to add
+        another, for example to go from one odd page to another odd page and we need a total of
+        two page-breaks.
+        """
+
+        def page_is_odd() -> bool:
+            return self._opts.page_number % 2 == 1
+
+        start_type = section.start_type
+
+        # -- This method is called upon entering a new section, which happens before any paragraphs
+        # -- in that section are partitioned. A rendered page-break due to a section-start occurs
+        # -- in the first paragraph of the section and so occurs _later_ in the proces. Here we
+        # -- predict when two page breaks will be needed and emit one of them. The second will be
+        # -- emitted by the rendered page-break to follow.
+
+        if start_type == WD_SECTION_START.EVEN_PAGE:  # noqa
+            # -- on an even page we need two total, add one to supplement the rendered page break
+            # -- to follow. There is no "first-document-page" special case because 1 is odd.
+            if not page_is_odd():
+                yield from self._opts.increment_page_number()
+
+        elif start_type == WD_SECTION_START.ODD_PAGE:
+            # -- the first page of the document is an implicit "new" odd-page, so no page-break --
+            if section_idx == 0:
+                return
+            if page_is_odd():
+                yield from self._opts.increment_page_number()
+
+        # -- otherwise, start-type is one of "continuous", "new-column", or "next-page", none of
+        # -- which need our help to get the page-breaks right.
+        return
+
+    def _iter_table_element(self, table: DocxTable) -> Iterator[Table]:
+        """Generate zero-or-one Table element for a DOCX `w:tbl` XML element."""
+        # -- at present, we always generate exactly one Table element, but we might want
+        # -- to skip, for example, an empty table.
+        html_table = (
+            self._convert_table_to_html(table) if self._opts.infer_table_structure else None
+        )
+        text_table = " ".join(self._iter_table_texts(table))
+        emphasized_text_contents, emphasized_text_tags = self._table_emphasis(table)
+
+        yield Table(
+            text_table,
+            detection_origin=DETECTION_ORIGIN,
+            metadata=ElementMetadata(
+                text_as_html=html_table,
+                filename=self._opts.metadata_file_path,
+                page_number=self._opts.metadata_page_number,
+                last_modified=self._opts.last_modified,
+                emphasized_text_contents=emphasized_text_contents or None,
+                emphasized_text_tags=emphasized_text_tags or None,
+            ),
+        )
+
+    def _iter_table_emphasis(self, table: DocxTable) -> Iterator[dict[str, str]]:
+        """Generate e.g. {"text": "word", "tag": "b"} for each emphasis in `table`."""
+        for row in table.rows:
+            try:
+                # -- row.cells may introduce `ValueError: no tc element at grid_offset=X` if the
+                # -- table has merged or malformed cells. always wrap in try/except.
+                for cell in row.cells:
+                    for paragraph in cell.paragraphs:
+                        yield from self._iter_paragraph_emphasis(paragraph)
+            except Exception as e:
+                logging.warning(f"Skipping row in _iter_table_emphasis due to: {e}")
+                continue
+
+    def _iter_table_texts(self, table: DocxTable) -> Iterator[str]:
+        """Generate text of each cell in `table` stripped of leading and trailing whitespace.
+
+        Nested tables are recursed into and their text contributes to the output in depth-first
+        pre-order. Empty strings due to empty or whitespace-only cells are dropped.
+        """
+
+        def iter_cell_texts(cell: _Cell) -> Iterator[str]:
+            """Generate each text item in `cell` stripped of leading and trailing whitespace.
+
+            This includes paragraphs as well as table cell contents.
+            """
+            for block_item in cell.iter_inner_content():
+                if isinstance(block_item, Paragraph):
+                    yield block_item.text.strip()
+                # -- can only be a Paragraph or Table so far but more types may come later --
+                elif isinstance(  # pyright: ignore[reportUnnecessaryIsInstance]
+                    block_item, DocxTable
+                ):
+                    yield from self._iter_table_texts(block_item)
+
+        for row in table.rows:
+            tr = row._tr
+            for tc in tr.tc_lst:
+                # -- vMerge="continue" indicates a spanned cell in a vertical merge --
+                if tc.vMerge == "continue":
+                    continue
+                # -- do not generate empty strings --
+                yield from (text for text in iter_cell_texts(_Cell(tc, table)) if text)
+
+    def _paragraph_emphasis(self, paragraph: Paragraph) -> tuple[list[str], list[str]]:
+        """[contents, tags] pair describing emphasized text in `paragraph`."""
+        iter_p_emph, iter_p_emph_2 = itertools.tee(self._iter_paragraph_emphasis(paragraph))
+        return ([e["text"] for e in iter_p_emph], [e["tag"] for e in iter_p_emph_2])
+
+    def _paragraph_link_meta(self, paragraph: Paragraph) -> tuple[list[str], list[str], list[Link]]:
+        """Describes hyperlinks in `paragraph`, if any."""
+        if not paragraph.hyperlinks:
+            return [], [], []
+
+        def iter_paragraph_links() -> Iterator[Link]:
+            """Generate `Link` typed-dict for each external link in `paragraph`.
+
+            Word uses hyperlinks for internal "jumps" within the document, as well as for web and
+            other external locations. Only generate the external ones.
+            """
+            offset = 0
+            for item in paragraph.iter_inner_content():
+                if isinstance(item, Run):
+                    offset += len(item.text)
+                elif isinstance(item, Hyperlink):  # pyright: ignore[reportUnnecessaryIsInstance]
+                    text = item.text
+                    url = item.url
+                    start_index = offset
+                    offset += len(text)
+                    # -- docx hyperlinks include "internal" links, like a table-of-contents
+                    # -- (TOC) entry has a jump to the named heading in the document (e.g.
+                    # -- '#_Toc147925734'. Such links have a fragment but not an address
+                    # -- (URL). Treat those as regular text.
+                    if not url:
+                        continue
+                    # -- all Word hyperlinks should contain text, otherwise they have no
+                    # -- visual appearance on the document. Not expected, but technically possible
+                    # -- so filter these out too.
+                    if not text:
+                        continue
+                    yield Link(text=text, url=url, start_index=start_index)
+
+        links = list(iter_paragraph_links())
+        # -- link["text"] is allowed to be None by the declared type for `Link`, but never will be
+        # -- here because such a link is filtered out above. Use empty str to satisfy type-checker.
+        link_texts = [link["text"] or "" for link in links]
+        link_urls = [link["url"] for link in links]
+        return link_texts, link_urls, links
+
+    def _paragraph_metadata(self, paragraph: Paragraph) -> ElementMetadata:
+        """ElementMetadata object describing `paragraph`."""
+        category_depth = self._parse_category_depth_by_style(paragraph)
+        emphasized_text_contents, emphasized_text_tags = self._paragraph_emphasis(paragraph)
+        link_texts, link_urls, links = self._paragraph_link_meta(paragraph)
+        element_metadata = ElementMetadata(
+            category_depth=category_depth,
+            emphasized_text_contents=emphasized_text_contents or None,
+            emphasized_text_tags=emphasized_text_tags or None,
+            filename=self._opts.metadata_file_path,
+            last_modified=self._opts.last_modified,
+            link_texts=link_texts or None,
+            link_urls=link_urls or None,
+            links=links or None,
+            page_number=self._opts.metadata_page_number,
+        )
+        element_metadata.detection_origin = "docx"
+        return element_metadata
+
+    def _parse_category_depth_by_style(self, paragraph: Paragraph) -> int:
+        """Determine category depth from paragraph metadata"""
+
+        # Determine category depth from paragraph ilvl xpath
+        xpath = paragraph._element.xpath("./w:pPr/w:numPr/w:ilvl/@w:val")
+        if xpath:
+            return round(float(xpath[0]))
+
+        # Determine category depth from style name
+        style_name = (paragraph.style and paragraph.style.name) or "Normal"
+        depth = self._parse_category_depth_by_style_name(style_name)
+
+        if depth > 0:
+            return depth
+        else:
+            # Check if category depth can be determined from style ilvl
+            return self._parse_category_depth_by_style_ilvl()
+
+    def _parse_category_depth_by_style_ilvl(self) -> int:
+        # TODO(newelh) Parsing category depth by style ilvl is not yet implemented
+        return 0
+
+    def _parse_category_depth_by_style_name(self, style_name: str) -> int:
+        """Parse category-depth from the style-name of `paragraph`.
+
+        Category depth is 0-indexed and relative to the other element types in the document.
+        """
+
+        def _extract_number(suffix: str) -> int:
+            parts = suffix.split()
+            return int(parts[-1]) - 1 if (parts and parts[-1].isdigit()) else 0
+
+        # Heading styles
+        if style_name.startswith("Heading"):
+            return _extract_number(style_name)
+
+        if style_name == "Subtitle":
+            return 1
+
+        # List styles
+        list_prefixes = ("List", "List Bullet", "List Continue", "List Number")
+        if style_name.startswith(list_prefixes):
+            return _extract_number(style_name)
+
+        # Other styles
+        return 0
+
+    def _parse_paragraph_text_for_element_type(self, paragraph: Paragraph) -> Type[Text] | None:
+        """Attempt to differentiate the element-type by inspecting the raw text."""
+        text = paragraph.text.strip()
+
+        if len(text) < 2:
+            return None
+        if is_us_city_state_zip(text):
+            return Address
+        if is_email_address(text):
+            return EmailAddress
+        if is_possible_narrative_text(text):
+            return NarrativeText
+
+        return None
+
+    def _style_based_element_type(self, paragraph: Paragraph) -> Type[Text] | None:
+        """Element-type for `paragraph` based on its paragraph-style.
+
+        Returns `None` when the style doesn't tell us anything useful, including when it
+        is the default "Normal" style.
+        """
+        # NOTE(robinson) - documentation on built-in styles at the link below:
+        # https://python-docx.readthedocs.io/en/latest/user/styles-understanding.html \
+        # #paragraph-styles-in-default-template
+
+        # -- paragraph.style can be None in rare cases, so can style.name. That's going
+        # -- to mean default style which is equivalent to "Normal" for our purposes.
+        style_name = (paragraph.style and paragraph.style.name) or "Normal"
+
+        # NOTE(robinson) - The "Normal" style name will return None since it's not
+        # in the mapping. Unknown style names will also return None.
+        return STYLE_TO_ELEMENT_MAPPING.get(style_name)
+
+    def _table_emphasis(self, table: DocxTable) -> tuple[list[str], list[str]]:
+        """[contents, tags] pair describing emphasized text in `table`."""
+        iter_tbl_emph, iter_tbl_emph_2 = itertools.tee(self._iter_table_emphasis(table))
+        return ([e["text"] for e in iter_tbl_emph], [e["tag"] for e in iter_tbl_emph_2])
+
+
+# ================================================================================================
+# SUB-PARTITIONERS
+# ================================================================================================
+
+
+class _NullPicturePartitioner:
+    """Does not parse the provided paragraph for pictures and generates zero `Image` elements."""
+
+    @classmethod
+    def iter_elements(cls, paragraph: Paragraph, opts: DocxPartitionerOptions) -> Iterator[Image]:
+        """No-op picture partitioner."""
+        return
+        yield

unstructured/partition/email.py

@@ -0,0 +1,441 @@
+"""Provides `partition_email()` function.
+
+Suitable for use with `.eml` files, which can be exported from many email clients.
+"""
+
+from __future__ import annotations
+
+import datetime as dt
+import email
+import email.policy
+import email.utils
+import io
+import os
+from email.message import EmailMessage, MIMEPart
+from functools import cached_property
+from typing import IO, Any, Final, Iterator, cast
+
+from dateutil import parser
+
+from unstructured.documents.elements import Element, ElementMetadata
+from unstructured.file_utils.model import FileType
+from unstructured.logger import logger
+from unstructured.partition.common import EXPECTED_ATTACHMENT_ERRORS
+from unstructured.partition.common.metadata import get_last_modified_date
+from unstructured.partition.html import partition_html
+from unstructured.partition.text import partition_text
+
+VALID_CONTENT_SOURCES: Final[tuple[str, ...]] = ("text/html", "text/plain")
+
+
+def partition_email(
+    filename: str | None = None,
+    *,
+    file: IO[bytes] | None = None,
+    content_source: str = "text/html",
+    metadata_filename: str | None = None,
+    metadata_last_modified: str | None = None,
+    process_attachments: bool = True,
+    **kwargs: Any,
+) -> list[Element]:
+    """Partitions an .eml file into document elements.
+
+    Args:
+        filename: str path of the target file.
+        file: A file-like object open for reading bytes (not str) e.g. --> open(filename, "rb").
+        content_source: The preferred message body. Many emails contain both a plain-text and an
+            HTML version of the message body. By default, the HTML version will be used when
+            available. Specifying "text/plain" will cause the plain-text version to be preferred.
+            When the preferred version is not available, the other version will be used.
+        metadata_filename: The file-path to use for metadata purposes. Useful when the target file
+            is specified as a file-like object or when `filename` is a temporary file and the
+            original file-path is known or a more meaningful file-path is desired.
+        metadata_last_modified: The last-modified timestamp to be applied in metadata. Useful when
+            a file-like object (which can have no last-modified date) target is used. The
+            last-modified metadata is otherwise drawn from the filesystem when a path is provided.
+        process_attachments: When True, also partition any attachments in the message after
+            partitioning the message body. All document elements appear in the single returned
+            element list. The filename of the attachment, when available, is used as the
+            `filename` metadata value for elements arising from the attachment.
+
+    Note that all global keyword arguments such as `unique_element_ids`, `language` and
+    `chunking_strategy` can be used and will be passed along to the decorators that implement
+    those functions. Further, any keyword arguments applicable to HTML will be passed along to the
+    HTML partitioner when processing an HTML message body.
+    """
+    ctx = EmailPartitioningContext.load(
+        file_path=filename,
+        file=file,
+        content_source=content_source,
+        metadata_file_path=metadata_filename,
+        metadata_last_modified=metadata_last_modified,
+        process_attachments=process_attachments,
+        kwargs=kwargs,
+    )
+
+    return list(_EmailPartitioner.iter_elements(ctx=ctx))
+
+
+class EmailPartitioningContext:
+    """Encapsulates partitioning option validation, computation, and application of defaults."""
+
+    def __init__(
+        self,
+        file_path: str | None = None,
+        file: IO[bytes] | None = None,
+        content_source: str = "text/html",
+        metadata_file_path: str | None = None,
+        metadata_last_modified: str | None = None,
+        process_attachments: bool = False,
+        kwargs: dict[str, Any] = {},
+    ):
+        self._file_path = file_path
+        self._file = file
+        self._content_source = content_source
+        self._metadata_file_path = metadata_file_path
+        self._metadata_last_modified = metadata_last_modified
+        self._process_attachments = process_attachments
+        self._kwargs = kwargs
+
+    @classmethod
+    def load(
+        cls,
+        file_path: str | None,
+        file: IO[bytes] | None,
+        content_source: str,
+        metadata_file_path: str | None,
+        metadata_last_modified: str | None,
+        process_attachments: bool,
+        kwargs: dict[str, Any],
+    ) -> EmailPartitioningContext:
+        """Construct and validate an instance."""
+        return cls(
+            file_path=file_path,
+            file=file,
+            content_source=content_source,
+            metadata_file_path=metadata_file_path,
+            metadata_last_modified=metadata_last_modified,
+            process_attachments=process_attachments,
+            kwargs=kwargs,
+        )._validate()
+
+    @cached_property
+    def bcc_addresses(self) -> list[str] | None:
+        """The "blind carbon-copy" Bcc: addresses of the message."""
+        bccs = self.msg.get_all("Bcc")
+        if not bccs:
+            return None
+        addrs = email.utils.getaddresses(bccs)
+        return [email.utils.formataddr(addr) for addr in addrs]
+
+    @cached_property
+    def body_part(self) -> MIMEPart | None:
+        """The message part containing the actual textual email message.
+
+        This is as opposed to attachments or "related" parts like an image that appears in the
+        message etc.
+        """
+        return self.msg.get_body(preferencelist=self.content_type_preference)
+
+    @cached_property
+    def cc_addresses(self) -> list[str] | None:
+        """The "carbon-copy" Cc: addresses of the message."""
+        ccs = self.msg.get_all("Cc")
+        if not ccs:
+            return None
+        addrs = email.utils.getaddresses(ccs)
+        return [email.utils.formataddr(addr) for addr in addrs]
+
+    @cached_property
+    def content_type_preference(self) -> tuple[str, ...]:
+        """Whether to prefer HTML or plain-text body when message-body has both.
+
+        The default order of preference is `("html", "plain")`. The order can be switched by
+        specifying `"text/plain"` as the `content_source` arg value.
+        """
+        return ("plain", "html") if self._content_source == "text/plain" else ("html", "plain")
+
+    @cached_property
+    def email_metadata(self) -> ElementMetadata:
+        """The email-specific metadata fields for this message.
+
+        Suitable for use with `.metadata.update()` on the base metadata applied to message body
+        elements by delegate partitioners for text and HTML.
+        """
+        return ElementMetadata(
+            bcc_recipient=self.bcc_addresses,
+            cc_recipient=self.cc_addresses,
+            email_message_id=self.message_id,
+            sent_from=[self.from_address] if self.from_address else None,
+            sent_to=self.to_addresses,
+            subject=self.subject,
+        )
+
+    @cached_property
+    def from_address(self) -> str | None:
+        """The address of the message sender."""
+        froms = self.msg.get_all("From")
+        if not froms:
+            # -- this should never occur because the From: header is mandatory per RFC 5322 --
+            return None
+        addrs = email.utils.getaddresses(froms)
+        formatted_addrs = [email.utils.formataddr(addr) for addr in addrs]
+        return formatted_addrs[0]
+
+    @cached_property
+    def message_id(self) -> str | None:
+        """The value of the Message-ID: header, when present."""
+        raw_id = self.msg.get("Message-ID")
+        if not raw_id:
+            return None
+        return raw_id.strip().strip("<>")
+
+    @cached_property
+    def metadata_file_path(self) -> str | None:
+        """The best available file-path information for this email message.
+
+        It's value is computed according to these rules, applied in order:
+
+          - The `metadata_filename` arg value when one was provided to `partition_email()`.
+          - The `file_path` value when one was provided.
+          - None otherwise.
+
+        This value is used as the `filename` metadata value for elements produced by partitioning
+        the email message (but not those from its attachments).
+        """
+        return self._metadata_file_path or self._file_path or None
+
+    @cached_property
+    def metadata_last_modified(self) -> str | None:
+        """The best available last-modified date for this message, as an ISO8601 string.
+
+        It's value is computed according to these rules, applied in order:
+
+          - The `metadata_last_modified` arg value when one was provided to `partition_email()`.
+          - The date-time in the `Sent:` header of the message, when present.
+          - The last-modified date recorded on the filesystem for `file_path` when it was provided.
+          - None otherwise.
+
+        This value is used as the `last_modified` metadata value for all elements produced by
+        partitioning this email message, including any attachments.
+        """
+        return self._metadata_last_modified or self._sent_date or self._filesystem_last_modified
+
+    @cached_property
+    def msg(self) -> EmailMessage:
+        """The Python stdlib `email.message.EmailMessage` object parsed from the EML file."""
+        if self._file_path is not None:
+            with open(self._file_path, "rb") as f:
+                return cast(
+                    EmailMessage, email.message_from_binary_file(f, policy=email.policy.default)
+                )
+
+        assert self._file is not None
+
+        file_bytes = self._file.read()
+
+        return cast(EmailMessage, email.message_from_bytes(file_bytes, policy=email.policy.default))
+
+    @cached_property
+    def partitioning_kwargs(self) -> dict[str, Any]:
+        """The "extra" keyword arguments received by `partition_email()`.
+
+        These are passed along to delegate partitioners which extract keyword args like
+        `chunking_strategy` etc. in their decorators to control metadata behaviors, etc.
+        """
+        return self._kwargs
+
+    @cached_property
+    def process_attachments(self) -> bool:
+        """When True, partition attachments in addition to the email message body.
+
+        Any attachment having file-format that cannot be partitioned by unstructured is silently
+        skipped.
+        """
+        return self._process_attachments
+
+    @cached_property
+    def subject(self) -> str | None:
+        """The value of the Subject: header, when present."""
+        subject = self.msg.get("Subject")
+        if not subject:
+            return None
+        return subject
+
+    @cached_property
+    def to_addresses(self) -> list[str] | None:
+        """The To: addresses of the message."""
+        tos = self.msg.get_all("To")
+        if not tos:
+            return None
+        addrs = email.utils.getaddresses(tos)
+        return [email.utils.formataddr(addr) for addr in addrs]
+
+    @cached_property
+    def _filesystem_last_modified(self) -> str | None:
+        """Last-modified retrieved from filesystem when a file-path was provided, None otherwise."""
+        return get_last_modified_date(self._file_path) if self._file_path else None
+
+    @cached_property
+    def _sent_date(self) -> str | None:
+        """ISO-8601 str representation of message sent-date, if available."""
+        date_str = self.msg.get("Date")
+        if not date_str:
+            return None
+        try:
+            sent_date = parser.parse(date_str)
+        except (parser.ParserError, TypeError, ValueError):
+            return None
+
+        return sent_date.astimezone(dt.timezone.utc).isoformat(timespec="seconds")
+
+    def _validate(self) -> EmailPartitioningContext:
+        """Raise on first invalid option, return self otherwise."""
+        if not self._file_path and not self._file:
+            raise ValueError(
+                "no document specified; either a `filename` or `file` argument must be provided."
+            )
+
+        if self._file:
+            if not isinstance(  # pyright: ignore[reportUnnecessaryIsInstance]
+                self._file.read(0), bytes
+            ):
+                raise ValueError("file object must be opened in binary mode")
+            self._file.seek(0)
+
+        if self._content_source not in VALID_CONTENT_SOURCES:
+            raise ValueError(
+                f"{repr(self._content_source)} is not a valid value for content_source;"
+                f" must be one of: {VALID_CONTENT_SOURCES}",
+            )
+
+        return self
+
+
+class _EmailPartitioner:
+    """Encapsulates the partitioning logic for email documents."""
+
+    def __init__(self, ctx: EmailPartitioningContext):
+        self._ctx = ctx
+
+    @classmethod
+    def iter_elements(cls, ctx: EmailPartitioningContext) -> Iterator[Element]:
+        """Generate the document elements for the email described by `ctx`."""
+        return cls(ctx=ctx)._iter_elements()
+
+    def _iter_elements(self) -> Iterator[Element]:
+        """Generate the document elements for the email described in the partitioning context.
+
+        This optionally includes elements generated by partitioning any partitionable attachments
+        in the message as well.
+        """
+        for e in self._iter_email_body_elements():
+            e.metadata.update(self._ctx.email_metadata)
+            yield e
+
+        if not self._ctx.process_attachments:
+            return
+
+        for attachment in self._ctx.msg.iter_attachments():
+            yield from _AttachmentPartitioner.iter_elements(attachment, self._ctx)
+
+    def _iter_email_body_elements(self) -> Iterator[Element]:
+        """Generate document elements from the email body."""
+        body_part = self._ctx.body_part
+
+        # -- it's possible to have no body part; that translates to zero elements --
+        if body_part is None:
+            return
+
+        content_type = body_part.get_content_type()
+        content = body_part.get_content()
+        assert isinstance(content, str)
+
+        if content_type == "text/html":
+            yield from partition_html(
+                text=content,
+                metadata_filename=self._ctx.metadata_file_path,
+                metadata_file_type=FileType.EML,
+                metadata_last_modified=self._ctx.metadata_last_modified,
+                **self._ctx.partitioning_kwargs,
+            )
+        else:
+            yield from partition_text(
+                text=content,
+                metadata_filename=self._ctx.metadata_file_path,
+                metadata_file_type=FileType.EML,
+                metadata_last_modified=self._ctx.metadata_last_modified,
+                **self._ctx.partitioning_kwargs,
+            )
+
+
+class _AttachmentPartitioner:
+    """Partitions an attachment to a MSG file."""
+
+    def __init__(self, attachment: EmailMessage, ctx: EmailPartitioningContext):
+        self._attachment = attachment
+        self._ctx = ctx
+
+    @classmethod
+    def iter_elements(
+        cls, attachment: EmailMessage, ctx: EmailPartitioningContext
+    ) -> Iterator[Element]:
+        """Partition an attachment MIME-part from a MIME email message (.eml file)."""
+        return cls(attachment, ctx)._iter_elements()
+
+    def _iter_elements(self) -> Iterator[Element]:
+        """Partition the byte-stream in the attachment MIME-part into elements.
+
+        Generates zero elements if the attachment is not partitionable.
+        """
+        # -- `auto.partition()` imports this module, so we need to defer the import to here to
+        # -- avoid a circular import.
+        from unstructured.partition.auto import partition
+
+        file = io.BytesIO(self._file_bytes)
+
+        # -- partition the attachment --
+        try:
+            elements = partition(
+                file=file,
+                metadata_filename=self._attachment_file_name,
+                metadata_last_modified=self._ctx.metadata_last_modified,
+                **self._ctx.partitioning_kwargs,
+            )
+        except BaseException as e:
+            if not isinstance(e, EXPECTED_ATTACHMENT_ERRORS):
+                raise
+            logger.warning(
+                "Skipping attachment %s: %s",
+                self._attachment_file_name,
+                f"{type(e).__name__}: {e}",
+            )
+            return
+
+        for e in elements:
+            e.metadata.attached_to_filename = self._attached_to_filename
+            yield e
+
+    @cached_property
+    def _attached_to_filename(self) -> str | None:
+        """The file-name (no path) of the message. `None` if not available."""
+        file_path = self._ctx.metadata_file_path
+        if file_path is None:
+            return None
+        return os.path.basename(file_path)
+
+    @cached_property
+    def _attachment_file_name(self) -> str | None:
+        """The original name of the attached file, `None` if not present in the MIME part."""
+        return self._attachment.get_filename()
+
+    @cached_property
+    def _file_bytes(self) -> bytes:
+        """The bytes of the attached file."""
+        content = self._attachment.get_content()
+
+        if isinstance(content, str):
+            return content.encode("utf-8")
+
+        assert isinstance(content, bytes)
+        return content

unstructured/partition/epub.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+from typing import IO, Any, Optional
+
+from unstructured.documents.elements import Element
+from unstructured.file_utils.file_conversion import convert_file_to_html_text_using_pandoc
+from unstructured.file_utils.model import FileType
+from unstructured.partition.common.common import exactly_one
+from unstructured.partition.common.metadata import get_last_modified_date
+from unstructured.partition.html import partition_html
+
+DETECTION_ORIGIN: str = "epub"
+
+
+def partition_epub(
+    filename: Optional[str] = None,
+    *,
+    file: Optional[IO[bytes]] = None,
+    metadata_filename: Optional[str] = None,
+    metadata_last_modified: Optional[str] = None,
+    languages: Optional[list[str]] = None,
+    detect_language_per_element: bool = False,
+    **kwargs: Any,
+) -> list[Element]:
+    """Partitions an EPUB document. The document is first converted to HTML and then
+    partitioned using partition_html.
+
+    Parameters
+    ----------
+    filename
+        A string defining the target filename path.
+    file
+        A file-like object using "rb" mode --> open(filename, "rb").
+    metadata_last_modified
+        The last modified date for the document.
+    languages
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
+        Additional Parameters:
+            detect_language_per_element
+                Detect language per element instead of at the document level.
+    """
+    exactly_one(filename=filename, file=file)
+
+    last_modified = get_last_modified_date(filename) if filename else None
+
+    html_text = convert_file_to_html_text_using_pandoc(
+        source_format="epub",
+        filename=filename,
+        file=file,
+    )
+
+    return partition_html(
+        text=html_text,
+        metadata_filename=metadata_filename or filename,
+        metadata_file_type=FileType.EPUB,
+        metadata_last_modified=metadata_last_modified or last_modified,
+        languages=languages,
+        detect_language_per_element=detect_language_per_element,
+        detection_origin=DETECTION_ORIGIN,
+        **kwargs,
+    )

unstructured/partition/html/__init__.py

@@ -0,0 +1,3 @@
+from unstructured.partition.html.partition import partition_html
+
+__all__ = ["partition_html"]

unstructured/partition/html/convert.py

@@ -0,0 +1,346 @@
+import logging
+from abc import ABC
+from collections import defaultdict
+from typing import Any, Optional, Union
+
+from bs4 import BeautifulSoup, Tag
+
+from unstructured.documents.elements import Element, ElementType
+from unstructured.documents.html_sanitization import sanitize_html_fragment
+
+logger = logging.getLogger(__name__)
+
+HTML_PARSER = "html.parser"
+HTML_TEMPLATE = """
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title></title>
+</head>
+<body>
+</body>
+</html>
+"""
+
+TABLE_BORDER_STYLE = "border: 1px solid black;"
+TABLE_BORDER_COLLAPSE_STYLE = "border-collapse: collapse;"
+
+
+class ElementHtml(ABC):
+    element: Element
+    children: list["ElementHtml"]
+    _html_tag: str = "div"
+
+    def __init__(self, element: Element, children: Optional[list["ElementHtml"]] = None):
+        self.element = element
+        self.children = children or []
+
+    @property
+    def html_tag(self) -> str:
+        return self._html_tag
+
+    def _inject_html_element_attrs(self, element_html: Tag) -> None:
+        return None
+
+    def _inject_html_element_content(self, element_html: Tag, **kwargs: Any) -> None:
+        element_html.string = self.element.text
+
+    def get_text_as_html(self) -> Union[Tag, None]:
+        element_html = BeautifulSoup(self.element.metadata.text_as_html or "", HTML_PARSER).find()
+        if not isinstance(element_html, Tag):
+            return None
+        return element_html
+
+    def _get_children_html(self, soup: BeautifulSoup, element_html: Tag, **kwargs: Any) -> Tag:
+        wrapper = soup.new_tag(name="div")
+        wrapper.append(element_html)
+        for child in self.children:
+            child_html = child.get_html_element(_soup=soup, **kwargs)
+            wrapper.append(child_html)
+        return wrapper
+
+    def get_html_element(self, **kwargs: Any) -> Tag:
+        soup: Optional[BeautifulSoup] = kwargs.pop("_soup", None)
+        if soup is None:
+            soup = BeautifulSoup("", HTML_PARSER)
+
+        element_html = self.get_text_as_html()
+        if element_html is None:
+            element_html = soup.new_tag(name=self.html_tag)
+            self._inject_html_element_content(element_html, **kwargs)
+        element_html["class"] = self.element.category
+        element_html["id"] = self.element.id
+        self._inject_html_element_attrs(element_html)
+        if self.children:  # if element has children wrap it with a 'div' tag
+            return self._get_children_html(soup, element_html, **kwargs)
+        return element_html
+
+    def set_children(self, children: list["ElementHtml"]) -> None:
+        self.children = children
+
+
+class TitleElementHtml(ElementHtml):
+    _html_tag = "h%d"
+
+    @property
+    def html_tag(self) -> str:
+        return self._html_tag % (self.element.metadata.category_depth or 1)
+
+
+class ImageElementHtml(ElementHtml):
+    _html_tag = "img"
+
+    def _inject_html_element_content(self, element_html: Tag, **kwargs: Any) -> None:
+        exclude_binary_image_data = kwargs.get("exclude_binary_image_data", False)
+        if self.element.metadata.image_base64 and not exclude_binary_image_data:
+            image_mime_type = self.element.metadata.image_mime_type or "image/png"
+            element_html["src"] = (
+                f"data:{image_mime_type};base64,{self.element.metadata.image_base64}"
+            )
+        element_html["alt"] = self.element.text
+
+
+class TableElementHtml(ElementHtml):
+    _html_tag = "table"
+
+    def _inject_html_element_content(self, element_html: Tag, **kwargs: Any) -> None:
+        # -- A table with no `text_as_html` carries only raw text. Wrap it in a
+        # -- proper row/cell so the markup is valid; loose text placed directly in
+        # -- a `<table>` is otherwise foster-parented out of the table by HTML5
+        # -- parsers (e.g. the nh3 sanitization pass), leaving the table empty. --
+        if not self.element.text:
+            return
+        soup = BeautifulSoup("", HTML_PARSER)
+        tbody = soup.new_tag(name="tbody")
+        row = soup.new_tag(name="tr")
+        cell = soup.new_tag(name="td")
+        cell.string = self.element.text
+        row.append(cell)
+        tbody.append(row)
+        element_html.append(tbody)
+
+    def _inject_html_element_attrs(self, element_html: Tag) -> None:
+        element_html["style"] = f"{TABLE_BORDER_STYLE} {TABLE_BORDER_COLLAPSE_STYLE}"
+        for tag in element_html.find_all(["tr", "th", "td"]):
+            tag["style"] = TABLE_BORDER_STYLE
+
+
+class LinkElementHtml(ElementHtml):
+    _html_tag = "a"
+
+    def _inject_html_element_attrs(self, element_html: Tag) -> None:
+        element_html["href"] = self.element.metadata.url or ""
+
+
+class TextElementHtml(ElementHtml):
+    _html_tag = "p"
+
+
+class UnorderedListElementHtml(ElementHtml):
+    _html_tag = "ul"
+
+    def _get_children_html(self, soup: BeautifulSoup, element_html: Tag, **kwargs: Any) -> Tag:
+        for child in self.children:
+            child_html = child.get_html_element(**kwargs)
+            element_html.append(child_html)
+        return element_html
+
+
+class OrderedListElementHtml(UnorderedListElementHtml):
+    _html_tag = "ol"
+
+
+class ListItemElementHtml(UnorderedListElementHtml):
+    _html_tag = "li"
+
+
+class LabelElementHtml(ElementHtml):
+    _html_tag = "label"
+
+
+class FormElementHtml(ElementHtml):
+    _html_tag = "form"
+
+
+class InputElementHtml(ElementHtml):
+    _html_tag = "input"
+
+
+class CheckboxElementHtml(InputElementHtml):
+    def _inject_html_element_attrs(self, element_html: Tag) -> None:
+        element_html["type"] = "checkbox"
+
+
+class CheckboxCheckedElementHtml(InputElementHtml):
+    def _inject_html_element_attrs(self, element_html: Tag) -> None:
+        element_html["type"] = "checkbox"
+        element_html["checked"] = "true"
+
+
+class RadioElementHtml(InputElementHtml):
+    def _inject_html_element_attrs(self, element_html: Tag) -> None:
+        element_html["type"] = "radio"
+
+
+class RadioCheckedElementHtml(InputElementHtml):
+    def _inject_html_element_attrs(self, element_html: Tag) -> None:
+        element_html["type"] = "radio"
+        element_html["checked"] = "true"
+
+
+LIST_ELEMENTS = [ElementType.LIST_ITEM, ElementType.LIST_ITEM_OTHER]
+
+TYPE_TO_HTML_MAP = {
+    ElementType.UNCATEGORIZED_TEXT: TextElementHtml,
+    ElementType.TITLE: TitleElementHtml,
+    ElementType.IMAGE: ImageElementHtml,
+    ElementType.TABLE: TableElementHtml,
+    ElementType.LINK: LinkElementHtml,
+    ElementType.TEXT: TextElementHtml,
+    ElementType.PARAGRAPH: TextElementHtml,
+    ElementType.LIST: OrderedListElementHtml,
+    ElementType.LIST_ITEM: ListItemElementHtml,
+    ElementType.LIST_ITEM_OTHER: ListItemElementHtml,
+    ElementType.FIELD_NAME: LabelElementHtml,
+    ElementType.BULLETED_TEXT: ListItemElementHtml,
+    ElementType.FORM: FormElementHtml,
+    ElementType.CAPTION: TextElementHtml,
+    ElementType.CHECKED: CheckboxCheckedElementHtml,
+    ElementType.UNCHECKED: CheckboxElementHtml,
+    ElementType.CHECK_BOX_CHECKED: CheckboxCheckedElementHtml,
+    ElementType.CHECK_BOX_UNCHECKED: CheckboxElementHtml,
+    ElementType.RADIO_BUTTON_CHECKED: RadioCheckedElementHtml,
+    ElementType.RADIO_BUTTON_UNCHECKED: RadioElementHtml,
+    ElementType.NARRATIVE_TEXT: TextElementHtml,
+    ElementType.FIGURE_CAPTION: TextElementHtml,
+    ElementType.VALUE: InputElementHtml,
+    ElementType.ABSTRACT: ElementHtml,
+    ElementType.THREADING: ElementHtml,
+    ElementType.COMPOSITE_ELEMENT: ElementHtml,
+    ElementType.PICTURE: ElementHtml,
+    ElementType.FIGURE: ElementHtml,
+    ElementType.ADDRESS: ElementHtml,
+    ElementType.EMAIL_ADDRESS: ElementHtml,
+    ElementType.PAGE_BREAK: ElementHtml,
+    ElementType.FORMULA: ElementHtml,
+    ElementType.HEADER: ElementHtml,
+    ElementType.HEADLINE: ElementHtml,
+    ElementType.SUB_HEADLINE: ElementHtml,
+    ElementType.PAGE_HEADER: ElementHtml,
+    ElementType.SECTION_HEADER: ElementHtml,
+    ElementType.FOOTER: ElementHtml,
+    ElementType.FOOTNOTE: ElementHtml,
+    ElementType.PAGE_FOOTER: ElementHtml,
+    ElementType.PAGE_NUMBER: ElementHtml,
+    ElementType.CODE_SNIPPET: ElementHtml,
+    ElementType.FORM_KEYS_VALUES: ElementHtml,
+    ElementType.DOCUMENT_DATA: ElementHtml,
+}
+
+
+def _group_element_children(children: list[ElementHtml]) -> list[ElementHtml]:
+    grouped_children: list[ElementHtml] = []
+    temp_group: list["ElementHtml"] = []
+    prev_grouping = False
+    for child in children:
+        grouping = child.element.category in LIST_ELEMENTS
+        if grouping:
+            temp_group.append(child)
+        elif prev_grouping:
+            grouped_children.append(OrderedListElementHtml(Element(), temp_group))
+            grouped_children.append(child)
+            temp_group = []
+        else:
+            grouped_children.append(child)
+        prev_grouping = grouping
+    if temp_group:
+        grouped_children.append(OrderedListElementHtml(Element(), temp_group))
+    return grouped_children
+
+
+def _elements_to_html_tags_by_parent(elements: list[ElementHtml]) -> list[ElementHtml]:
+    parent_to_children_map: dict[str, list[ElementHtml]] = defaultdict(list)
+    for element in elements:
+        if element.element.metadata.parent_id is not None:
+            parent_to_children_map[element.element.metadata.parent_id].append(element)
+    for parent_id, children in parent_to_children_map.items():
+        grouped_children = _group_element_children(children)
+        parent = next((el for el in elements if el.element.id == parent_id), None)
+        if parent is None:
+            logger.warning(f"Parent element with id {parent_id} not found. Skipping.")
+            continue
+        parent.set_children(grouped_children)
+    return [el for el in elements if el.element.metadata.parent_id is None]
+
+
+def _elements_to_html_tags(
+    elements: list[Element], exclude_binary_image_data: bool = False
+) -> list[Tag]:
+    elements_html = [
+        TYPE_TO_HTML_MAP.get(element.category, ElementHtml)(element) for element in elements
+    ]
+    elements_html = _elements_to_html_tags_by_parent(elements_html)
+    return [
+        element_html.get_html_element(exclude_binary_image_data=exclude_binary_image_data)
+        for element_html in elements_html
+    ]
+
+
+def _elements_to_html_tags_by_page(
+    elements: list[Element], exclude_binary_image_data: bool = False
+) -> list[Tag]:
+    soup = BeautifulSoup("", HTML_PARSER)
+    pages_tags: list[Tag] = []
+    grouped_elements = group_elements_by_page(elements)
+    for page, g_elements in enumerate(grouped_elements, start=1):
+        page_html = soup.new_tag(name="div", attrs={"data-page_number": page})
+        elements_html = _elements_to_html_tags(g_elements, exclude_binary_image_data)
+        for element_html in elements_html:
+            page_html.append(element_html)
+        pages_tags.append(page_html)
+    return pages_tags
+
+
+def group_elements_by_page(
+    unstructured_elements: list[Element],
+) -> list[list[Element]]:
+    pages_dict: defaultdict[int, list[Element]] = defaultdict(list)
+
+    for element in unstructured_elements:
+        page_number = element.metadata.page_number
+        if page_number is None:
+            logger.warning(f"Page number is not set for an element {element.id}. Skipping.")
+            continue
+        pages_dict[page_number].append(element)
+
+    pages_list = list(pages_dict.values())
+    return pages_list
+
+
+def elements_to_html(
+    elements: list[Element],
+    exclude_binary_image_data: bool = False,
+    no_group_by_page: bool = False,
+) -> str:
+    soup = BeautifulSoup(HTML_TEMPLATE, HTML_PARSER)
+    if soup.body is None:
+        raise ValueError("Body tag not found in the HTML template")
+    elements_html = (
+        _elements_to_html_tags(elements, exclude_binary_image_data)
+        if no_group_by_page
+        else _elements_to_html_tags_by_page(elements, exclude_binary_image_data)
+    )
+    # -- Defense-in-depth (GHSA-v5mq-3xhg-98m9): sanitize the assembled body content
+    # -- through nh3 before returning. This neutralizes any dangerous markup that
+    # -- survived into `text_as_html` as well as attributes injected here directly
+    # -- (e.g. an `href` built from `metadata.url`). We sanitize the fragment and
+    # -- reinsert it, since nh3 strips the surrounding <html>/<head>/<body> wrapper. --
+    body_fragment = "".join(str(element_html) for element_html in elements_html)
+    sanitized_fragment = sanitize_html_fragment(body_fragment)
+    # -- materialize the parsed nodes before appending: `append` moves a node out
+    # -- of the source soup's `.contents`, so iterating it live would skip nodes. --
+    for element_html in list(BeautifulSoup(sanitized_fragment, HTML_PARSER).contents):
+        soup.body.append(element_html)
+    return soup.prettify()

unstructured/partition/html/html_utils.py

@@ -0,0 +1,25 @@
+from bs4 import BeautifulSoup
+
+
+def indent_html(html_string: str, html_parser="html.parser") -> str:
+    """
+    Formats / indents HTML.
+
+    This function takes an HTML string and formats it using the specified HTML parser.
+    It parses the HTML content and returns a prettified version of it.
+
+    Args:
+        html_string (str): The HTML content to be formatted.
+        html_parser (str, optional): The parser to use for parsing the HTML. Defaults to 'html5lib':
+            - 'html.parser': The built-in HTML parser. Use when you need just parsing
+            - 'html5lib': The slowest. Use when you expect valid HTML parsed
+                          the same way a browser does. It adds some extra
+                          tags and attributes like <html>, <head>, <body>
+            More in docs https://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-a-parser
+
+    Returns:
+        str: The formatted and indented HTML content.
+    """
+    soup = BeautifulSoup(html_string, html_parser)
+    pretty_html = soup.prettify()
+    return pretty_html

unstructured/partition/html/partition.py

@@ -0,0 +1,292 @@
+# pyright: reportPrivateUsage=false
+
+"""Provides `partition_html()."""
+
+from __future__ import annotations
+
+from functools import cached_property
+from typing import IO, Any, Callable, Iterator, List, Literal, Optional, cast
+
+from lxml import etree
+
+from unstructured.chunking import add_chunking_strategy
+from unstructured.documents.elements import Element, ElementType
+from unstructured.file_utils.encoding import read_txt_file
+from unstructured.file_utils.model import FileType
+from unstructured.partition.common.metadata import apply_metadata, get_last_modified_date
+from unstructured.partition.html.parser import Flow, html_parser
+from unstructured.partition.html.transformations import (
+    ontology_to_unstructured_elements,
+    parse_html_to_ontology,
+)
+from unstructured.safe_http import safe_get
+from unstructured.utils import is_temp_file_path
+
+
+@apply_metadata(FileType.HTML)
+@add_chunking_strategy
+def partition_html(
+    filename: Optional[str] = None,
+    *,
+    file: Optional[IO[bytes]] = None,
+    text: Optional[str] = None,
+    encoding: Optional[str] = None,
+    url: Optional[str] = None,
+    headers: dict[str, str] = {},
+    ssl_verify: bool = True,
+    skip_headers_and_footers: bool = False,
+    detection_origin: Optional[str] = None,
+    html_parser_version: Literal["v1", "v2"] = "v1",
+    image_alt_mode: Optional[Literal["to_text"]] = "to_text",
+    extract_image_block_to_payload: bool = False,
+    extract_image_block_types: Optional[list[str]] = None,
+    languages: Optional[list[str]] = None,
+    detect_language_per_element: bool = False,
+    language_fallback: Optional[Callable[[str], Optional[list[str]]]] = None,
+    **kwargs: Any,
+) -> list[Element]:
+    """Partitions an HTML document into its constituent elements.
+
+    HTML source parameters
+    ----------------------
+    The HTML to be partitioned can be specified four different ways:
+
+    filename
+        A string defining the target filename path.
+    file
+        A file-like object using "r" mode --> open(filename, "r").
+    text
+        The string representation of the HTML document.
+    url
+        The URL of a webpage to parse. Only for URLs that return an HTML document.
+    headers
+        The HTTP headers to be used in the HTTP request when `url` is specified.
+    ssl_verify
+        If the URL parameter is set, determines whether or not SSL verification is performed
+        on the HTTP request.
+    encoding
+        The encoding method used to decode the text input. If None, utf-8 will be used.
+    skip_headers_and_footers
+        If True, ignores any content that is within <header> or <footer> tags
+
+    html_parser_version (Literal['v1', 'v2']):
+        The version of the HTML parser to use. The default is 'v1'. For 'v2' the parser will
+        use the ontology schema to parse the HTML document.
+
+    image_alt_mode (Literal['to_text']):
+        When set 'to_text', the v2 parser will include the alternative text of images in the output.
+
+    languages
+        The languages present in the document. Use ``["auto"]`` to detect; use ``[""]`` to disable.
+    detect_language_per_element
+        Detect language per element instead of at the document level.
+    language_fallback
+        Optional callable for short text; called with the text, return ISO 639-3 codes or None.
+    """
+    # -- parser rejects an empty str, nip that edge-case in the bud here --
+    if text is not None and text.strip() == "" and not file and not filename and not url:
+        return []
+
+    opts = HtmlPartitionerOptions(
+        file_path=filename,
+        file=file,
+        text=text,
+        encoding=encoding,
+        url=url,
+        headers=headers,
+        ssl_verify=ssl_verify,
+        skip_headers_and_footers=skip_headers_and_footers,
+        detection_origin=detection_origin,
+        html_parser_version=html_parser_version,
+        image_alt_mode=image_alt_mode,
+        extract_image_block_types=extract_image_block_types,
+        extract_image_block_to_payload=extract_image_block_to_payload,
+    )
+
+    return list(_HtmlPartitioner.iter_elements(opts))
+
+
+class HtmlPartitionerOptions:
+    """Encapsulates partitioning option validation, computation, and application of defaults."""
+
+    def __init__(
+        self,
+        *,
+        file_path: str | None,
+        file: IO[bytes] | None,
+        text: str | None,
+        encoding: str | None,
+        url: str | None,
+        headers: dict[str, str],
+        ssl_verify: bool,
+        skip_headers_and_footers: bool,
+        detection_origin: str | None,
+        html_parser_version: Literal["v1", "v2"] = "v1",
+        image_alt_mode: Optional[Literal["to_text"]] = "to_text",
+        extract_image_block_types: Optional[list[str]] = None,
+        extract_image_block_to_payload: bool = False,
+    ):
+        self._file_path = file_path
+        self._file = file
+        self._text = text
+        self._encoding = encoding
+        self._url = url
+        self._headers = headers
+        self._ssl_verify = ssl_verify
+        self._skip_headers_and_footers = skip_headers_and_footers
+        self._detection_origin = detection_origin
+        self._html_parser_version = html_parser_version
+        self._image_alt_mode = image_alt_mode
+        self._extract_image_block_types = extract_image_block_types
+        self._extract_image_block_to_payload = extract_image_block_to_payload
+
+    @cached_property
+    def detection_origin(self) -> str | None:
+        """Trace of initial partitioner to be included in metadata for debugging purposes."""
+        return self._detection_origin
+
+    @cached_property
+    def html_text(self) -> str:
+        """The HTML document as a string, loaded from wherever the caller specified."""
+        if self._file_path:
+            return read_txt_file(filename=self._file_path, encoding=self._encoding)[1]
+
+        if self._file:
+            return read_txt_file(file=self._file, encoding=self._encoding)[1]
+
+        if self._text:
+            return str(self._text)
+
+        if self._url:
+            response = safe_get(self._url, headers=self._headers, verify=self._ssl_verify)
+            if not response.ok:
+                raise ValueError(
+                    f"Error status code on GET of provided URL: {response.status_code}"
+                )
+            content_type = response.headers.get("Content-Type", "")
+            if not content_type.startswith("text/html"):
+                raise ValueError(f"Expected content type text/html. Got {content_type}.")
+
+            return response.text
+
+        raise ValueError("Exactly one of filename, file, text, or url must be specified.")
+
+    @cached_property
+    def last_modified(self) -> str | None:
+        """The best last-modified date available, None if no sources are available."""
+        return (
+            None
+            if not self._file_path or is_temp_file_path(self._file_path)
+            else get_last_modified_date(self._file_path)
+        )
+
+    @cached_property
+    def skip_headers_and_footers(self) -> bool:
+        """When True, elements located within a header or footer are pruned."""
+        return self._skip_headers_and_footers
+
+    @cached_property
+    def html_parser_version(self) -> Literal["v1", "v2"]:
+        """When html_parser_version=='v2', HTML elements follow ontology schema."""
+        return self._html_parser_version
+
+    @cached_property
+    def add_img_alt_text(self) -> bool:
+        """When True, the alternative text of images is included in the output."""
+        return self._image_alt_mode == "to_text"
+
+
+class _HtmlPartitioner:
+    """Partition HTML document into document-elements."""
+
+    def __init__(self, opts: HtmlPartitionerOptions):
+        self._opts = opts
+
+    def _should_include_image_base64(self, element: Element) -> bool:
+        """Determines if an image_base64 element should be included in the output."""
+        return (
+            element.category == ElementType.IMAGE
+            and self._opts._extract_image_block_to_payload
+            and self._opts._extract_image_block_types is not None
+            and "Image" in self._opts._extract_image_block_types
+        )
+
+    @classmethod
+    def iter_elements(cls, opts: HtmlPartitionerOptions) -> Iterator[Element]:
+        """Partition HTML document provided by `opts` into document-elements."""
+        yield from cls(opts)._iter_elements()
+
+    def _iter_elements(self) -> Iterator[Element]:
+        """Generated document-elements (e.g. Title, NarrativeText, etc.) parsed from document.
+
+        Elements appear in document order.
+        """
+        # -- handle empty or whitespace-only HTML content --
+        html_text = self._opts.html_text
+        if not html_text or html_text.strip() == "":
+            return
+
+        elements_iter = (
+            self._main.iter_elements()
+            if self._opts.html_parser_version == "v1"
+            else self._from_ontology
+        )
+
+        for e in elements_iter:
+            e.metadata.last_modified = self._opts.last_modified
+            e.metadata.detection_origin = self._opts.detection_origin
+
+            # -- remove <image_base64> if not requested --
+            if not self._should_include_image_base64(e):
+                e.metadata.image_base64 = None
+                e.metadata.image_mime_type = None
+            yield e
+
+    @cached_property
+    def _main(self) -> Flow:
+        """The root HTML element."""
+        # NOTE(scanny) - get `html_text` first so any encoding error raised is not confused with a
+        # recoverable parsing error.
+        html_text = self._opts.html_text
+
+        # NOTE(scanny) - `lxml` will not parse a `str` that includes an XML encoding declaration
+        # and will raise the following error:
+        #     ValueError: Unicode strings with encoding declaration are not supported. ...
+        # This is not valid HTML (would be in XHTML), but Chrome accepts it so we work around it
+        # by UTF-8 encoding the str bytes and parsing those.
+        try:
+            root = etree.fromstring(html_text, html_parser)
+        except ValueError:
+            root = etree.fromstring(html_text.encode("utf-8"), html_parser)
+
+        # -- remove a variety of HTML element types like <script> and <style> that we prefer not
+        # -- to encounter while parsing.
+        etree.strip_elements(
+            root, ["del", "link", "meta", "noscript", "script", "style"], with_tail=False
+        )
+
+        # -- remove <header> and <footer> tags if the caller doesn't want their contents --
+        if self._opts.skip_headers_and_footers:
+            etree.strip_elements(root, ["header", "footer"], with_tail=False)
+
+        # -- jump to the core content if the document indicates where it is --
+        if (main := root.find(".//main")) is not None:
+            return cast(Flow, main)
+        if (body := root.find(".//body")) is not None:
+            return cast(Flow, body)
+        return cast(Flow, root)
+
+    @cached_property
+    def _from_ontology(self) -> List[Element]:
+        """Convert an ontology elements represented in HTML to an ontology element."""
+        html_text = self._opts.html_text
+
+        # -- handle empty or whitespace-only HTML content --
+        if not html_text or html_text.strip() == "":
+            return []
+
+        ontology = parse_html_to_ontology(html_text)
+        unstructured_elements = ontology_to_unstructured_elements(
+            ontology, add_img_alt_text=self._opts.add_img_alt_text
+        )
+        return unstructured_elements

unstructured/partition/html/transformations.py

@@ -0,0 +1,590 @@
+from __future__ import annotations
+
+from itertools import chain
+from typing import Sequence, Type
+
+from bs4 import BeautifulSoup, Tag
+
+from unstructured.documents import elements, ontology
+from unstructured.documents.html_sanitization import sanitize_attributes
+from unstructured.documents.mappings import (
+    CSS_CLASS_TO_ELEMENT_TYPE_MAP,
+    HTML_TAG_AND_CSS_NAME_TO_ELEMENT_TYPE_MAP,
+    HTML_TAG_TO_DEFAULT_ELEMENT_TYPE_MAP,
+    ONTOLOGY_CLASS_TO_UNSTRUCTURED_ELEMENT_TYPE,
+)
+from unstructured.partition.common.metadata import (
+    HEADING_TAGS,
+    category_depth_from_html_tag,
+)
+
+RECURSION_LIMIT = 50
+
+
+def ontology_to_unstructured_elements(
+    ontology_element: ontology.OntologyElement,
+    parent_id: str | None = None,
+    page_number: int | None = None,
+    depth: int = 0,
+    filename: str | None = None,
+    add_img_alt_text: bool = True,
+) -> list[elements.Element]:
+    """
+    Converts an OntologyElement object to a list of unstructured Element objects.
+
+    To preserve the structure of the ontology, the function is recursive
+    and the tree structure is represented in flatten list by the parent_id
+    attribute in the metadata of each Element object.
+    To preserve all the attributes of the ontology element, the HTML code
+    is injected to unstructured Element in ElementMetadata.text_as_html attribute.
+
+    For Layout elements, the function creates an empty Text Element (with the
+    HTML code injected the same way).
+
+    TODO (Pluto): Better way would be to have special Element type in Unstructured
+
+    Args:
+        ontology_element (OntologyElement): The ontology element to be converted.
+        parent_id (str, optional): The ID of the parent element. Defaults to None.
+        page_number (int, optional): The page number of the element. Defaults to None.
+        depth (int, optional): The depth of the element in the hierarchy. Defaults to 0.
+        filename (str, optional): The name of the file the element comes from. Defaults to None.
+        add_img_alt_text (bool): Whether to include the alternative text of images
+                                            in the output. Defaults to True.
+    Returns:
+        list[Element]: A list of unstructured Element objects.
+
+    Note on `category_depth` and `parent_id` (ML-1328):
+        `category_depth` is derived from the element's HTML *heading level* (h1 -> 0, h2 -> 1, ...)
+        via the shared `category_depth_from_html_tag` helper, NOT from DOM/recursion nesting depth.
+        This keeps the v2 (ontology) parser consistent with the v1 parser and with the documented
+        metadata contract, and makes depth independent of layout (e.g. multi-column pages no longer
+        bump every element's depth).
+
+        `parent_id` is left to the metadata layer, like every other partitioner. Layout/container
+        elements (Page, Column, ...) keep their tree parent so the physical layout structure is
+        preserved; content elements are emitted with ``parent_id=None``. The `@apply_metadata`
+        decorator that wraps `partition_html` then runs `set_element_hierarchy`, which fills each
+        content element's heading-based parent (a subsection's parent becomes its enclosing heading)
+        from the heading-level `category_depth` and skips the containers that already have a parent.
+        Both production callers -- `partition_html` and the VLM partitioner -- go through that
+        decorator, so this converter does not run `set_element_hierarchy` itself.
+    """
+    # -- The worker carries each element's DOM-nesting depth alongside it (used only to decide
+    # -- inline merging); strip those depths here so the public output is plain Elements. --
+    elements_with_depth = _ontology_to_unstructured_elements(
+        ontology_element,
+        parent_id=parent_id,
+        page_number=page_number,
+        depth=depth,
+        filename=filename,
+        add_img_alt_text=add_img_alt_text,
+    )
+    return [element for element, _nesting_depth in elements_with_depth]
+
+
+def _ontology_to_unstructured_elements(
+    ontology_element: ontology.OntologyElement,
+    parent_id: str | None = None,
+    page_number: int | None = None,
+    depth: int = 0,
+    filename: str | None = None,
+    add_img_alt_text: bool = True,
+) -> list[tuple[elements.Element, int]]:
+    """Recursive worker for `ontology_to_unstructured_elements`.
+
+    Builds the flat element list with layout-container `parent_id` set to the tree parent and
+    content `parent_id` left as ``None`` -- the `@apply_metadata` decorator on `partition_html`
+    fills in content elements' heading-based `parent_id` via `set_element_hierarchy`.
+
+    Each element is returned paired with its DOM-nesting `depth`. That depth is recursion-local
+    bookkeeping consumed only by `combine_inline_elements` (to gate inline merging by tree level);
+    it is deliberately NOT stored on the element or its `ElementMetadata`, and the public wrapper
+    discards it.
+    """
+    elements_to_return: list[tuple[elements.Element, int]] = []
+    if ontology_element.elementType == ontology.ElementTypeEnum.layout and depth <= RECURSION_LIMIT:
+        if page_number is None and isinstance(ontology_element, ontology.Page):
+            page_number = ontology_element.page_number
+
+        if not isinstance(ontology_element, ontology.Document):
+            # -- Layout/container element (Page, Column, ...). Keep its tree `parent_id` so the
+            # -- physical layout structure is preserved, and leave `category_depth` unset -- a
+            # -- container is not a heading. --
+            container_element = elements.Text(
+                text="",
+                element_id=ontology_element.id,
+                detection_origin="vlm_partitioner",
+                metadata=elements.ElementMetadata(
+                    parent_id=parent_id,
+                    text_as_html=ontology_element.to_html(add_children=False),
+                    page_number=page_number,
+                    category_depth=None,
+                    filename=filename,
+                ),
+            )
+            # -- pair the container with its DOM-nesting depth, used only to decide inline merging;
+            # -- `category_depth` now carries heading level, not nesting, so it can't be reused. --
+            elements_to_return += [(container_element, depth)]
+        children: list[tuple[elements.Element, int]] = []
+        for child in ontology_element.children:
+            child = _ontology_to_unstructured_elements(
+                child,
+                parent_id=ontology_element.id,
+                page_number=page_number,
+                depth=0 if isinstance(ontology_element, ontology.Document) else depth + 1,
+                filename=filename,
+                add_img_alt_text=add_img_alt_text,
+            )
+            children += child
+
+        combined_children = combine_inline_elements(children)
+        elements_to_return += combined_children
+    else:
+        element_class: type[elements.Element] = ONTOLOGY_CLASS_TO_UNSTRUCTURED_ELEMENT_TYPE[
+            ontology_element.__class__
+        ]
+        html_code_of_ontology_element = ontology_element.to_html()
+        element_text = ontology_element.to_text(add_img_alt_text=add_img_alt_text)
+
+        # -- `category_depth` from heading level (not nesting depth); see function docstring. --
+        category_depth = category_depth_from_html_tag(
+            element_class,
+            ontology_element.html_tag_name,
+        )
+
+        unstructured_element = element_class(
+            text=element_text,  # type: ignore
+            element_id=ontology_element.id,
+            detection_origin="vlm_partitioner",
+            metadata=elements.ElementMetadata(
+                # -- `parent_id` left unset; `@apply_metadata` runs `set_element_hierarchy` to
+                # -- assign a heading-based parent (see the docstring). --
+                parent_id=None,
+                text_as_html=html_code_of_ontology_element,
+                page_number=page_number,
+                category_depth=category_depth,
+                filename=filename,
+            ),
+        )
+        elements_to_return = [(unstructured_element, depth)]
+
+    return elements_to_return
+
+
+def combine_inline_elements(
+    elements_with_depth: list[tuple[elements.Element, int]],
+) -> list[tuple[elements.Element, int]]:
+    """
+    Combines consecutive inline elements into a single element. Inline elements
+    can be also combined with text elements.
+
+    Combined elements contains multiple HTML tags together eg.
+    {
+        'text': "Text from element 1 Text from element 2",
+        'metadata': {
+            'text_as_html': "<p>Text from element 1</p><a>Text from element 2</a>"
+        }
+    }
+
+    Each element is paired with its DOM-nesting depth; merging is only allowed between elements at
+    the same depth (see `can_unstructured_elements_be_merged`). The depth travels with the element
+    rather than being stored on it.
+
+    Args:
+        elements_with_depth (list[tuple[Element, int]]): (element, nesting-depth) pairs to combine.
+
+    Returns:
+        list[tuple[Element, int]]: The combined (element, nesting-depth) pairs.
+    """
+    result_elements: list[tuple[elements.Element, int]] = []
+
+    current: tuple[elements.Element, int] | None = None
+    for nxt in elements_with_depth:
+        if current is None:
+            current = nxt
+            continue
+
+        current_element, current_depth = current
+        next_element, next_depth = nxt
+        if can_unstructured_elements_be_merged(
+            current_element, next_element, current_depth=current_depth, next_depth=next_depth
+        ):
+            current_element.text += " " + next_element.text
+            current_element.metadata.text_as_html += next_element.metadata.text_as_html
+        else:
+            result_elements.append(current)
+            current = nxt
+
+    if current is not None:
+        result_elements.append(current)
+
+    return result_elements
+
+
+def can_unstructured_elements_be_merged(
+    current_element: elements.Element,
+    next_element: elements.Element,
+    *,
+    current_depth: int,
+    next_depth: int,
+) -> bool:
+    """
+    Elements can be merged when:
+    - They are on the same level in the HTML tree
+    - Neither of them has children
+    - All elements are inline elements or text element
+    """
+    # NOTE(ML-1328): "same level in the HTML tree" is the DOM-nesting depth, passed in alongside
+    # each element. It used to live on `category_depth`, but that field now carries heading level,
+    # so it can no longer be used as the nesting signal here.
+    if current_depth != next_depth:
+        return False
+
+    current_html_tags = BeautifulSoup(
+        current_element.metadata.text_as_html, "html.parser"
+    ).find_all(recursive=False)
+    next_html_tags = BeautifulSoup(next_element.metadata.text_as_html, "html.parser").find_all(
+        recursive=False
+    )
+
+    ontology_elements = [
+        parse_html_to_ontology_element(html_tag)
+        for html_tag in chain(current_html_tags, next_html_tags)
+    ]
+
+    for ontology_element in ontology_elements:
+        if ontology_element.children:
+            return False
+
+        if not (is_inline_element(ontology_element) or is_text_element(ontology_element)):
+            return False
+
+    return True
+
+
+def is_text_element(ontology_element: ontology.OntologyElement) -> bool:
+    """Categories or classes that we want to combine with inline text"""
+
+    text_classes = [
+        ontology.NarrativeText,
+        ontology.Quote,
+        ontology.Paragraph,
+        ontology.Footnote,
+        ontology.FootnoteReference,
+        ontology.Citation,
+        ontology.Bibliography,
+        ontology.Glossary,
+    ]
+    text_categories = [ontology.ElementTypeEnum.metadata]
+
+    if any(isinstance(ontology_element, class_) for class_ in text_classes):
+        return True
+
+    return any(ontology_element.elementType == category for category in text_categories)
+
+
+def is_inline_element(ontology_element: ontology.OntologyElement) -> bool:
+    """Categories or classes that we want to combine with text elements"""
+
+    inline_classes = [ontology.Hyperlink]
+    inline_categories = [
+        ontology.ElementTypeEnum.specialized_text,
+        ontology.ElementTypeEnum.annotation,
+    ]
+
+    if any(isinstance(ontology_element, class_) for class_ in inline_classes):
+        return True
+
+    return any(ontology_element.elementType == category for category in inline_categories)
+
+
+def unstructured_elements_to_ontology(
+    unstructured_elements: Sequence[elements.Element],
+) -> ontology.OntologyElement:
+    """
+    Converts a sequence of unstructured Element objects to an OntologyElement object.
+
+    The function caches the elements in a dictionary and each element is assigned to its parent.
+    At the end the root element is popped from the dictionary and returned.
+
+    Such approach comes with limitations:
+        - The parent element has to be in the list before the child element
+
+    Args:
+        unstructured_elements (Sequence[Element]): The sequence of unstructured Element objects.
+
+    Returns:
+        OntologyElement: The converted OntologyElement object.
+    """
+    if not unstructured_elements:
+        # -- empty input -> empty Document; avoid an IndexError dereferencing element[0] --
+        return ontology.Document(
+            additional_attributes={"id": ontology.OntologyElement.generate_unique_id()}
+        )
+
+    root_element_id = unstructured_elements[0].metadata.parent_id
+
+    if root_element_id is None:
+        root_element_id = ontology.OntologyElement.generate_unique_id()
+        unstructured_elements[0].metadata.parent_id = root_element_id
+
+    root_element = ontology.Document(additional_attributes={"id": root_element_id})
+
+    # NOTE(ML-1328): Tree reconstruction is driven by the *layout-container* elements (Page,
+    # Column, Section, ...), which retain their tree `parent_id`. Content-element `parent_id` is no
+    # longer the tree parent -- it is the heading-based parent assigned by `set_element_hierarchy`
+    # -- so it must NOT be used to rebuild the layout tree. Instead, each content element is nested
+    # in the innermost open layout container, tracked with a stack keyed on the containers' own
+    # (tree) `parent_id`. This is independent of document content `parent_id` and reproduces the
+    # original layout nesting exactly.
+    container_stack: list[tuple[str, ontology.OntologyElement]] = [(root_element_id, root_element)]
+
+    for element in unstructured_elements:
+        # -- an element with no HTML payload carries nothing to rebuild the tree from;
+        # -- skip it per-element rather than letting BeautifulSoup(None) abort the whole
+        # -- reconstruction (e.g. mixed/partially-stripped element streams). --
+        if not element.metadata.text_as_html:
+            continue
+        html_as_tags = BeautifulSoup(element.metadata.text_as_html, "html.parser").find_all(
+            recursive=False
+        )
+        for html_as_tag in html_as_tags:
+            ontology_element = parse_html_to_ontology_element(html_as_tag)
+
+            is_layout_container = ontology_element.elementType == ontology.ElementTypeEnum.layout
+            if is_layout_container:
+                # -- pop back to this container's tree parent, then attach + open it. Only pop if
+                # -- that parent is actually open on the stack; a `parent_id` matching no open
+                # -- container (e.g. malformed/reordered input that violates the documented
+                # -- parent-before-child precondition) must not pop past valid ancestors to root --
+                # -- which would mis-nest later content. In that case attach to the current
+                # -- innermost container instead, preserving document order and losing nothing. --
+                # -- a container with no `parent_id` is a top-level container -> attach at root --
+                parent_id = element.metadata.parent_id or root_element_id
+                if any(container_id == parent_id for container_id, _ in container_stack):
+                    while len(container_stack) > 1 and container_stack[-1][0] != parent_id:
+                        container_stack.pop()
+                container_stack[-1][1].children.append(ontology_element)
+                container_stack.append((element.id, ontology_element))
+            else:
+                # -- content nests in the innermost currently-open layout container --
+                container_stack[-1][1].children.append(ontology_element)
+
+    return root_element
+
+
+def parse_html_to_ontology(html_code: str) -> ontology.OntologyElement:
+    """
+    Parses the given HTML code and converts it into an Element object.
+
+    Args:
+        html_code (str): The HTML code to be parsed.
+            Parsing HTML will start from <div class="Page">.
+    Returns:
+        OntologyElement: The parsed Element object.
+
+    Raises:
+        ValueError: If no <body class="Document"> element is found in the HTML.
+    """
+    html_code = remove_empty_divs_from_html_content(html_code)
+    html_code = remove_empty_tags_from_html_content(html_code)
+    soup = BeautifulSoup(html_code, "html.parser")
+    document = soup.find("body", class_="Document")
+    if not document:
+        document = soup.find("div", class_="Page")
+
+    if not document:
+        raise ValueError(
+            "No <body class='Document'> or <div class='Page'> element found in the HTML."
+        )
+
+    document_element = parse_html_to_ontology_element(document)
+    return document_element
+
+
+def remove_empty_divs_from_html_content(html_content: str) -> str:
+    soup = BeautifulSoup(html_content, "html.parser")
+    divs = soup.find_all("div")
+    for div in reversed(divs):
+        if not div.attrs:
+            div.unwrap()
+    return str(soup)
+
+
+def remove_empty_tags_from_html_content(html_content: str) -> str:
+    soup = BeautifulSoup(html_content, "html.parser")
+
+    def is_empty(tag):
+        # Remove only specific tags, omit self-closing ones
+        if tag.name not in (*HEADING_TAGS, "p", "span", "div"):
+            return False
+
+        if tag.find():
+            return False
+
+        if tag.attrs:
+            return False
+
+        return bool(not tag.get_text(strip=True))
+
+    def remove_empty_tags(soup):
+        for tag in soup.find_all():
+            if is_empty(tag):
+                tag.decompose()
+
+    remove_empty_tags(soup)
+
+    return str(soup)
+
+
+def parse_html_to_ontology_element(soup: Tag, recursion_depth: int = 1) -> ontology.OntologyElement:
+    """
+    Converts a BeautifulSoup Tag object into an OntologyElement object. This function is recursive.
+    First tries to recognize a class from Unstructured Ontology, then if class is matched tries
+    to go deeper inside HTML tree. The recursive parsing is ended if the class is not recognized or
+    there are no HTML Tags inside HTML - just text. Then it is parsed to
+    Paragraph or UncategorizedText object.
+
+    Args:
+        soup (Tag): The BeautifulSoup Tag object to be converted.
+        recursion_depth (int): Flag to control limit of recursion depth.
+    Returns:
+        OntologyElement: The converted OntologyElement object.
+    """
+    ontology_html_tag, ontology_class = extract_tag_and_ontology_class_from_tag(soup)
+    escaped_attrs = get_sanitized_attributes(soup, tag_name=ontology_html_tag)
+
+    if soup.name == "br":  # Note(Pluto) should it be <br class="UncategorizedText">?
+        return ontology.Paragraph(
+            text="",
+            css_class_name=None,
+            html_tag_name="br",
+            additional_attributes=escaped_attrs,
+        )
+
+    has_children = (
+        (ontology_class != ontology.UncategorizedText)
+        and any(isinstance(content, Tag) for content in soup.contents)
+        or ontology_class().elementType == ontology.ElementTypeEnum.layout
+    )
+    should_unwrap_html = has_children and recursion_depth <= RECURSION_LIMIT
+
+    if should_unwrap_html:
+        text = ""
+        children = [
+            (
+                parse_html_to_ontology_element(child, recursion_depth=recursion_depth + 1)
+                if isinstance(child, Tag)
+                else ontology.Paragraph(text=str(child).strip())
+            )
+            for child in soup.children
+            if str(child).strip()
+        ]
+    else:
+        text = "\n".join([str(content).strip() for content in soup.contents]).strip()
+        children = []
+
+    output_element = ontology_class(
+        text=text,
+        children=children,
+        html_tag_name=ontology_html_tag,
+        additional_attributes=escaped_attrs,
+    )
+    # TODO (Pluto): <input class="FormFieldValue"/> requires being wrapped in <label> tags
+    return output_element
+
+
+def extract_tag_and_ontology_class_from_tag(
+    soup: Tag,
+) -> tuple[str, Type[ontology.OntologyElement]]:
+    """
+    Extracts the HTML tag and corresponding ontology class
+    from a BeautifulSoup Tag object. The CSS class is prioritized over
+    the HTML tag. If not recognized soup.name and UnstructuredText is returned.
+
+    Args:
+        soup (Tag): The BeautifulSoup Tag object to extract information from.
+
+    Returns:
+        tuple: A tuple containing the HTML tag (str) and the ontology class (Type[OntologyElement]).
+    """
+    html_tag, element_class = None, None
+
+    # Scenario 1: Valid Ontology Element
+    if soup.attrs.get("class"):
+        html_tag, element_class = (
+            soup.name,
+            HTML_TAG_AND_CSS_NAME_TO_ELEMENT_TYPE_MAP.get((soup.name, soup.attrs["class"][0])),
+        )
+
+    # Scenario 2: HTML tag incorrect, CSS class correct
+    # Fallback to css name selector and overwrite html tag
+    if (
+        not element_class
+        and soup.attrs.get("class")
+        and soup.attrs["class"][0] in CSS_CLASS_TO_ELEMENT_TYPE_MAP
+    ):
+        element_class = CSS_CLASS_TO_ELEMENT_TYPE_MAP.get(soup.attrs["class"][0])
+        html_tag = element_class().allowed_tags[0]
+
+    # Scenario 3: <input> elements, handled explicitly based on their 'type' attribute
+    if not element_class and soup.name == "input":
+        input_type = (str(soup.get("type")) or "").lower()
+        if input_type == "checkbox":
+            element_class = ontology.Checkbox
+        elif input_type == "radio":
+            element_class = ontology.RadioButton
+        else:
+            # Any other input (including missing type or text/number/etc.) is considered
+            # a generic form field value.
+            element_class = ontology.FormFieldValue
+        html_tag = "input"
+
+    # Scenario 4: CSS class incorrect, but HTML tag correct and exclusive in ontology
+    if not element_class and soup.name in HTML_TAG_TO_DEFAULT_ELEMENT_TYPE_MAP:
+        html_tag, element_class = soup.name, HTML_TAG_TO_DEFAULT_ELEMENT_TYPE_MAP[soup.name]
+
+    # Scenario 5: CSS class incorrect, HTML tag incorrect
+    # Fallback to default UncategorizedText
+    if not element_class:
+        # TODO (Pluto): Sometimes we could infer that from parent type and soup.name
+        #  e.g. parent=FormField soup.name=input -> element=FormFieldInput
+
+        html_tag = "span"
+        element_class = ontology.UncategorizedText
+
+    # Scenario 6: UncategorizedText has image and no text
+    # Typically, this happens with a span or div tag with an image inside
+    if element_class == ontology.UncategorizedText and soup.find("img") and not soup.text.strip():
+        element_class = ontology.Image
+
+    return html_tag, element_class
+
+
+def get_sanitized_attributes(soup: Tag, tag_name: str | None = None) -> dict[str, str | list[str]]:
+    """
+    Sanitizes the attributes of a BeautifulSoup Tag object for the ontology.
+
+    Drops event-handler (``on*``) attributes and URL attributes with unsafe
+    schemes (``javascript:`` / ``vbscript:`` / non-image ``data:``) so an
+    ontology element never carries a dangerous attribute in the first place.
+
+    Values are intentionally left un-escaped: HTML-escaping is done exactly once,
+    at emit time, in ``OntologyElement.to_html`` (see
+    ``unstructured.documents.html_sanitization``). Escaping here as well would
+    double-encode entities (e.g. ``&`` -> ``&amp;amp;``).
+
+    Args:
+        soup (Tag): The BeautifulSoup Tag object whose attributes to sanitize.
+        tag_name: The ontology tag that will be emitted for this element.
+
+    Returns:
+        dict: A dictionary with the safe subset of attributes.
+    """
+    return sanitize_attributes(dict(soup.attrs), tag_name=tag_name)  # type: ignore[return-value]
+
+
+# -- Backwards-compatible alias; the previous name implied it html-escaped, which
+# -- it no longer does (escaping moved to emit time). --
+get_escaped_attributes = get_sanitized_attributes

unstructured/partition/image.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+from typing import IO, Any, Optional
+
+from unstructured.chunking import add_chunking_strategy
+from unstructured.documents.elements import Element, process_metadata
+from unstructured.file_utils.filetype import add_metadata
+from unstructured.partition.common.common import exactly_one
+from unstructured.partition.common.lang import check_language_args
+from unstructured.partition.pdf import partition_pdf_or_image
+from unstructured.partition.utils.constants import PartitionStrategy
+
+
+@process_metadata()  # TODO(shreya): update to use `apply_metadata` decorator
+@add_metadata
+@add_chunking_strategy
+def partition_image(
+    filename: Optional[str] = None,
+    file: Optional[IO[bytes]] = None,
+    include_page_breaks: bool = False,
+    infer_table_structure: bool = False,
+    ocr_languages: Optional[str] = None,
+    languages: Optional[list[str]] = None,
+    detect_language_per_element: bool = False,
+    strategy: str = PartitionStrategy.HI_RES,
+    metadata_last_modified: Optional[str] = None,
+    chunking_strategy: Optional[str] = None,
+    hi_res_model_name: Optional[str] = None,
+    extract_images_in_pdf: bool = False,
+    extract_image_block_types: Optional[list[str]] = None,
+    extract_image_block_output_dir: Optional[str] = None,
+    extract_image_block_to_payload: bool = False,
+    starting_page_number: int = 1,
+    extract_forms: bool = False,
+    form_extraction_skip_tables: bool = True,
+    password: Optional[str] = None,
+    **kwargs: Any,
+) -> list[Element]:
+    """Parses an image into a list of interpreted elements.
+
+    Parameters
+    ----------
+    filename
+        A string defining the target filename path.
+    file
+        A file-like object as bytes --> open(filename, "rb").
+    include_page_breaks
+        If True, includes page breaks at the end of each page in the document.
+    infer_table_structure
+        Only applicable if `strategy=hi_res`.
+        If True, any Table elements that are extracted will also have a metadata field
+        named "text_as_html" where the table's text content is rendered into an html string.
+        I.e., rows and cells are preserved.
+        Whether True or False, the "text" field is always present in any Table element
+        and is the text content of the table (no structure).
+    languages
+        The languages present in the document, for use in partitioning and/or OCR. To use a language
+        with Tesseract, you'll first need to install the appropriate Tesseract language pack.
+    strategy
+        The strategy to use for partitioning the image. Valid strategies are "hi_res" and
+        "ocr_only". When using the "hi_res" strategy, the function uses a layout detection
+        model if to identify document elements. When using the "ocr_only" strategy,
+        partition_image simply extracts the text from the document using OCR and processes it.
+        The default strategy is `hi_res`.
+    metadata_last_modified
+        The last modified date for the document.
+    hi_res_model_name
+        The layout detection model used when partitioning strategy is set to `hi_res`.
+    extract_images_in_pdf
+        Only applicable if `strategy=hi_res`.
+        If True, any detected images will be saved in the path specified by
+        'extract_image_block_output_dir' or stored as base64 encoded data within metadata fields.
+        Deprecation Note: This parameter is marked for deprecation. Future versions will use
+        'extract_image_block_types' for broader extraction capabilities.
+    extract_image_block_types
+        Only applicable if `strategy=hi_res`.
+        Images of the element type(s) specified in this list (e.g., ["Image", "Table"]) will be
+        saved in the path specified by 'extract_image_block_output_dir' or stored as base64 encoded
+        data within metadata fields.
+    extract_image_block_to_payload
+        Only applicable if `strategy=hi_res`.
+        If True, images of the element type(s) defined in 'extract_image_block_types' will be
+        encoded as base64 data and stored in two metadata fields: 'image_base64' and
+        'image_mime_type'.
+        This parameter facilitates the inclusion of element data directly within the payload,
+        especially for web-based applications or APIs.
+    extract_image_block_output_dir
+        Only applicable if `strategy=hi_res` and `extract_image_block_to_payload=False`.
+        The filesystem path for saving images of the element type(s)
+        specified in 'extract_image_block_types'.
+    extract_forms
+        Whether the form extraction logic should be run
+        (results in adding FormKeysValues elements to output).
+    form_extraction_skip_tables
+        Whether the form extraction logic should ignore regions designated as Tables.
+    password
+        The password to decrypt the PDF file.
+    """
+    exactly_one(filename=filename, file=file)
+
+    languages = check_language_args(languages or [], ocr_languages)
+
+    return partition_pdf_or_image(
+        filename=filename,
+        file=file,
+        is_image=True,
+        include_page_breaks=include_page_breaks,
+        infer_table_structure=infer_table_structure,
+        languages=languages,
+        detect_language_per_element=detect_language_per_element,
+        strategy=strategy,
+        metadata_last_modified=metadata_last_modified,
+        hi_res_model_name=hi_res_model_name,
+        extract_images_in_pdf=extract_images_in_pdf,
+        extract_image_block_types=extract_image_block_types,
+        extract_image_block_output_dir=extract_image_block_output_dir,
+        extract_image_block_to_payload=extract_image_block_to_payload,
+        starting_page_number=starting_page_number,
+        extract_forms=extract_forms,
+        form_extraction_skip_tables=form_extraction_skip_tables,
+        password=password,
+        **kwargs,
+    )

unstructured/partition/json.py

@@ -0,0 +1,88 @@
+"""Provides `partition_json()`.
+
+Note this does not partition arbitrary JSON. Its only use-case is to "rehydrate" unstructured
+document elements serialized to JSON, essentially the same function as `elements_from_json()`, but
+this allows a document of already-partitioned elements to be combined transparently with other
+documents in a partitioning run. It also allows multiple (low-cost) chunking runs to be performed on
+a document while only incurring partitioning cost once.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import IO, Any, Optional
+
+from unstructured.chunking import add_chunking_strategy
+from unstructured.documents.elements import Element, process_metadata
+from unstructured.file_utils.filetype import (
+    FileType,
+    add_metadata_with_filetype,
+    is_json_processable,
+)
+from unstructured.partition.common.common import exactly_one
+from unstructured.partition.common.metadata import get_last_modified_date
+from unstructured.staging.base import elements_from_dicts
+
+
+@process_metadata()
+@add_metadata_with_filetype(FileType.JSON)
+@add_chunking_strategy
+def partition_json(
+    filename: Optional[str] = None,
+    file: Optional[IO[bytes]] = None,
+    text: Optional[str] = None,
+    metadata_last_modified: Optional[str] = None,
+    **kwargs: Any,
+) -> list[Element]:
+    """Partitions serialized Unstructured output into its constituent elements.
+
+    Parameters
+    ----------
+    filename
+        A string defining the target filename path.
+    file
+        A file-like object as bytes --> open(filename, "rb").
+    text
+        The string representation of the .json document.
+    metadata_last_modified
+        The last modified date for the document.
+    """
+    if text is not None and text.strip() == "" and not file and not filename:
+        return []
+
+    exactly_one(filename=filename, file=file, text=text)
+
+    last_modified = get_last_modified_date(filename) if filename else None
+    file_text = ""
+    if filename is not None:
+        with open(filename, encoding="utf8") as f:
+            file_text = f.read()
+
+    elif file is not None:
+        file_content = file.read()
+        file_text = file_content if isinstance(file_content, str) else file_content.decode()
+        file.seek(0)
+
+    elif text is not None:
+        file_text = str(text)
+
+    if not is_json_processable(file_text=file_text):
+        raise ValueError(
+            "JSON cannot be partitioned. Schema does not match the Unstructured schema.",
+        )
+
+    try:
+        element_dicts = json.loads(file_text)
+        elements = elements_from_dicts(element_dicts)
+        # if we found at least one json element, but no unstructured elements were found, throw 422
+        if len(element_dicts) > 0 and len(elements) == 0:
+            raise ValueError(
+                "JSON cannot be partitioned. Schema does not match the Unstructured schema.",
+            )
+    except json.JSONDecodeError:
+        raise ValueError("Not a valid json")
+
+    for element in elements:
+        element.metadata.last_modified = metadata_last_modified or last_modified
+
+    return elements

unstructured/partition/md.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+from typing import IO, Any, Optional
+
+import markdown
+from markdown.extensions import Extension
+
+from unstructured.documents.elements import Element
+from unstructured.file_utils.encoding import read_txt_file
+from unstructured.file_utils.model import FileType
+from unstructured.partition.common.common import exactly_one
+from unstructured.partition.common.metadata import get_last_modified_date
+from unstructured.partition.html import partition_html
+from unstructured.safe_http import safe_get
+
+
+def optional_decode(contents: str | bytes) -> str:
+    if isinstance(contents, bytes):
+        return contents.decode("utf-8")
+    return contents
+
+
+DETECTION_ORIGIN: str = "md"
+
+_DEFAULT_MARKDOWN_EXTENSIONS: list[str] = ["tables", "fenced_code"]
+
+
+def _validate_markdown_extensions(extensions: Any) -> list[Any]:
+    """Return ``extensions`` if it is a list of strings and/or ``Extension`` instances.
+
+    Python-Markdown accepts extension entry points as registered names (``str``) or configured
+    ``Extension`` instances; both are supported here. Any other shape raises ``ValueError``.
+    """
+    if not isinstance(extensions, list):
+        raise ValueError(
+            "'extensions' must be a list of extension names (str) and/or "
+            f"markdown.extensions.Extension instances, got {type(extensions).__name__!r}"
+        )
+    for item in extensions:
+        if not isinstance(item, (str, Extension)):
+            raise ValueError(
+                "Each entry in 'extensions' must be a str or markdown.extensions.Extension "
+                f"instance, got {type(item).__name__}: {item!r}"
+            )
+    return extensions
+
+
+def partition_md(
+    filename: str | None = None,
+    file: IO[bytes] | None = None,
+    text: str | None = None,
+    url: str | None = None,
+    metadata_filename: str | None = None,
+    metadata_last_modified: str | None = None,
+    languages: Optional[list[str]] = None,
+    **kwargs: Any,
+) -> list[Element]:
+    """Partitions a markdown file into its constituent elements
+
+    Parameters
+    ----------
+    filename
+        A string defining the target filename path.
+    file
+        A file-like object using "rb" mode --> open(filename, "rb").
+    text
+        The string representation of the markdown document.
+    url
+        The URL of a webpage to parse. Only for URLs that return a markdown document.
+    metadata_last_modified
+        The last modified date for the document.
+    languages
+        The languages present in the document. Use ``["auto"]`` to detect (default when None).
+        Use ``[""]`` to disable language detection.
+
+    Other keyword arguments are forwarded to ``partition_html``. In addition, ``extensions`` may be
+    passed to ``markdown.markdown()`` as a list of registered extension names (``str``) and/or
+    configured ``markdown.extensions.Extension`` instances. The default is
+    ``["tables", "fenced_code"]``. Pass e.g. ``extensions=["tables"]`` if you need the legacy
+    behavior where ``#`` inside unfenced content is parsed as a heading (see #4006).
+    """
+    if text is None:
+        text = ""
+
+    # -- verify that only one of the arguments was provided --
+    exactly_one(filename=filename, file=file, text=text, url=url)
+
+    last_modified = get_last_modified_date(filename) if filename else None
+
+    if filename is not None:
+        _, text = read_txt_file(filename=filename)
+
+    elif file is not None:
+        _, text = read_txt_file(file=file)
+
+    elif url is not None:
+        response = safe_get(url)
+        if not response.ok:
+            raise ValueError(f"URL return an error: {response.status_code}")
+
+        content_type = response.headers.get("Content-Type", "")
+        if not content_type.startswith("text/markdown"):
+            raise ValueError(
+                f"Expected content type text/markdown. Got {content_type}.",
+            )
+
+        text = response.text
+
+    # -- optional markdown extensions; default matches historical partition_md behavior --
+    extensions = _validate_markdown_extensions(
+        kwargs.pop("extensions", _DEFAULT_MARKDOWN_EXTENSIONS)
+    )
+
+    html = markdown.markdown(text, extensions=extensions)
+
+    html_kwargs: dict[str, Any] = {
+        "text": html,
+        "metadata_filename": metadata_filename or filename,
+        "metadata_file_type": FileType.MD,
+        "metadata_last_modified": metadata_last_modified or last_modified,
+        "detection_origin": DETECTION_ORIGIN,
+        **kwargs,
+    }
+    if languages is not None:
+        html_kwargs["languages"] = languages
+    return partition_html(**html_kwargs)

unstructured/partition/model_init.py

@@ -0,0 +1,16 @@
+import os
+
+from unstructured_inference.models.base import get_model
+
+
+def initialize():
+    """Download default model or model specified by UNSTRUCTURED_HI_RES_MODEL_NAME environment
+    variable (avoids subprocesses all doing the same)"""
+
+    # If more than one model will be supported and left up to user selection
+    supported_model = os.environ.get("UNSTRUCTURED_HI_RES_SUPPORTED_MODEL", "")
+    if supported_model:
+        for model_name in supported_model.split(","):
+            get_model(model_name=model_name)
+
+    get_model(os.environ.get("UNSTRUCTURED_HI_RES_MODEL_NAME"))

unstructured/partition/msg.py

@@ -0,0 +1,323 @@
+from __future__ import annotations
+
+import os
+import re
+import tempfile
+from functools import cached_property
+from typing import IO, Any, Iterator, Optional
+
+from oxmsg import Message
+from oxmsg.attachment import Attachment
+
+from unstructured.documents.elements import Element, ElementMetadata
+from unstructured.file_utils.model import FileType
+from unstructured.logger import logger
+from unstructured.partition.common import EXPECTED_ATTACHMENT_ERRORS
+from unstructured.partition.common.metadata import get_last_modified_date
+from unstructured.partition.html import partition_html
+from unstructured.partition.text import partition_text
+from unstructured.utils import is_temp_file_path
+
+
+def partition_msg(
+    filename: Optional[str] = None,
+    *,
+    file: Optional[IO[bytes]] = None,
+    metadata_filename: Optional[str] = None,
+    metadata_last_modified: Optional[str] = None,
+    process_attachments: bool = True,
+    **kwargs: Any,
+) -> list[Element]:
+    """Partitions a MSFT Outlook .msg file
+
+    Parameters
+    ----------
+    filename
+        A string defining the target filename path.
+    file
+        A file-like object using "rb" mode --> open(filename, "rb").
+    metadata_filename
+        The filename to use for the metadata.
+    metadata_last_modified
+        The last modified date for the document.
+    process_attachments
+        If True, partition_email will process email attachments in addition to
+        processing the content of the email itself.
+    """
+    opts = MsgPartitionerOptions(
+        file=file,
+        file_path=filename,
+        metadata_file_path=metadata_filename,
+        metadata_last_modified=metadata_last_modified,
+        partition_attachments=process_attachments,
+        kwargs=kwargs,
+    )
+
+    return list(_MsgPartitioner.iter_message_elements(opts))
+
+
+class MsgPartitionerOptions:
+    """Encapsulates partitioning option validation, computation, and application of defaults."""
+
+    def __init__(
+        self,
+        *,
+        file: IO[bytes] | None,
+        file_path: str | None,
+        metadata_file_path: str | None,
+        metadata_last_modified: str | None,
+        partition_attachments: bool,
+        kwargs: dict[str, Any],
+    ):
+        self._file = file
+        self._file_path = file_path
+        self._metadata_file_path = metadata_file_path
+        self._metadata_last_modified = metadata_last_modified
+        self._partition_attachments = partition_attachments
+        self._kwargs = kwargs
+
+    @cached_property
+    def extra_msg_metadata(self) -> ElementMetadata:
+        """ElementMetadata suitable for use on an element formed from message content.
+
+        These are only the metadata fields specific to email messages. The remaining metadata
+        fields produced by the delegate partitioner are used as produced.
+
+        None of these metadata fields change based on the element, so we just compute it once.
+        """
+        msg = self.msg
+
+        sent_from = [s.strip() for s in sender.split(",")] if (sender := msg.sender) else None
+        sent_to = [r.email_address for r in msg.recipients] or None
+        bcc_recipient = (
+            [c.strip() for c in bcc.split(",")] if (bcc := msg.message_headers.get("Bcc")) else None
+        )
+        cc_recipient = (
+            [c.strip() for c in cc.split(",")] if (cc := msg.message_headers.get("Cc")) else None
+        )
+        if email_message_id := msg.message_headers.get("Message-Id"):
+            email_message_id = re.sub(r"^<|>$", "", email_message_id)  # Strip angle brackets
+
+        element_metadata = ElementMetadata(
+            bcc_recipient=bcc_recipient,
+            cc_recipient=cc_recipient,
+            email_message_id=email_message_id,
+            sent_from=sent_from,
+            sent_to=sent_to,
+            subject=msg.subject or None,
+        )
+        element_metadata.detection_origin = "msg"
+
+        return element_metadata
+
+    @cached_property
+    def is_encrypted(self) -> bool:
+        """True when message is encrypted."""
+        # NOTE(robinson) - Per RFC 2015, the content type for emails with PGP encrypted content
+        # is multipart/encrypted (ref: https://www.ietf.org/rfc/rfc2015.txt)
+        # NOTE(scanny) - pretty sure we're going to want to dig deeper to discover messages that are
+        # encrypted with something other than PGP.
+        #   - might be able to distinguish based on PID_MESSAGE_CLASS = 'IPM.Note.Signed'
+        #   - Content-Type header might include "application/pkcs7-mime" for Microsoft S/MIME
+        #     encryption.
+        return "encrypted" in self.msg.message_headers.get("Content-Type", "")
+
+    @cached_property
+    def metadata_file_path(self) -> str | None:
+        """Best available path for MSG file.
+
+        The value is the caller supplied `metadata_filename` if present, falling back to the
+        source file-path if that was provided, otherwise `None`.
+        """
+        return self._metadata_file_path or self._file_path
+
+    @cached_property
+    def metadata_last_modified(self) -> str | None:
+        """Caller override for `.metadata.last_modified` to be applied to all elements."""
+        email_date = sent_date.isoformat() if (sent_date := self.msg.sent_date) else None
+        return self._metadata_last_modified or email_date or self._last_modified
+
+    @cached_property
+    def msg(self) -> Message:
+        """The `oxmsg.Message` object loaded from file or filename."""
+        return Message.load(self._msg_file)
+
+    @cached_property
+    def partition_attachments(self) -> bool:
+        """True when message attachments should also be partitioned."""
+        return self._partition_attachments
+
+    @cached_property
+    def partitioning_kwargs(self) -> dict[str, Any]:
+        """The "extra" keyword arguments received by `partition_msg()`.
+
+        These are passed along to delegate partitioners which extract keyword args like
+        `chunking_strategy` etc. in their decorators to control metadata behaviors, etc.
+        """
+        return self._kwargs
+
+    @cached_property
+    def _last_modified(self) -> str | None:
+        """The best last-modified date available from source-file, None if not available."""
+        if not self._file_path or is_temp_file_path(self._file_path):
+            return None
+
+        return get_last_modified_date(self._file_path)
+
+    @cached_property
+    def _msg_file(self) -> str | IO[bytes]:
+        """The source for the bytes of the message, either a file-path or a file-like object."""
+        if file_path := self._file_path:
+            return file_path
+
+        if file := self._file:
+            return file
+
+        raise ValueError("one of `file` or `filename` arguments must be provided")
+
+
+class _MsgPartitioner:
+    """Partitions Outlook email message (MSG) files."""
+
+    def __init__(self, opts: MsgPartitionerOptions):
+        self._opts = opts
+
+    @classmethod
+    def iter_message_elements(cls, opts: MsgPartitionerOptions) -> Iterator[Element]:
+        """Partition MS Outlook email messages (.msg files) into elements."""
+        if opts.is_encrypted:
+            logger.warning("Encrypted email detected. Partitioner will return an empty list.")
+            return
+
+        yield from cls(opts)._iter_message_elements()
+
+    def _iter_message_elements(self) -> Iterator[Element]:
+        """Partition MS Outlook email messages (.msg files) into elements."""
+        yield from self._iter_message_body_elements()
+
+        if not self._opts.partition_attachments:
+            return
+
+        for attachment in self._attachments:
+            yield from _AttachmentPartitioner.iter_elements(attachment, self._opts)
+
+    @cached_property
+    def _attachments(self) -> tuple[Attachment, ...]:
+        """The `oxmsg.attachment.Attachment` objects for this message."""
+        return tuple(self._opts.msg.attachments)
+
+    def _iter_message_body_elements(self) -> Iterator[Element]:
+        """Partition the message body (but not the attachments)."""
+        msg = self._opts.msg
+
+        if html_body := msg.html_body:
+            elements = partition_html(
+                text=html_body,
+                metadata_filename=self._opts.metadata_file_path,
+                metadata_file_type=FileType.MSG,
+                metadata_last_modified=self._opts.metadata_last_modified,
+                **self._opts.partitioning_kwargs,
+            )
+        elif msg.body:
+            elements = partition_text(
+                text=msg.body,
+                metadata_filename=self._opts.metadata_file_path,
+                metadata_file_type=FileType.MSG,
+                metadata_last_modified=self._opts.metadata_last_modified,
+                **self._opts.partitioning_kwargs,
+            )
+        else:
+            elements: list[Element] = []
+
+        # -- augment the element metadata with email-specific values --
+        email_specific_metadata = self._opts.extra_msg_metadata
+        for e in elements:
+            e.metadata.update(email_specific_metadata)
+            yield e
+
+
+class _AttachmentPartitioner:
+    """Partitions an attachment to a MSG file."""
+
+    def __init__(self, attachment: Attachment, opts: MsgPartitionerOptions):
+        self._attachment = attachment
+        self._opts = opts
+
+    @classmethod
+    def iter_elements(
+        cls, attachment: Attachment, opts: MsgPartitionerOptions
+    ) -> Iterator[Element]:
+        """Partition an `oxmsg.attachment.Attachment` from an Outlook email message (.msg file)."""
+        return cls(attachment, opts)._iter_elements()
+
+    def _iter_elements(self) -> Iterator[Element]:
+        """Partition the file in an `oxmsg.attachment.Attachment` into elements."""
+        from unstructured.partition.auto import partition
+
+        with tempfile.TemporaryDirectory() as tmp_dir_path:
+            # -- save attachment as file in this temporary directory --
+            detached_file_path = os.path.join(tmp_dir_path, self._attachment_file_name)
+            with open(detached_file_path, "wb") as f:
+                f.write(self._file_bytes)
+
+            # -- partition the attachment --
+            try:
+                elements = partition(
+                    detached_file_path,
+                    metadata_filename=self._attachment_file_name,
+                    metadata_last_modified=self._attachment_last_modified,
+                    **self._opts.partitioning_kwargs,
+                )
+            except BaseException as e:
+                if not isinstance(e, EXPECTED_ATTACHMENT_ERRORS):
+                    raise
+                logger.warning(
+                    "Skipping attachment %s: %s",
+                    self._attachment_file_name,
+                    f"{type(e).__name__}: {e}",
+                )
+                return
+
+            for e in elements:
+                e.metadata.attached_to_filename = self._opts.metadata_file_path
+                yield e
+
+    @cached_property
+    def _attachment_file_name(self) -> str:
+        """The original name of the attached file, no path.
+
+        This value is 'unknown' if it is not present in the MSG file (not expected).
+        The filename is sanitized to prevent path traversal attacks.
+        """
+        raw_filename = self._attachment.file_name or "unknown"
+
+        # Sanitize the filename to prevent path traversal attacks
+        # Remove any path components for both Unix and Windows paths
+        # Use both separators to handle cross-platform attacks
+        safe_filename = os.path.basename(raw_filename.replace("\\", "/"))
+
+        # Remove null bytes and other control characters
+        safe_filename = safe_filename.replace("\0", "")
+
+        # If the filename becomes empty after sanitization, use a default
+        if not safe_filename or safe_filename in (".", ".."):
+            safe_filename = "unknown"
+
+        return safe_filename
+
+    @cached_property
+    def _attachment_last_modified(self) -> str | None:
+        """ISO8601 string timestamp of attachment last-modified date.
+
+        This value generally available on the attachment and will be the most reliable last-modifed
+        time. There are fallbacks for when it is not present, ultimately `None` if we have no way
+        of telling.
+        """
+        if last_modified := self._attachment.last_modified:
+            return last_modified.isoformat()
+        return self._opts.metadata_last_modified
+
+    @cached_property
+    def _file_bytes(self) -> bytes:
+        """The bytes of the attached file."""
+        return self._attachment.file_bytes or b""

unstructured/partition/ndjson.py

@@ -0,0 +1,89 @@
+"""Provides `partition_ndjson()`.
+
+Note this does not partition arbitrary NDJSON. Its only use-case is to "rehydrate" unstructured
+document elements serialized to JSON, essentially the same function as `elements_from_json()`, but
+this allows a document of already-partitioned elements to be combined transparently with other
+documents in a partitioning run. It also allows multiple (low-cost) chunking runs to be performed on
+a document while only incurring partitioning cost once.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import IO, Any, Optional
+
+from unstructured.chunking import add_chunking_strategy
+from unstructured.documents.elements import Element, process_metadata
+from unstructured.file_utils.filetype import (
+    FileType,
+    add_metadata_with_filetype,
+    is_ndjson_processable,
+)
+from unstructured.file_utils.ndjson import loads as ndjson_loads
+from unstructured.partition.common.common import exactly_one
+from unstructured.partition.common.metadata import get_last_modified_date
+from unstructured.staging.base import elements_from_dicts
+
+
+@process_metadata()
+@add_metadata_with_filetype(FileType.NDJSON)
+@add_chunking_strategy
+def partition_ndjson(
+    filename: Optional[str] = None,
+    file: Optional[IO[bytes]] = None,
+    text: Optional[str] = None,
+    metadata_last_modified: Optional[str] = None,
+    **kwargs: Any,
+) -> list[Element]:
+    """Partitions serialized Unstructured output into its constituent elements.
+
+    Parameters
+    ----------
+    filename
+        A string defining the target filename path.
+    file
+        A file-like object as bytes --> open(filename, "rb").
+    text
+        The string representation of the .json document.
+    metadata_last_modified
+        The last modified date for the document.
+    """
+    if text is not None and text.strip() == "" and not file and not filename:
+        return []
+
+    exactly_one(filename=filename, file=file, text=text)
+
+    last_modified = get_last_modified_date(filename) if filename else None
+    file_text = ""
+    if filename is not None:
+        with open(filename, encoding="utf8") as f:
+            file_text = f.read()
+
+    elif file is not None:
+        file_content = file.read()
+        file_text = file_content if isinstance(file_content, str) else file_content.decode()
+        file.seek(0)
+
+    elif text is not None:
+        file_text = str(text)
+
+    if not is_ndjson_processable(file_text=file_text):
+        raise ValueError(
+            "NDJSON cannot be partitioned. Schema does not match the Unstructured schema.",
+        )
+
+    try:
+        element_dicts = ndjson_loads(file_text)
+        elements = elements_from_dicts(element_dicts)
+        # if we found at least one json element, but no unstructured elements were found, throw 422
+        if len(element_dicts) > 0 and len(elements) == 0:
+            raise ValueError(
+                "JSON cannot be partitioned. Schema does not match the Unstructured schema.",
+            )
+    except json.JSONDecodeError:
+        raise ValueError("Not a valid ndjson")
+
+    for element in elements:
+        element.metadata.last_modified = metadata_last_modified or last_modified
+
+    return elements

unstructured/partition/odt.py

@@ -0,0 +1,125 @@
+from __future__ import annotations
+
+import os
+import tempfile
+from typing import IO, Any, Optional, cast
+
+from unstructured.documents.elements import Element
+from unstructured.file_utils.model import FileType
+from unstructured.partition.common.common import exactly_one
+from unstructured.partition.common.metadata import get_last_modified_date
+from unstructured.partition.docx import partition_docx
+from unstructured.utils import requires_dependencies
+
+
+def partition_odt(
+    filename: Optional[str] = None,
+    *,
+    file: Optional[IO[bytes]] = None,
+    metadata_filename: Optional[str] = None,
+    metadata_last_modified: Optional[str] = None,
+    **kwargs: Any,
+) -> list[Element]:
+    """Partitions Open Office Documents in .odt format into its document elements.
+
+    All parameters that are available on `partition_docx()` are also available here.
+
+    Parameters
+    ----------
+    filename
+        A string defining the target filename path.
+    file
+        A file-like object using "rb" mode --> open(filename, "rb").
+    infer_table_structure
+        If True, any Table elements that are extracted will also have a metadata field
+        named "text_as_html" where the table's text content is rendered into an html string.
+        I.e., rows and cells are preserved.
+        Whether True or False, the "text" field is always present in any Table element
+        and is the text content of the table (no structure).
+    metadata_last_modified
+        The last modified date for the document.
+    languages
+        User defined value for `metadata.languages` if provided. Otherwise language is detected
+        using naive Bayesian filter via `langdetect`. Multiple languages indicates text could be
+        in either language.
+        Additional Parameters:
+            detect_language_per_element
+                Detect language per element instead of at the document level.
+    """
+
+    last_modified = get_last_modified_date(filename) if filename else None
+
+    with tempfile.TemporaryDirectory() as target_dir:
+        docx_path = _convert_odt_to_docx(target_dir, filename, file)
+        elements = partition_docx(
+            filename=docx_path,
+            metadata_filename=metadata_filename or filename,
+            metadata_file_type=FileType.ODT,
+            metadata_last_modified=metadata_last_modified or last_modified,
+            **kwargs,
+        )
+
+    return elements
+
+
+@requires_dependencies("pypandoc")
+def _convert_odt_to_docx(
+    target_dir: str, filename: Optional[str], file: Optional[IO[bytes]]
+) -> str:
+    """Convert ODT document to DOCX returning the new .docx file's path.
+
+    Parameters
+    ----------
+    target_dir
+        The str directory-path to use for conversion purposes. The new DOCX file is written to this
+        directory. When passed as a file-like object, a copy of the source file is written here as
+        well. It is the caller's responsibility to remove this directory and its contents when
+        they are no longer needed.
+    filename
+        A str file-path specifying the location of the source ODT file on the local filesystem.
+    file
+        A file-like object open for reading in binary mode ("rb" mode).
+    """
+    exactly_one(filename=filename, file=file)
+
+    # -- validate file-path when provided so we can provide a more meaningful error than whatever
+    # -- would come from pandoc.
+    if filename is not None and not os.path.exists(filename):
+        raise ValueError(f"The file {filename} does not exist.")
+
+    # -- Pandoc is a command-line program running in its own memory-space. It can therefore only
+    # -- operate on files on the filesystem. If the source document was passed as `file`, write
+    # -- it to `target_dir/document.odt` and use that path as the source-path.
+    source_file_path = f"{target_dir}/document.odt" if file is not None else cast(str, filename)
+    if file is not None:
+        with open(source_file_path, "wb") as f:
+            f.write(file.read())
+
+    # -- Compute the path of the resulting .docx document. We want its file-name to be preserved
+    # -- if the source-document was provided as `filename`.
+    # -- a/b/foo.odt -> foo.odt --
+    file_name = os.path.basename(source_file_path)
+    # -- foo.odt -> foo --
+    base_name, _ = os.path.splitext(file_name)
+    # -- foo -> foo.docx --
+    target_docx_path = os.path.join(target_dir, f"{base_name}.docx")
+
+    import pypandoc
+
+    # Try with sandbox=True first for security. If it fails due to Pandoc's known limitation
+    # with DOCX output in sandbox mode (see jgm/pandoc#8128), fallback without sandbox only
+    # if ALLOW_PANDOC_NO_SANDBOX env var is set to "true".
+    try:
+        pypandoc.convert_file(
+            source_file_path, "docx", format="odt", outputfile=target_docx_path, sandbox=True
+        )
+    except RuntimeError:
+        allow_no_sandbox = os.environ.get("ALLOW_PANDOC_NO_SANDBOX", "").lower() == "true"
+        if allow_no_sandbox:
+            pypandoc.convert_file(
+                source_file_path, "docx", format="odt", outputfile=target_docx_path
+            )
+        else:
+            raise
+
+    return target_docx_path

unstructured/partition/org.py

@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from typing import IO, Any
+
+from unstructured.documents.elements import Element
+from unstructured.file_utils.file_conversion import convert_file_to_html_text_using_pandoc
+from unstructured.file_utils.model import FileType
+from unstructured.partition.common.common import exactly_one
+from unstructured.partition.common.metadata import get_last_modified_date
+from unstructured.partition.html import partition_html
+
+DETECTION_ORIGIN: str = "org"
+
+
+def partition_org(
+    filename: str | None = None,
+    *,
+    file: IO[bytes] | None = None,
+    metadata_filename: str | None = None,
+    metadata_last_modified: str | None = None,
+    **kwargs: Any,
+) -> list[Element]:
+    """Partitions an org document. The document is first converted to HTML and then
+    partitioned using partition_html.
+
+    Parameters
+    ----------
+    filename
+        A string defining the target filename path.
+    file
+        A file-like object using "rb" mode --> open(filename, "rb").
+    metadata_last_modified
+        The last modified date for the document.
+    """
+    exactly_one(filename=filename, file=file)
+
+    last_modified = get_last_modified_date(filename) if filename else None
+
+    html_text = convert_file_to_html_text_using_pandoc(
+        source_format="org", filename=filename, file=file
+    )
+
+    return partition_html(
+        text=html_text,
+        metadata_filename=metadata_filename or filename,
+        metadata_file_type=FileType.ORG,
+        metadata_last_modified=metadata_last_modified or last_modified,
+        detection_origin=DETECTION_ORIGIN,
+        **kwargs,
+    )

unstructured/partition/pdf_image/analysis/bbox_visualisation.py

@@ -0,0 +1,712 @@
+import logging
+import math
+import tempfile
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from enum import Enum
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Generator, List, Optional, TypeVar, Union
+
+import numpy as np
+from matplotlib import colors, font_manager
+from numba import njit
+from PIL import Image, ImageDraw, ImageFont
+from unstructured_inference.constants import ElementType
+
+from unstructured.partition.pdf_image.analysis.processor import AnalysisProcessor
+from unstructured.partition.pdf_image.pdf_image_utils import convert_pdf_to_image
+
+PageImage = TypeVar("PageImage", Image.Image, np.ndarray)
+
+
+def get_font():
+    preferred_fonts = ["Arial.ttf"]
+    available_fonts = font_manager.findSystemFonts()
+    if not available_fonts:
+        raise ValueError("No fonts available")
+    for font in preferred_fonts:
+        for available_font in available_fonts:
+            if font in available_font:
+                return available_font
+    return available_fonts[0]
+
+
+COLOR_WHITE = ("white", (255, 255, 255))
+COLOR_BLACK = ("black", (0, 0, 0))
+
+
+class TextAlignment(Enum):
+    TOP_LEFT = "top_left"
+    TOP_RIGHT = "top_right"
+    BOTTOM_LEFT = "bottom_left"
+    BOTTOM_RIGHT = "bottom_right"
+    CENTER = "center"
+
+
+@dataclass
+class BboxLabels:
+    top_left: Optional[str] = None
+    top_right: Optional[str] = None
+    bottom_left: Optional[str] = None
+    bottom_right: Optional[str] = None
+    center: Optional[str] = None
+
+
+@dataclass
+class BBox:
+    points: tuple[int, int, int, int]
+    labels: Optional[BboxLabels] = None
+
+
+def get_rgb_color(color: str) -> tuple[int, int, int]:
+    """Convert a color name to RGB values.
+
+    Args:
+        color: A color name supported by matplotlib.
+
+    Returns:
+        A tuple of three integers representing the RGB values of the color.
+    """
+    try:
+        rgb_colors = colors.to_rgb(color)
+    except ValueError:
+        print("Error")
+        raise
+    return int(rgb_colors[0] * 255), int(rgb_colors[1] * 255), int(rgb_colors[2] * 255)
+
+
+@njit(cache=True, fastmath=True)
+def _get_bbox_to_page_ratio(bbox: tuple[int, int, int, int], page_size: tuple[int, int]) -> float:
+    """Compute the ratio of the bounding box to the page size.
+
+    Args:
+        bbox: Tuple containing coordinates of the bbox: (x1, y1, x2, y2).
+        page_size: Tuple containing page size: (width, height).
+
+    Returns:
+        The ratio of the bounding box to the page size.
+    """
+    x1, y1, x2, y2 = bbox
+    page_width, page_height = page_size
+    page_diagonal = math.sqrt(page_height**2 + page_width**2)
+    bbox_width = x2 - x1
+    bbox_height = y2 - y1
+    bbox_diagonal = math.sqrt(bbox_height**2 + bbox_width**2)
+    return bbox_diagonal / page_diagonal
+
+
+def _get_optimal_value_for_bbox(
+    bbox: tuple[int, int, int, int],
+    page_size: tuple[int, int],
+    min_value: int,
+    max_value: int,
+    ratio_for_min_value: float = 0.01,
+    ratio_for_max_value: float = 0.5,
+) -> int:
+    """Compute the optimal value for a given bounding box using a linear function
+    generated for given min and max values and ratios
+
+    Args:
+        bbox: Tuple containing coordinates of the bbox: (x1, y1, x2, y2).
+        page_size: Tuple containing page size: (width, height).
+        min_value: The minimum value returned by the function.
+        max_value: The maximum value returned by the function.
+        ratio_for_min_value: The ratio of the bbox to page size for the min value.
+        ratio_for_max_value: The ratio of the bbox to page size for the max value.
+
+    Returns:
+        The optimal value for the given bounding box and parameters given.
+    """
+    bbox_to_page_ratio = _get_bbox_to_page_ratio(bbox, page_size)
+    slope, intercept = _linear_polyfit_2point(
+        ratio_for_min_value, ratio_for_max_value, min_value, max_value
+    )
+    value = int(bbox_to_page_ratio * slope + intercept)
+    return max(min_value, min(max_value, value))
+
+
+def get_bbox_text_size(
+    bbox: tuple[int, int, int, int],
+    page_size: tuple[int, int],
+    min_font_size: int = 16,
+    max_font_size: int = 32,
+) -> int:
+    """Compute the optimal font size for a given bounding box.
+
+    Args:
+        bbox: Tuple containing coordinates of the bbox: (x1, y1, x2, y2).
+        page_size: Tuple containing page size: (width, height).
+        min_font_size: The minimum font size returned by the function.
+        max_font_size: The maximum font size returned by the function.
+
+    Returns:
+        The optimal font size for the given bounding box.
+    """
+    return _get_optimal_value_for_bbox(
+        bbox=bbox,
+        page_size=page_size,
+        min_value=min_font_size,
+        max_value=max_font_size,
+    )
+
+
+def get_bbox_thickness(
+    bbox: tuple[int, int, int, int],
+    page_size: tuple[int, int],
+    min_thickness: int = 1,
+    max_thickness: int = 4,
+) -> float:
+    """Compute the optimal thickness for a given bounding box.
+
+    Args:
+        bbox: Tuple containing coordinates of the bbox: (x1, y1, x2, y2).
+        page_size: Tuple containing page size: (width, height).
+        min_thickness: The minimum font size returned by the function.
+        max_thickness: The maximum font size returned by the function.
+
+    Returns:
+
+    """
+    return _get_optimal_value_for_bbox(
+        bbox=bbox,
+        page_size=page_size,
+        min_value=min_thickness,
+        max_value=max_thickness,
+    )
+
+
+def get_text_color(
+    background_color: Union[str, tuple[int, int, int]], brightness_threshold: float = 0.5
+) -> tuple[str, tuple[int, int, int]]:
+    """Returns the contrastive text color (black or white) for a given background color.
+
+    Args:
+        background_color: Tuple containing RGB values of the background color.
+
+    Returns:
+        Tuple containing RGB values of the text color.
+    """
+    if isinstance(background_color, str):
+        background_color = get_rgb_color(background_color)
+    background_brightness = (
+        0.299 * background_color[0] + 0.587 * background_color[1] + 0.114 * background_color[0]
+    ) / 255
+    if background_brightness > brightness_threshold:
+        return COLOR_BLACK
+    else:
+        return COLOR_WHITE
+
+
+def get_label_rect_and_coords(
+    alignment: TextAlignment,
+    bbox_points: tuple[int, int, int, int],
+    text_width: int,
+    text_height: int,
+):
+    indent = max(int(text_width * 0.2), 10)
+    vertical_correction = max(int(text_height * 0.3), 10)
+
+    # with this the text should be centered in the rectangle
+    rect_width = text_width + indent * 2
+    # we apply a correction to the height to make the text not overlap over the rectangle
+    # (the text height getter looks to return too small value)
+    rect_height = text_height + vertical_correction
+    x1, y1, x2, y2 = bbox_points
+    if alignment is TextAlignment.CENTER:
+        # center:
+        horizontal_half = int(rect_width / 2 * 1.05)
+        vertical_half = int(rect_height / 2 * 1.05)
+        center_point = x1 + (x2 - x1) // 2, y1 + (y2 - y1) // 2
+        # resize rectangle to make it look better in the center
+        label_rectangle = (
+            (
+                center_point[0] - horizontal_half,
+                center_point[1] - vertical_half,
+            ),
+            (
+                center_point[0] + horizontal_half,
+                center_point[1] + vertical_half,
+            ),
+        )
+        label_coords = (
+            center_point[0] - horizontal_half + int(indent * 1.05),
+            center_point[1] - vertical_half * 1.05,
+        )
+    elif alignment is TextAlignment.TOP_LEFT:
+        label_rectangle = (
+            (x1, y1 - rect_height),
+            (x1 + rect_width, y1),
+        )
+        label_coords = (x1 + indent, y1 - rect_height)
+    elif alignment is TextAlignment.TOP_RIGHT:
+        label_rectangle = (
+            (x2 - rect_width, y1),
+            (x2, y1 + rect_height),
+        )
+        label_coords = (x2 - text_width - indent, y1)
+    elif alignment is TextAlignment.BOTTOM_LEFT:
+        label_rectangle = (
+            (x1, y2 - rect_height),
+            (x1 + rect_width, y2),
+        )
+        label_coords = (x1 + indent, y2 - rect_height)
+    elif alignment is TextAlignment.BOTTOM_RIGHT:
+        label_rectangle = (
+            (x2 - rect_width, y2 - rect_height),
+            (x2, y2),
+        )
+        label_coords = (x2 - text_width - indent, y2 - rect_height)
+    else:
+        raise ValueError(f"Unknown alignment {alignment}")
+    return label_rectangle, label_coords
+
+
+def draw_bbox_label(
+    image_draw: ImageDraw.ImageDraw,
+    text: str,
+    bbox_points: tuple[int, int, int, int],
+    alignment: TextAlignment,
+    font_size: int,
+    background_color: str,
+):
+    """Draw a label stick to a bounding box.
+    The alignment parameter specifies where the label should be placed.
+
+    Args:
+        image_draw:         ImageDraw object to draw on the image.
+        text:               Text to draw.
+        bbox_points:        Bounding box points.
+        alignment:          Text alignment.
+        font_size:          Font size of the text.
+        background_color:   RGB values of the background color.
+    """
+    font = ImageFont.truetype(get_font(), font_size)
+    text_x1, text_y1, text_x2, text_y2 = image_draw.textbbox(
+        (0, 0), text, font=font, align="center"
+    )
+    text_width = text_x2 - text_x1
+    text_height = text_y2 - text_y1
+
+    label_rectangle, label_coords = get_label_rect_and_coords(
+        alignment, bbox_points, text_width, text_height
+    )
+
+    rgb_background_color = get_rgb_color(background_color)
+    try:
+        image_draw.rectangle(
+            label_rectangle,
+            fill=background_color,
+            outline=background_color,
+        )
+    except TypeError:
+        image_draw.rectangle(
+            label_rectangle,
+            fill=rgb_background_color,
+            outline=rgb_background_color,
+        )
+    text_color, text_color_rgb = get_text_color(background_color)
+    try:
+        image_draw.text(label_coords, text, fill=text_color, font=font, align="center")
+    except TypeError:
+        image_draw.text(label_coords, text, fill=text_color_rgb, font=font, align="center")
+
+
+def draw_bbox_on_image(
+    image_draw: ImageDraw.ImageDraw,
+    bbox: BBox,
+    color: str,
+):
+    """Draw bbox with additional labels on the image..
+
+    Args:
+        image_draw:     ImageDraw object to draw on the image.
+        bbox:           Bounding box to draw.
+        color:          RGB values of the color of the bounding box (edges + label backgrounds).
+    """
+    x1, y1, x2, y2 = bbox.points
+    if x1 >= x2 or y1 >= y2:
+        print(f"Invalid bbox coordinates: {bbox.points}")
+        return
+    top_left = x1, y1  # the main
+    bottom_right = x2, y2
+    box_thickness = get_bbox_thickness(bbox=bbox.points, page_size=image_draw.im.size)
+    font_size = get_bbox_text_size(bbox=bbox.points, page_size=image_draw.im.size)
+
+    try:
+        image_draw.rectangle((top_left, bottom_right), outline=color, width=box_thickness)
+    except TypeError:
+        rgb_color = get_rgb_color(color)
+        image_draw.rectangle((top_left, bottom_right), outline=rgb_color, width=box_thickness)
+
+    if bbox.labels is not None:
+        if top_left_label := bbox.labels.top_left:
+            draw_bbox_label(
+                image_draw,
+                top_left_label,
+                bbox_points=bbox.points,
+                alignment=TextAlignment.TOP_LEFT,
+                font_size=font_size,
+                background_color=color,
+            )
+        if top_right_label := bbox.labels.top_right:
+            draw_bbox_label(
+                image_draw,
+                top_right_label,
+                bbox_points=bbox.points,
+                alignment=TextAlignment.TOP_RIGHT,
+                font_size=font_size,
+                background_color=color,
+            )
+        if bottom_left_label := bbox.labels.bottom_left:
+            draw_bbox_label(
+                image_draw,
+                bottom_left_label,
+                bbox_points=bbox.points,
+                alignment=TextAlignment.BOTTOM_LEFT,
+                font_size=font_size,
+                background_color=color,
+            )
+        if bottom_right_label := bbox.labels.bottom_right:
+            draw_bbox_label(
+                image_draw,
+                bottom_right_label,
+                bbox_points=bbox.points,
+                alignment=TextAlignment.BOTTOM_RIGHT,
+                font_size=font_size,
+                background_color=color,
+            )
+        if center_label := bbox.labels.center:
+            draw_bbox_label(
+                image_draw,
+                center_label,
+                bbox_points=bbox.points,
+                alignment=TextAlignment.CENTER,
+                font_size=font_size * 2,
+                background_color=color,
+            )
+
+
+@njit(cache=True, fastmath=True)
+def _linear_polyfit_2point(x0: float, x1: float, y0: float, y1: float):
+    """Compute slope and intercept for a line passing through (x0, y0), (x1, y1)."""
+    if x1 == x0:
+        slope = 0.0
+        intercept = (y0 + y1) / 2.0
+    else:
+        slope = (y1 - y0) / (x1 - x0)
+        intercept = y0 - slope * x0
+    return slope, intercept
+
+
+class LayoutDrawer(ABC):
+    layout_source: str = "unknown"
+    laytout_dump: dict
+
+    def __init__(self, layout_dump: dict):
+        self.layout_dump = layout_dump
+
+    def draw_layout_on_page(self, page_image: Image.Image, page_num: int) -> Image.Image:
+        """Draw the layout bboxes with additional metadata on the image."""
+        layout_pages = self.layout_dump.get("pages")
+        if not layout_pages:
+            print(f"Warning: layout in drawer {self.__class__.__name__} is empty - skipping")
+            return page_image
+        if len(layout_pages) < page_num:
+            print(f"Error! Page {page_num} not found in layout (pages: {len(layout_pages)})")
+            return page_image
+        image_draw = ImageDraw.ImageDraw(page_image)
+        page_layout_dump = layout_pages[page_num - 1]
+        if page_num != page_layout_dump.get("number"):
+            dump_page_num = page_layout_dump.get("number")
+            print(f"Warning: Requested page num {page_num} differs from dump {dump_page_num}")
+        for idx, elements in enumerate(page_layout_dump["elements"], 1):
+            self.render_element_on_page(idx, image_draw, elements)
+        return page_image
+
+    @abstractmethod
+    def render_element_on_page(self, idx: int, image_draw: ImageDraw, elements: dict[str, Any]):
+        """Draw a single element on the image."""
+
+
+class SimpleLayoutDrawer(LayoutDrawer, ABC):
+    color: str
+    show_order: bool = False
+    show_text_length: bool = False
+
+    def render_element_on_page(self, idx: int, image_draw: ImageDraw, elements: dict[str, Any]):
+        text_len = len(elements["text"]) if elements.get("text") else 0
+        element_prob = elements.get("prob")
+        element_order = f"{idx}" if self.show_order else None
+        text_len = f"len: {text_len}" if self.show_text_length else None
+        bbox = BBox(
+            points=elements["bbox"],
+            labels=BboxLabels(
+                top_right=f"prob: {element_prob:.2f}" if element_prob else None,
+                bottom_left=text_len,
+                center=element_order,
+            ),
+        )
+        draw_bbox_on_image(image_draw, bbox, color=self.color)
+
+
+class PdfminerLayoutDrawer(SimpleLayoutDrawer):
+    layout_source = "pdfminer"
+
+    def __init__(self, layout_dump: dict, color: str = "red"):
+        self.layout_dump = layout_dump
+        self.color = color
+        self.show_order = True
+        super().__init__(layout_dump)
+
+
+class OCRLayoutDrawer(SimpleLayoutDrawer):
+    layout_source = "ocr"
+
+    def __init__(self, layout_dump: dict, color: str = "red"):
+        self.color = color
+        self.show_order = False
+        self.show_text_length = False
+        super().__init__(layout_dump)
+
+
+class ODModelLayoutDrawer(LayoutDrawer):
+    layout_source = "od_model"
+
+    color_map = {
+        ElementType.CAPTION: "salmon",
+        ElementType.FOOTNOTE: "orange",
+        ElementType.FORMULA: "mediumpurple",
+        ElementType.LIST_ITEM: "navy",
+        ElementType.PAGE_FOOTER: "deeppink",
+        ElementType.PAGE_HEADER: "green",
+        ElementType.PICTURE: "sienna",
+        ElementType.SECTION_HEADER: "darkorange",
+        ElementType.TABLE: "blue",
+        ElementType.TEXT: "turquoise",
+        ElementType.TITLE: "greenyellow",
+    }
+
+    def render_element_on_page(self, idx: int, image_draw: ImageDraw, elements: dict[str, Any]):
+        element_type = elements["type"]
+        element_prob = elements.get("prob")
+        bbox_points = elements["bbox"]
+        color = self.get_element_type_color(element_type)
+        bbox = BBox(
+            points=bbox_points,
+            labels=BboxLabels(
+                top_left=f"{element_type}",
+                top_right=f"prob: {element_prob:.2f}" if element_prob else None,
+            ),
+        )
+        draw_bbox_on_image(image_draw, bbox, color=color)
+
+    def get_element_type_color(self, element_type: str) -> str:
+        return self.color_map.get(element_type, "cyan")
+
+
+class FinalLayoutDrawer(LayoutDrawer):
+    layout_source = "final"
+
+    color_map = {
+        "CheckBox": "brown",
+        "ListItem": "red",
+        "Title": "greenyellow",
+        "NarrativeText": "turquoise",
+        "Header": "green",
+        "Footer": "orange",
+        "FigureCaption": "sienna",
+        "Image": "sienna",
+        "Table": "blue",
+        "Address": "gold",
+        "EmailAddress": "lightskyblue",
+        "Formula": "mediumpurple",
+        "CodeSnippet": "magenta",
+        "PageNumber": "crimson",
+    }
+
+    def __init__(self, layout_dump: dict):
+        self.layout_dump = layout_dump
+
+    def render_element_on_page(self, idx: int, image_draw: ImageDraw, elements: dict[str, Any]):
+        element_type = elements["type"]
+        element_prob = elements.get("prob")
+        text_len = len(elements["text"]) if elements.get("text") else 0
+        bbox_points = elements["bbox"]
+        color = self.get_element_type_color(element_type)
+        cluster = elements.get("cluster")
+        bbox = BBox(
+            points=bbox_points,
+            labels=BboxLabels(
+                top_left=f"{element_type}",
+                top_right=f"prob: {element_prob:.2f}" if element_prob else None,
+                bottom_right=f"len: {text_len}",
+                bottom_left=f"cl: {cluster}" if cluster else None,
+                center=f"{idx}",
+            ),
+        )
+        draw_bbox_on_image(image_draw, bbox, color=color)
+
+    def get_element_type_color(self, element_type: str) -> str:
+        return self.color_map.get(element_type, "cyan")
+
+
+class AnalysisDrawer(AnalysisProcessor):
+    def __init__(
+        self,
+        filename: Optional[Union[str, Path]],
+        is_image: bool,
+        save_dir: Union[str, Path],
+        file: Optional[BytesIO] = None,
+        draw_caption: bool = True,
+        draw_grid: bool = False,
+        resize: Optional[float] = None,
+        format: str = "png",
+    ):
+        self.draw_caption = draw_caption
+        self.draw_grid = draw_grid
+        self.resize = resize
+        self.is_image = is_image
+        self.format = format
+        self.drawers = []
+        self.file = file
+
+        super().__init__(filename, save_dir)
+
+    def add_drawer(self, drawer: LayoutDrawer):
+        self.drawers.append(drawer)
+
+    def process(self):
+        filename_stem = Path(self.filename).stem
+        analysis_save_dir = Path(self.save_dir) / "analysis" / filename_stem / "bboxes"
+        analysis_save_dir.mkdir(parents=True, exist_ok=True)
+
+        for page_idx, orig_image_page in enumerate(self.load_source_image()):
+            images_for_grid = []
+            page_num = page_idx + 1
+            for drawer in self.drawers:
+                try:
+                    image = drawer.draw_layout_on_page(orig_image_page.copy(), page_num=page_num)
+                except:  # noqa: E722
+                    logging.exception(
+                        f"Error while drawing layout for page {page_num} "
+                        f"for file {self.filename} with drawer "
+                        f"{drawer.__class__.__name__}"
+                    )
+                    continue
+                if self.draw_caption:
+                    image = self.add_caption(
+                        image, caption=f"Layout source: {drawer.layout_source}"
+                    )
+                if not self.draw_grid:
+                    if self.resize is not None:
+                        image = image.resize(
+                            (int(image.width * self.resize), int(image.height * self.resize)),
+                        )
+                    image.save(
+                        analysis_save_dir / f"page{page_num}"
+                        f"_layout_{drawer.layout_source}.{self.format}",
+                        optimize=True,
+                        quality=85,
+                    )
+                    image.close()
+                else:
+                    images_for_grid.append(image)
+            if images_for_grid:
+                grid_image = self.paste_images_on_grid(images_for_grid)
+                if self.resize is not None:
+                    grid_image = grid_image.resize(
+                        (int(grid_image.width * self.resize), int(grid_image.height * self.resize))
+                    )
+                grid_image.save(
+                    analysis_save_dir / f"page{page_num}_layout_all.{self.format}",
+                    optimize=True,
+                    quality=85,
+                )
+                grid_image.close()
+
+    def add_caption(self, image: Image.Image, caption: str):
+        font = ImageFont.truetype(get_font(), 52)
+        draw = ImageDraw.ImageDraw(image)
+        text_x1, text_y1, text_x2, text_y2 = draw.textbbox(
+            (0, 0), caption, font=font, align="center"
+        )
+        text_width = text_x2 - text_x1
+        text_height = int((text_y2 - text_y1) * 1.5)
+        text_xy = (image.width - text_width) // 2, 10
+        caption_image = Image.new("RGB", (image.width, text_height), color=(255, 255, 255))
+        caption_draw = ImageDraw.ImageDraw(caption_image)
+        caption_draw.text(text_xy, caption, (0, 0, 0), font=font)
+
+        expanded_image = Image.new("RGB", (image.width, image.height + text_height))
+        expanded_image.paste(caption_image, (0, 0))
+        expanded_image.paste(image, (0, text_height))
+        image.close()
+        return expanded_image
+
+    def paste_images_on_grid(self, images: List[Image.Image]) -> Image.Image:
+        """Creates a single image that presents all the images on a grid 2 x n/2"""
+
+        pairs = []
+        for i in range(0, len(images), 2):
+            left_image = images[i]
+            right_image = images[i + 1] if i < len(images) - 1 else None
+            pairs.append((left_image, right_image))
+
+        max_pair_width = max([pair[0].width + (pair[1].width if pair[1] else 0) for pair in pairs])
+        sum_height = sum([max(pair[0].height, pair[1].height if pair[1] else 0) for pair in pairs])
+
+        new_im = Image.new("RGB", (max_pair_width, sum_height))
+
+        height_shift = 0
+        for image_left, image_right in pairs:
+            new_im.paste(image_left, (0, height_shift))
+            if image_right:
+                new_im.paste(image_right, (image_left.width, height_shift))
+            height_shift += max(image_left.height, image_right.height if image_right else 0)
+
+        for image in images:
+            image.close()
+        return new_im
+
+    def load_source_image(self) -> Generator[Image.Image, None, None]:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            image_paths = []
+            if self.is_image:
+                if self.file:
+                    try:
+                        image = Image.open(self.file)
+                        output_file = Path(temp_dir) / self.filename
+                        image.save(output_file, format="PNG")
+                        image_paths = [output_file]
+                    except Exception as ex:  # noqa: E722
+                        print(
+                            f"Error while converting image to PNG for file {self.filename}, "
+                            f"exception: {ex}"
+                        )
+                else:
+                    image_paths = [self.filename]
+            else:
+                try:
+                    str_filename = (
+                        ""
+                        if (self.filename is None or self.file is not None)
+                        else (
+                            str(self.filename) if isinstance(self.filename, Path) else self.filename
+                        )
+                    )
+                    image_paths = convert_pdf_to_image(
+                        filename=str_filename,
+                        file=self.file,
+                        output_folder=temp_dir,
+                        path_only=True,
+                    )
+                except Exception as ex:  # noqa: E722
+                    print(
+                        f"Error while converting pdf to image for file {self.filename}",
+                        f"exception: {ex}",
+                    )
+
+            for image_path in image_paths:
+                with Image.open(str(image_path)) as image:
+                    yield image.convert("RGB")

unstructured/partition/pdf_image/analysis/layout_dump.py

@@ -0,0 +1,203 @@
+import json
+from abc import ABC, abstractmethod
+from collections import defaultdict
+from pathlib import Path
+from typing import List, Optional
+
+from unstructured_inference.inference.elements import ImageTextRegion, TextRegion
+from unstructured_inference.inference.layout import DocumentLayout
+from unstructured_inference.models.base import get_model
+from unstructured_inference.models.detectron2onnx import (
+    DEFAULT_LABEL_MAP as DETECTRON_LABEL_MAP,
+)
+from unstructured_inference.models.detectron2onnx import (
+    UnstructuredDetectronONNXModel,
+)
+from unstructured_inference.models.yolox import YOLOX_LABEL_MAP, UnstructuredYoloXModel
+
+from unstructured.documents.elements import Element, Text
+from unstructured.partition.pdf_image.analysis.processor import AnalysisProcessor
+from unstructured.partition.utils.sorting import coordinates_to_bbox
+
+
+class LayoutDumper(ABC):
+    layout_source: str = "unknown"
+
+    @abstractmethod
+    def dump(self) -> dict:
+        """Transforms the results to a dict convertible structured formats like JSON or YAML"""
+
+
+def extract_document_layout_info(layout: DocumentLayout) -> dict:
+    pages = []
+
+    for page in layout.pages:
+        size = {
+            "width": page.image_metadata.get("width"),
+            "height": page.image_metadata.get("height"),
+        }
+        elements = []
+        for element in page.elements:
+            bbox = element.bbox
+            elements.append(
+                {
+                    "bbox": [bbox.x1, bbox.y1, bbox.x2, bbox.y2],
+                    "type": element.type,
+                    "prob": element.prob,
+                }
+            )
+        pages.append({"number": page.number, "size": size, "elements": elements})
+    return {"pages": pages}
+
+
+def object_detection_classes(model_name) -> List[str]:
+    model = get_model(model_name)
+    if isinstance(model, UnstructuredYoloXModel):
+        return list(YOLOX_LABEL_MAP.values())
+    if isinstance(model, UnstructuredDetectronONNXModel):
+        return list(DETECTRON_LABEL_MAP.values())
+    else:
+        raise ValueError(f"Cannot get OD model classes - unknown model type: {model_name}")
+
+
+class ObjectDetectionLayoutDumper(LayoutDumper):
+    """Forms the results in COCO format and saves them to a file"""
+
+    layout_source = "object_detection"
+
+    def __init__(self, layout: DocumentLayout, model_name: Optional[str] = None):
+        self.layout: dict = extract_document_layout_info(layout)
+        self.model_name = model_name
+
+    def dump(self) -> dict:
+        """Transforms the results to COCO format and saves them to a file"""
+        try:
+            classes_dict = {"object_detection_classes": object_detection_classes(self.model_name)}
+        except ValueError:
+            classes_dict = {"object_detection_classes": []}
+        self.layout.update(classes_dict)
+        return self.layout
+
+
+def _get_info_from_extracted_page(page: List[TextRegion]) -> List[dict]:
+    elements = []
+    for element in page:
+        is_image = isinstance(element, ImageTextRegion)
+        bbox = element.bbox
+        elements.append(
+            {
+                "bbox": [bbox.x1, bbox.y1, bbox.x2, bbox.y2],
+                "text": element.text,
+                "source": str(element.source.value),
+                "is_image": is_image,
+            }
+        )
+    return elements
+
+
+def extract_text_regions_info(layout: List[List[TextRegion]]) -> dict:
+    pages = []
+    for page_num, page in enumerate(layout, 1):
+        elements = _get_info_from_extracted_page(page)
+        pages.append({"number": page_num, "elements": elements})
+    return {"pages": pages}
+
+
+class ExtractedLayoutDumper(LayoutDumper):
+    layout_source = "pdfminer"
+
+    def __init__(self, layout: List[List[TextRegion]]):
+        self.layout = extract_text_regions_info(layout)
+
+    def dump(self) -> dict:
+        return self.layout
+
+
+class OCRLayoutDumper(LayoutDumper):
+    layout_source = "ocr"
+
+    def __init__(self):
+        self.layout = []
+        self.page_number = 1
+
+    def add_ocred_page(self, page: List[TextRegion]):
+        elements = _get_info_from_extracted_page(page)
+        self.layout.append({"number": self.page_number, "elements": elements})
+        self.page_number += 1
+
+    def dump(self) -> dict:
+        return {"pages": self.layout}
+
+
+def _extract_final_element_info(element: Element) -> dict:
+    element_type = (
+        element.category if isinstance(element, Text) else str(element.__class__.__name__)
+    )
+    element_prob = getattr(element.metadata, "detection_class_prob", None)
+    text = element.text
+    bbox_points = coordinates_to_bbox(element.metadata.coordinates)
+    cluster = getattr(element.metadata, "cluster", None)
+    return {
+        "type": element_type,
+        "prob": element_prob,
+        "text": text,
+        "bbox": bbox_points,
+        "cluster": cluster,
+    }
+
+
+def _extract_final_element_page_size(element: Element) -> dict:
+    try:
+        return {
+            "width": element.metadata.coordinates.system.width,
+            "height": element.metadata.coordinates.system.height,
+        }
+    except AttributeError:
+        return {
+            "width": None,
+            "height": None,
+        }
+
+
+class FinalLayoutDumper(LayoutDumper):
+    layout_source = "final"
+
+    def __init__(self, layout: List[Element]):
+        pages = defaultdict(list)
+        for element in layout:
+            element_page_number = element.metadata.page_number
+            pages[element_page_number].append(_extract_final_element_info(element))
+        extracted_pages = [
+            {
+                "number": page_number,
+                "size": (
+                    _extract_final_element_page_size(page_elements[0]) if page_elements else None
+                ),
+                "elements": page_elements,
+            }
+            for page_number, page_elements in pages.items()
+        ]
+        self.layout = {"pages": sorted(extracted_pages, key=lambda x: x["number"])}
+
+    def dump(self) -> dict:
+        return self.layout
+
+
+class JsonLayoutDumper(AnalysisProcessor):
+    """Dumps the results of the analysis to a JSON file"""
+
+    def __init__(self, filename: str, save_dir: str):
+        self.dumpers = []
+        super().__init__(filename, save_dir)
+
+    def add_layout_dumper(self, dumper: LayoutDumper):
+        self.dumpers.append(dumper)
+
+    def process(self):
+        filename_stem = Path(self.filename).stem
+        analysis_save_dir = Path(self.save_dir) / "analysis" / filename_stem / "layout_dump"
+        analysis_save_dir.mkdir(parents=True, exist_ok=True)
+        for dumper in self.dumpers:
+            results = dumper.dump()
+            with open(analysis_save_dir / f"{dumper.layout_source}.json", "w") as f:
+                f.write(json.dumps(results, indent=2))

unstructured/partition/pdf_image/analysis/processor.py

@@ -0,0 +1,18 @@
+from abc import ABC, abstractmethod
+from pathlib import Path
+from typing import Union
+
+
+class AnalysisProcessor(ABC):
+    def __init__(
+        self,
+        filename: Union[str, Path],
+        save_dir: Union[str, Path],
+    ):
+        self.filename = filename
+        self.save_dir = save_dir
+
+    @abstractmethod
+    def process(self):
+        """Performs the analysis and saves the results"""
+        raise NotImplementedError()

unstructured/partition/pdf_image/analysis/tools.py

@@ -0,0 +1,190 @@
+import json
+import uuid
+from io import BytesIO
+from pathlib import Path
+from typing import Optional
+
+from unstructured.partition.pdf_image.analysis.bbox_visualisation import (
+    AnalysisDrawer,
+    FinalLayoutDrawer,
+    LayoutDrawer,
+    OCRLayoutDrawer,
+    ODModelLayoutDrawer,
+    PdfminerLayoutDrawer,
+)
+from unstructured.partition.pdf_image.analysis.layout_dump import (
+    ExtractedLayoutDumper,
+    FinalLayoutDumper,
+    JsonLayoutDumper,
+    LayoutDumper,
+    ObjectDetectionLayoutDumper,
+    OCRLayoutDumper,
+)
+
+
+def _get_drawer_for_dumper(dumper: LayoutDumper) -> Optional[LayoutDrawer]:
+    """For a given layout dumper, return the corresponding layout drawer instance initialized with
+    a dumped layout dict.
+
+    Args:
+        dumper: The layout dumper instance
+
+    Returns:
+        LayoutDrawer: The corresponding layout drawer instance
+    """
+    if isinstance(dumper, ObjectDetectionLayoutDumper):
+        return ODModelLayoutDrawer(layout_dump=dumper.dump())
+    elif isinstance(dumper, ExtractedLayoutDumper):
+        return PdfminerLayoutDrawer(layout_dump=dumper.dump())
+    elif isinstance(dumper, OCRLayoutDumper):
+        return OCRLayoutDrawer(layout_dump=dumper.dump())
+    elif isinstance(dumper, FinalLayoutDumper):
+        return FinalLayoutDrawer(layout_dump=dumper.dump())
+    else:
+        raise ValueError(f"Unknown dumper type: {dumper}")
+
+
+def _generate_filename(is_image: bool):
+    """Generate a filename for the analysis artifacts based on the file type.
+    Adds a random uuid suffix
+    """
+    suffix = uuid.uuid4().hex[:5]
+    if is_image:
+        return f"image_{suffix}.png"
+    return f"pdf_{suffix}.pdf"
+
+
+def save_analysis_artifiacts(
+    *layout_dumpers: LayoutDumper,
+    is_image: bool,
+    analyzed_image_output_dir_path: str,
+    filename: Optional[str] = None,
+    file: Optional[BytesIO] = None,
+    skip_bboxes: bool = False,
+    skip_dump_od: bool = False,
+    draw_grid: bool = False,
+    draw_caption: bool = True,
+    resize: Optional[float] = None,
+    format: str = "png",
+):
+    """Save the analysis artifacts for a given file. Loads some settings from
+    the environment configuration.
+
+    Args:
+        layout_dumpers: The layout dumpers to save and use for bboxes rendering
+        is_image: Flag for the file type (pdf/image)
+        analyzed_image_output_dir_path: The directory to save the analysis artifacts
+        filename: The filename of the sources analyzed file (pdf/image).
+            Only one of filename or file should be provided.
+        file: The file object for the analyzed file.
+            Only one of filename or file should be provided.
+        draw_grid: Flag for drawing the analysis bboxes on a single image (as grid)
+        draw_caption: Flag for drawing the caption above the analyzed page (for e.g. layout source)
+        resize: Output image resize value. If not provided, the image will not be resized.
+        format: The format for analyzed pages with bboxes drawn on them. Default is 'png'.
+    """
+    if not filename:
+        filename = _generate_filename(is_image)
+    if skip_bboxes or skip_dump_od:
+        return
+
+    output_path = Path(analyzed_image_output_dir_path)
+    output_path.mkdir(parents=True, exist_ok=True)
+    if not skip_dump_od:
+        json_layout_dumper = JsonLayoutDumper(
+            filename=filename,
+            save_dir=output_path,
+        )
+        for layout_dumper in layout_dumpers:
+            json_layout_dumper.add_layout_dumper(layout_dumper)
+        json_layout_dumper.process()
+
+    if not skip_bboxes:
+        analysis_drawer = AnalysisDrawer(
+            filename=filename,
+            file=file,
+            is_image=is_image,
+            save_dir=output_path,
+            draw_grid=draw_grid,
+            draw_caption=draw_caption,
+            resize=resize,
+            format=format,
+        )
+
+        for layout_dumper in layout_dumpers:
+            drawer = _get_drawer_for_dumper(layout_dumper)
+            analysis_drawer.add_drawer(drawer)
+        analysis_drawer.process()
+
+
+def render_bboxes_for_file(
+    filename: str,
+    analyzed_image_output_dir_path: str,
+    renders_output_dir_path: Optional[str] = None,
+    draw_grid: bool = False,
+    draw_caption: bool = True,
+    resize: Optional[float] = None,
+    format: str = "png",
+):
+    """Render the bounding boxes for a given layout dimp file.
+    To be used for analysis after the partition is performed for
+    only dumping the layouts - the bboxes can be rendered later.
+
+    Expects that the analyzed_image_output_dir_path keeps the structure
+    that was created by the save_analysis_artifacts function.
+
+    Args:
+        filename: The filename of the sources analyzed file (pdf/image)
+        analyzed_image_output_dir_path: The directory where the analysis artifacts
+          (layout dumps) are saved. It should be the root directory of the structure
+          created by the save_analysis_artifacts function.
+        renders_output_dir_path: Optional directory to save the rendered bboxes -
+          if not provided, it will be saved in the analysis directory.
+        draw_grid: Flag for drawing the analysis bboxes on a single image (as grid)
+        draw_caption: Flag for drawing the caption above the analyzed page (for e.g. layout source)
+        resize: Output image resize value. If not provided, the image will not be resized.
+        format: The format for analyzed pages with bboxes drawn on them. Default is 'png'.
+    """
+    filename_stem = Path(filename).stem
+    is_image = not Path(filename).suffix.endswith("pdf")
+    analysis_dumps_dir = (
+        Path(analyzed_image_output_dir_path) / "analysis" / filename_stem / "layout_dump"
+    )
+    if not analysis_dumps_dir.exists():
+        return
+    layout_drawers = []
+    for analysis_dump_filename in analysis_dumps_dir.iterdir():
+        if not analysis_dump_filename.is_file():
+            continue
+        with open(analysis_dump_filename) as f:
+            layout_dump = json.load(f)
+        if analysis_dump_filename.stem == "final":
+            layout_drawers.append(FinalLayoutDrawer(layout_dump=layout_dump))
+        if analysis_dump_filename.stem == "object_detection":
+            layout_drawers.append(ODModelLayoutDrawer(layout_dump=layout_dump))
+        if analysis_dump_filename.stem == "ocr":
+            layout_drawers.append(OCRLayoutDrawer(layout_dump=layout_dump))
+        if analysis_dump_filename.stem == "pdfminer":
+            layout_drawers.append(PdfminerLayoutDrawer(layout_dump=layout_dump))
+
+    if layout_drawers:
+        if not renders_output_dir_path:
+            output_path = (
+                Path(analyzed_image_output_dir_path) / "analysis" / filename_stem / "bboxes"
+            )
+        else:
+            output_path = Path(renders_output_dir_path)
+        output_path.mkdir(parents=True, exist_ok=True)
+        analysis_drawer = AnalysisDrawer(
+            filename=filename,
+            save_dir=output_path,
+            is_image=is_image,
+            draw_grid=draw_grid,
+            draw_caption=draw_caption,
+            resize=resize,
+            format=format,
+        )
+
+        for drawer in layout_drawers:
+            analysis_drawer.add_drawer(drawer)
+        analysis_drawer.process()

unstructured/partition/pdf_image/form_extraction.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from typing import IO
+
+from unstructured.documents.elements import Element, FormKeysValues
+
+
+def run_form_extraction(
+    filename: str,
+    file: IO[bytes],
+    model_name: str,
+    elements: list[Element],
+    skip_table_regions: bool,
+) -> list[FormKeysValues]:
+    raise NotImplementedError("Form extraction not yet available.")

unstructured/partition/pdf_image/inference_utils.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Optional
+
+import numpy as np
+from unstructured_inference.constants import Source
+from unstructured_inference.inference.elements import TextRegion, TextRegions
+from unstructured_inference.inference.layoutelement import (
+    LayoutElement,
+    LayoutElements,
+    partition_groups_from_regions,
+)
+
+from unstructured.documents.elements import ElementType
+
+if TYPE_CHECKING:
+    from unstructured_inference.inference.elements import Rectangle
+
+
+def build_text_region_from_coords(
+    x1: int | float,
+    y1: int | float,
+    x2: int | float,
+    y2: int | float,
+    text: Optional[str] = None,
+    source: Optional[Source] = None,
+) -> TextRegion:
+    """"""
+    return TextRegion.from_coords(x1, y1, x2, y2, text=text, source=source)
+
+
+def build_layout_element(
+    bbox: "Rectangle",
+    text: Optional[str] = None,
+    source: Optional[Source] = None,
+    element_type: Optional[str] = None,
+) -> LayoutElement:
+    """"""
+
+    return LayoutElement(bbox=bbox, text=text, source=source, type=element_type)
+
+
+def build_layout_elements_from_ocr_regions(
+    ocr_regions: TextRegions,
+    ocr_text: Optional[str] = None,
+    group_by_ocr_text: bool = False,
+) -> LayoutElements:
+    """
+    Get layout elements from OCR regions
+    """
+
+    grouped_regions = []
+    if group_by_ocr_text:
+        text_sections = ocr_text.split("\n\n")
+        mask = np.ones(ocr_regions.texts.shape).astype(bool)
+        indices = np.arange(len(mask))
+        for text_section in text_sections:
+            regions = []
+            words = text_section.replace("\n", " ").split()
+            for i, text in enumerate(ocr_regions.texts[mask]):
+                if not words:
+                    break
+                if text in words:
+                    regions.append(indices[mask][i])
+                    words.remove(text)
+
+            if not regions:
+                continue
+
+            mask[regions] = False
+            grouped_regions.append(ocr_regions.slice(regions))
+    else:
+        grouped_regions = partition_groups_from_regions(ocr_regions)
+
+    merged_regions = TextRegions.from_list([merge_text_regions(group) for group in grouped_regions])
+    return LayoutElements(
+        element_coords=merged_regions.element_coords,
+        texts=merged_regions.texts,
+        sources=merged_regions.sources,
+        element_class_ids=np.zeros(merged_regions.texts.shape),
+        element_class_id_map={0: ElementType.UNCATEGORIZED_TEXT},
+    )
+
+
+def merge_text_regions(regions: TextRegions) -> TextRegion:
+    """
+    Merge a list of TextRegion objects into a single TextRegion.
+
+    Parameters:
+    - group (TextRegions): A group of TextRegion objects to be merged.
+
+    Returns:
+    - TextRegion: A single merged TextRegion object.
+    """
+
+    if not regions:
+        raise ValueError("The text regions to be merged must be provided.")
+
+    min_x1 = regions.x1.min().astype(float)
+    min_y1 = regions.y1.min().astype(float)
+    max_x2 = regions.x2.max().astype(float)
+    max_y2 = regions.y2.max().astype(float)
+
+    merged_text = " ".join([text for text in regions.texts if text])
+    # assumption is the regions has the same source
+    source = regions.sources[0]
+
+    return TextRegion.from_coords(min_x1, min_y1, max_x2, max_y2, merged_text, source)

unstructured/partition/pdf_image/ocr.py

@@ -0,0 +1,491 @@
+from __future__ import annotations
+
+import os
+import tempfile
+from typing import IO, TYPE_CHECKING, Any, List, Optional, cast
+
+import numpy as np
+
+# NOTE(yuming): Rename PIL.Image to avoid conflict with
+# unstructured.documents.elements.Image
+from PIL import Image as PILImage
+from PIL import ImageSequence
+
+from unstructured.documents.elements import ElementType
+from unstructured.metrics.table.table_formats import SimpleTableCell
+from unstructured.partition.common.lang import tesseract_to_paddle_language
+from unstructured.partition.pdf_image.analysis.layout_dump import OCRLayoutDumper
+from unstructured.partition.pdf_image.pdf_image_utils import convert_pdf_to_image, valid_text
+from unstructured.partition.pdf_image.pdfminer_processing import (
+    aggregate_embedded_text_by_block,
+    bboxes1_is_almost_subregion_of_bboxes2,
+)
+from unstructured.partition.utils.config import env_config
+from unstructured.partition.utils.constants import OCR_AGENT_PADDLE, OCR_AGENT_TESSERACT, OCRMode
+from unstructured.partition.utils.ocr_models.ocr_interface import OCRAgent
+from unstructured.utils import requires_dependencies
+
+if TYPE_CHECKING:
+    from unstructured_inference.inference.elements import TextRegion, TextRegions
+    from unstructured_inference.inference.layout import DocumentLayout, PageLayout
+    from unstructured_inference.inference.layoutelement import LayoutElement, LayoutElements
+    from unstructured_inference.models.tables import UnstructuredTableTransformerModel
+
+
+def process_data_with_ocr(
+    data: bytes | IO[bytes],
+    out_layout: "DocumentLayout",
+    extracted_layout: List[List["TextRegion"]],
+    is_image: bool = False,
+    infer_table_structure: bool = False,
+    ocr_agent: str = OCR_AGENT_TESSERACT,
+    ocr_languages: str = "eng",
+    ocr_mode: str = OCRMode.FULL_PAGE.value,
+    pdf_image_dpi: int = env_config.PDF_RENDER_DPI,
+    ocr_layout_dumper: Optional[OCRLayoutDumper] = None,
+    password: Optional[str] = None,
+    table_ocr_agent: str = OCR_AGENT_TESSERACT,
+) -> "DocumentLayout":
+    """
+    Process OCR data from a given data and supplement the output DocumentLayout
+    from unstructured_inference with ocr.
+
+    Parameters:
+    - data (Union[bytes, BinaryIO]): The input file data,
+        which can be either bytes or a BinaryIO object.
+
+    - out_layout (DocumentLayout): The output layout from unstructured-inference.
+
+    - is_image (bool, optional): Indicates if the input data is an image (True) or not (False).
+        Defaults to False.
+
+    - infer_table_structure (bool, optional):  If true, extract the table content.
+
+    - ocr_languages (str, optional): The languages for OCR processing. Defaults to "eng" (English).
+
+    - ocr_mode (str, optional): The OCR processing mode, e.g., "entire_page" or "individual_blocks".
+        Defaults to "entire_page". If choose "entire_page" OCR, OCR processes the entire image
+        page and will be merged with the output layout. If choose "individual_blocks" OCR,
+        OCR is performed on individual elements by cropping the image.
+
+    - pdf_image_dpi (int, optional): DPI (dots per inch) for processing PDF images. Defaults to
+      env_config.PDF_RENDER_DPI's value.
+
+    - ocr_layout_dumper (OCRLayoutDumper, optional): The OCR layout dumper to save the OCR layout.
+
+    Returns:
+        DocumentLayout: The merged layout information obtained after OCR processing.
+    """
+    data_bytes = data if isinstance(data, bytes) else data.read()
+
+    with tempfile.TemporaryDirectory() as tmp_dir_path:
+        tmp_file_path = os.path.join(tmp_dir_path, "tmp_file")
+        with open(tmp_file_path, "wb") as tmp_file:
+            tmp_file.write(data_bytes)
+
+        merged_layouts = process_file_with_ocr(
+            filename=tmp_file_path,
+            out_layout=out_layout,
+            extracted_layout=extracted_layout,
+            is_image=is_image,
+            infer_table_structure=infer_table_structure,
+            ocr_agent=ocr_agent,
+            ocr_languages=ocr_languages,
+            ocr_mode=ocr_mode,
+            pdf_image_dpi=pdf_image_dpi,
+            ocr_layout_dumper=ocr_layout_dumper,
+            password=password,
+            table_ocr_agent=table_ocr_agent,
+        )
+
+    return merged_layouts
+
+
+@requires_dependencies("unstructured_inference")
+def process_file_with_ocr(
+    filename: str,
+    out_layout: "DocumentLayout",
+    extracted_layout: List[TextRegions],
+    is_image: bool = False,
+    infer_table_structure: bool = False,
+    ocr_agent: str = OCR_AGENT_TESSERACT,
+    ocr_languages: str = "eng",
+    ocr_mode: str = OCRMode.FULL_PAGE.value,
+    pdf_image_dpi: int = env_config.PDF_RENDER_DPI,
+    ocr_layout_dumper: Optional[OCRLayoutDumper] = None,
+    password: Optional[str] = None,
+    table_ocr_agent: str = OCR_AGENT_TESSERACT,
+) -> "DocumentLayout":
+    """
+    Process OCR data from a given file and supplement the output DocumentLayout
+    from unstructured-inference with ocr.
+
+    Parameters:
+    - filename (str): The path to the input file, which can be an image or a PDF.
+
+    - out_layout (DocumentLayout): The output layout from unstructured-inference.
+
+    - extracted_layout (List[TextRegions]): a list of text regions extracted by pdfminer, one for
+      each page
+
+    - is_image (bool, optional): Indicates if the input data is an image (True) or not (False).
+        Defaults to False.
+
+    - infer_table_structure (bool, optional):  If true, extract the table content.
+
+    - ocr_languages (str, optional): The languages for OCR processing. Defaults to "eng" (English).
+
+    - ocr_mode (str, optional): The OCR processing mode, e.g., "entire_page" or "individual_blocks".
+        Defaults to "entire_page". If choose "entire_page" OCR, OCR processes the entire image
+        page and will be merged with the output layout. If choose "individual_blocks" OCR,
+        OCR is performed on individual elements by cropping the image.
+
+    - pdf_image_dpi (int, optional): DPI (dots per inch) for processing PDF images. Defaults to
+      env_config.PDF_RENDER_DPI.
+
+    Returns:
+        DocumentLayout: The merged layout information obtained after OCR processing.
+    """
+
+    from unstructured_inference.inference.layout import DocumentLayout
+
+    merged_page_layouts: list[PageLayout] = []
+    try:
+        if is_image:
+            with PILImage.open(filename) as images:
+                image_format = images.format
+                for i, image in enumerate(ImageSequence.Iterator(images)):
+                    image = image.convert("RGB")
+                    image.format = image_format
+                    extracted_regions = extracted_layout[i] if i < len(extracted_layout) else None
+                    merged_page_layout = supplement_page_layout_with_ocr(
+                        page_layout=out_layout.pages[i],
+                        image=image,
+                        infer_table_structure=infer_table_structure,
+                        ocr_agent=ocr_agent,
+                        ocr_languages=ocr_languages,
+                        ocr_mode=ocr_mode,
+                        extracted_regions=extracted_regions,
+                        ocr_layout_dumper=ocr_layout_dumper,
+                        table_ocr_agent=table_ocr_agent,
+                    )
+                    merged_page_layouts.append(merged_page_layout)
+                return DocumentLayout.from_pages(merged_page_layouts)
+        else:
+            with tempfile.TemporaryDirectory() as temp_dir:
+                _image_paths = convert_pdf_to_image(
+                    filename,
+                    dpi=pdf_image_dpi,
+                    output_folder=temp_dir,
+                    path_only=True,
+                    password=password,
+                )
+                image_paths = cast(List[str], _image_paths)
+
+                for i, image_path in enumerate(image_paths):
+                    extracted_regions = extracted_layout[i] if i < len(extracted_layout) else None
+                    with PILImage.open(image_path) as image:
+                        merged_page_layout = supplement_page_layout_with_ocr(
+                            page_layout=out_layout.pages[i],
+                            image=image,
+                            infer_table_structure=infer_table_structure,
+                            ocr_agent=ocr_agent,
+                            ocr_languages=ocr_languages,
+                            ocr_mode=ocr_mode,
+                            extracted_regions=extracted_regions,
+                            ocr_layout_dumper=ocr_layout_dumper,
+                            table_ocr_agent=table_ocr_agent,
+                        )
+                        merged_page_layouts.append(merged_page_layout)
+
+                return DocumentLayout.from_pages(merged_page_layouts)
+    except Exception as e:
+        if os.path.isdir(filename) or os.path.isfile(filename):
+            raise e
+        else:
+            raise FileNotFoundError(f'File "{filename}" not found!') from e
+
+
+@requires_dependencies("unstructured_inference")
+def supplement_page_layout_with_ocr(
+    page_layout: "PageLayout",
+    image: PILImage.Image,
+    infer_table_structure: bool = False,
+    ocr_agent: str = OCR_AGENT_TESSERACT,
+    ocr_languages: str = "eng",
+    ocr_mode: str = OCRMode.FULL_PAGE.value,
+    extracted_regions: Optional[TextRegions] = None,
+    ocr_layout_dumper: Optional[OCRLayoutDumper] = None,
+    table_ocr_agent: str = OCR_AGENT_TESSERACT,
+) -> "PageLayout":
+    """
+    Supplement an PageLayout with OCR results depending on OCR mode.
+    If mode is "entire_page", we get the OCR layout for the entire image and
+    merge it with PageLayout.
+    If mode is "individual_blocks", we find the elements from PageLayout
+    with no text and add text from OCR to each element.
+    """
+
+    language = ocr_languages
+    if ocr_agent == OCR_AGENT_PADDLE:
+        language = tesseract_to_paddle_language(ocr_languages)
+    _ocr_agent = OCRAgent.get_instance(ocr_agent_module=ocr_agent, language=language)
+    if ocr_mode == OCRMode.FULL_PAGE.value:
+        ocr_layout = _ocr_agent.get_layout_from_image(image)
+        if ocr_layout_dumper:
+            ocr_layout_dumper.add_ocred_page(ocr_layout.as_list())
+        page_layout.elements_array = merge_out_layout_with_ocr_layout(
+            out_layout=page_layout.elements_array,
+            ocr_layout=ocr_layout,
+        )
+    elif ocr_mode == OCRMode.INDIVIDUAL_BLOCKS.value:
+        # individual block mode still keeps using the list data structure for elements instead of
+        # the vectorized page_layout.elements_array data structure
+        for i, text in enumerate(page_layout.elements_array.texts):
+            if text:
+                continue
+            padding = env_config.IMAGE_CROP_PAD
+            cropped_image = image.crop(
+                (
+                    page_layout.elements_array.x1[i] - padding,
+                    page_layout.elements_array.y1[i] - padding,
+                    page_layout.elements_array.x2[i] + padding,
+                    page_layout.elements_array.y2[i] + padding,
+                ),
+            )
+            # Note(yuming): instead of getting OCR layout, we just need
+            # the text extraced from OCR for individual elements
+            text_from_ocr = _ocr_agent.get_text_from_image(cropped_image)
+            page_layout.elements_array.texts[i] = text_from_ocr
+    else:
+        raise ValueError(
+            "Invalid OCR mode. Parameter `ocr_mode` "
+            "must be set to `entire_page` or `individual_blocks`.",
+        )
+
+    # Note(yuming): use the OCR data from entire page OCR for table extraction
+    if infer_table_structure:
+        language = ocr_languages
+        if table_ocr_agent == OCR_AGENT_PADDLE:
+            language = tesseract_to_paddle_language(ocr_languages)
+        _table_ocr_agent = OCRAgent.get_instance(
+            ocr_agent_module=table_ocr_agent, language=language
+        )
+        from unstructured_inference.models import tables
+
+        tables.load_agent()
+        if tables.tables_agent is None:
+            raise RuntimeError("Unable to load table extraction agent.")
+
+        page_layout.elements_array = supplement_element_with_table_extraction(
+            elements=page_layout.elements_array,
+            image=image,
+            tables_agent=tables.tables_agent,
+            ocr_agent=_table_ocr_agent,
+            extracted_regions=extracted_regions,
+        )
+
+    return page_layout
+
+
+@requires_dependencies("unstructured_inference")
+def supplement_element_with_table_extraction(
+    elements: LayoutElements,
+    image: PILImage.Image,
+    tables_agent: "UnstructuredTableTransformerModel",
+    ocr_agent,
+    extracted_regions: Optional[TextRegions] = None,
+) -> List["LayoutElement"]:
+    """Supplement the existing layout with table extraction. Any Table elements
+    that are extracted will have a metadata fields "text_as_html" where
+    the table's text content is rendered into a html string and "table_as_cells"
+    with the raw table cells output from table agent if env_config.EXTRACT_TABLE_AS_CELLS is True
+    """
+    from unstructured_inference.models.tables import cells_to_html
+
+    table_id = {v: k for k, v in elements.element_class_id_map.items()}.get(ElementType.TABLE)
+    if table_id is None:
+        # no table found in this page
+        return elements
+
+    table_ele_indices = np.where(elements.element_class_ids == table_id)[0]
+    table_elements = elements.slice(table_ele_indices)
+    padding = env_config.TABLE_IMAGE_CROP_PAD
+    for i, element_coords in enumerate(table_elements.element_coords):
+        cropped_image = image.crop(
+            (
+                element_coords[0] - padding,
+                element_coords[1] - padding,
+                element_coords[2] + padding,
+                element_coords[3] + padding,
+            ),
+        )
+        table_tokens = get_table_tokens(
+            table_element_image=cropped_image,
+            ocr_agent=ocr_agent,
+        )
+        tatr_cells = tables_agent.predict(
+            cropped_image, ocr_tokens=table_tokens, result_format="cells"
+        )
+
+        # NOTE(christine): `tatr_cells == ""` means that the table was not recognized
+        text_as_html = "" if tatr_cells == "" else cells_to_html(tatr_cells)
+        elements.text_as_html[table_ele_indices[i]] = text_as_html
+
+        if env_config.EXTRACT_TABLE_AS_CELLS:
+            simple_table_cells = [
+                SimpleTableCell.from_table_transformer_cell(cell).to_dict() for cell in tatr_cells
+            ]
+            elements.table_as_cells[table_ele_indices[i]] = simple_table_cells
+
+    return elements
+
+
+def get_table_tokens(
+    table_element_image: PILImage.Image,
+    ocr_agent: OCRAgent,
+) -> List[dict[str, Any]]:
+    """Get OCR tokens from either paddleocr or tesseract"""
+
+    ocr_layout = ocr_agent.get_layout_from_image(image=table_element_image)
+    table_tokens = []
+    for i, text in enumerate(ocr_layout.texts):
+        table_tokens.append(
+            {
+                "bbox": [
+                    ocr_layout.x1[i],
+                    ocr_layout.y1[i],
+                    ocr_layout.x2[i],
+                    ocr_layout.y2[i],
+                ],
+                "text": text,
+                # 'table_tokens' is a list of tokens
+                # Need to be in a relative reading order
+                "span_num": i,
+                "line_num": 0,
+                "block_num": 0,
+            }
+        )
+
+    return table_tokens
+
+
+def merge_out_layout_with_ocr_layout(
+    out_layout: LayoutElements,
+    ocr_layout: TextRegions,
+    supplement_with_ocr_elements: bool = True,
+    subregion_threshold: float = env_config.OCR_LAYOUT_SUBREGION_THRESHOLD,
+) -> LayoutElements:
+    """
+    Merge the out layout with the OCR-detected text regions on page level.
+
+    This function iterates over each out layout element and aggregates the associated text from
+    the OCR layout using the specified threshold. The out layout's text attribute is then updated
+    with this aggregated text. If `supplement_with_ocr_elements` is `True`, the out layout will be
+    supplemented with the OCR layout.
+    """
+
+    if len(out_layout) == 0 or len(ocr_layout) == 0:
+        # what if od model finds nothing but ocr finds something? should we use ocr output at all
+        # currently we require some kind of bounding box, from `out_layout` to aggreaget ocr
+        # results. Can we just use ocr bounding boxes (gonna be many but at least we save
+        # information)
+        return out_layout
+
+    invalid_text_indices = [i for i, text in enumerate(out_layout.texts) if not valid_text(text)]
+    out_layout.texts = out_layout.texts.astype(object)
+
+    for idx in invalid_text_indices:
+        out_layout.texts[idx], _ = aggregate_embedded_text_by_block(
+            target_region=out_layout.slice([idx]),
+            source_regions=ocr_layout,
+            subregion_threshold=subregion_threshold,
+        )
+
+    final_layout = (
+        supplement_layout_with_ocr_elements(out_layout, ocr_layout)
+        if supplement_with_ocr_elements
+        else out_layout
+    )
+
+    return final_layout
+
+
+def aggregate_ocr_text_by_block(
+    ocr_layout: List["TextRegion"],
+    region: "TextRegion",
+    subregion_threshold: float = env_config.OCR_LAYOUT_SUBREGION_THRESHOLD,
+) -> Optional[str]:
+    """Extracts the text aggregated from the regions of the ocr layout that lie within the given
+    block."""
+
+    extracted_texts = []
+
+    for ocr_region in ocr_layout:
+        ocr_region_is_subregion_of_given_region = ocr_region.bbox.is_almost_subregion_of(
+            region.bbox,
+            subregion_threshold,
+        )
+        if ocr_region_is_subregion_of_given_region and ocr_region.text:
+            extracted_texts.append(ocr_region.text)
+
+    return " ".join(extracted_texts) if extracted_texts else ""
+
+
+@requires_dependencies("unstructured_inference")
+def supplement_layout_with_ocr_elements(
+    layout: LayoutElements,
+    ocr_layout: TextRegions,
+    subregion_threshold: float = env_config.OCR_LAYOUT_SUBREGION_THRESHOLD,
+) -> LayoutElements:
+    """
+    Supplement the existing layout with additional OCR-derived elements.
+
+    This function takes two lists: one list of pre-existing layout elements (`layout`)
+    and another list of OCR-detected text regions (`ocr_layout`). It identifies OCR regions
+    that are subregions of the elements in the existing layout and removes them from the
+    OCR-derived list. Then, it appends the remaining OCR-derived regions to the existing layout.
+
+    Parameters:
+    - layout (LayoutElements): A collection of existing layout elements in array structures
+    - ocr_layout (TextRegions): A collection of OCR-derived text regions in array structures
+
+    Returns:
+    - List[LayoutElement]: The final combined layout consisting of both the original layout
+                           elements and the new OCR-derived elements.
+
+    Note:
+    - The function relies on `is_almost_subregion_of()` method to determine if an OCR region
+      is a subregion of an existing layout element.
+    - It also relies on `build_layout_elements_from_ocr_regions()` to convert OCR regions to
+     layout elements.
+    - The env_config `OCR_LAYOUT_SUBREGION_THRESHOLD` is used to specify the subregion matching
+     threshold.
+    """
+
+    from unstructured_inference.inference.layoutelement import LayoutElements
+
+    from unstructured.partition.pdf_image.inference_utils import (
+        build_layout_elements_from_ocr_regions,
+    )
+
+    if len(layout) == 0:
+        if len(ocr_layout) == 0:
+            return layout
+        else:
+            ocr_regions_to_add = ocr_layout
+    else:
+        mask = ~bboxes1_is_almost_subregion_of_bboxes2(
+            ocr_layout.element_coords, layout.element_coords, subregion_threshold
+        ).sum(axis=1).astype(bool)
+
+        # add ocr regions that are not covered by layout
+        ocr_regions_to_add = ocr_layout.slice(mask)
+
+    if len(ocr_regions_to_add):
+        ocr_elements_to_add = build_layout_elements_from_ocr_regions(ocr_regions_to_add)
+        final_layout = LayoutElements.concatenate([layout, ocr_elements_to_add])
+    else:
+        final_layout = layout
+
+    return final_layout

unstructured/partition/pdf_image/pdf_image_utils.py

@@ -0,0 +1,439 @@
+from __future__ import annotations
+
+import base64
+import os
+import re
+import tempfile
+import unicodedata
+from copy import deepcopy
+from io import BytesIO
+from pathlib import Path, PurePath
+from typing import IO, TYPE_CHECKING, BinaryIO, Iterator, List, Optional, Tuple, Union, cast
+
+import cv2
+import numpy as np
+import pdf2image
+from PIL import Image
+from unstructured_inference.inference.layout import convert_pdf_to_image as render_pdf_to_image
+from unstructured_inference.inference.pdf_image import PdfRenderTooLargeError
+
+from unstructured.documents.elements import ElementType
+from unstructured.errors import UnprocessableEntityError
+from unstructured.logger import logger
+from unstructured.partition.common.common import convert_to_bytes, exactly_one
+from unstructured.partition.utils.config import env_config
+
+if TYPE_CHECKING:
+    from unstructured_inference.inference.elements import TextRegion
+    from unstructured_inference.inference.layout import DocumentLayout, PageLayout
+    from unstructured_inference.inference.layoutelement import LayoutElement
+
+    from unstructured.documents.elements import Element
+
+
+def write_image(image: Union[Image.Image, np.ndarray], output_image_path: str):
+    """
+    Write an image to a specified file path, supporting both PIL Image and numpy ndarray formats.
+
+    Parameters:
+    - image (Union[Image.Image, np.ndarray]): The image to be written, which can be in PIL Image
+      format or a numpy ndarray format.
+    - output_image_path (str): The path to which the image will be written.
+
+    Raises:
+    - ValueError: If the provided image type is neither PIL Image nor numpy ndarray.
+
+    Returns:
+    - None: The function writes the image to the specified path but does not return any value.
+    """
+
+    if isinstance(image, Image.Image):
+        image.save(output_image_path)
+    elif isinstance(image, np.ndarray):
+        cv2.imwrite(output_image_path, image)
+    else:
+        raise ValueError("Unsupported Image Type")
+
+
+def convert_pdf_to_image(
+    filename: str,
+    file: Optional[Union[bytes, BinaryIO]] = None,
+    dpi: Optional[int] = None,
+    output_folder: Optional[Union[str, PurePath]] = None,
+    path_only: bool = False,
+    password: Optional[str] = None,
+) -> Union[List[Image.Image], List[str]]:
+    exactly_one(filename=filename, file=file)
+
+    if dpi is None:
+        dpi = env_config.PDF_RENDER_DPI
+
+    try:
+        return render_pdf_to_image(
+            filename=filename,
+            file=file,
+            dpi=dpi,
+            output_folder=output_folder,
+            path_only=path_only,
+            password=password,
+            pdf_render_max_pixels_per_page=env_config.PDF_RENDER_MAX_PIXELS_PER_PAGE,
+        )
+    except PdfRenderTooLargeError as exc:
+        raise UnprocessableEntityError(str(exc)) from exc
+
+
+def pad_element_bboxes(
+    element: "LayoutElement",
+    padding: Union[int, float],
+) -> "LayoutElement":
+    """Increases (or decreases, if padding is negative) the size of the bounding
+    boxes of the element by extending the boundary outward (resp. inward)"""
+
+    out_element = deepcopy(element)
+    out_element.bbox.x1 -= padding
+    out_element.bbox.x2 += padding
+    out_element.bbox.y1 -= padding
+    out_element.bbox.y2 += padding
+
+    return out_element
+
+
+def pad_bbox(
+    bbox: Tuple[float, float, float, float],
+    padding: Tuple[Union[int, float], Union[int, float]],
+) -> Tuple[float, float, float, float]:
+    """Pads a bounding box (bbox) by a specified horizontal and vertical padding."""
+
+    x1, y1, x2, y2 = bbox
+    h_padding, v_padding = padding
+    x1 -= h_padding
+    x2 += h_padding
+    y1 -= v_padding
+    y2 += v_padding
+
+    return x1, y1, x2, y2
+
+
+def save_elements(
+    elements: List["Element"],
+    starting_page_number: int,
+    element_category_to_save: str,
+    pdf_image_dpi: int,
+    filename: str = "",
+    file: bytes | IO[bytes] | None = None,
+    is_image: bool = False,
+    extract_image_block_to_payload: bool = False,
+    output_dir_path: str | None = None,
+    password: Optional[str] = None,
+):
+    """
+    Saves specific elements from a PDF as images either to a directory or embeds them in the
+    element's payload.
+
+    This function processes a list of elements partitioned from a PDF file. For each element of
+    a specified category, it extracts and saves the image. The images can either be saved to
+    a specified directory or embedded into the element's payload as a base64-encoded string.
+    """
+
+    # Determine the output directory path
+    if not extract_image_block_to_payload:
+        output_dir_path = output_dir_path or (
+            str(Path(env_config.GLOBAL_WORKING_PROCESS_DIR) / "figures")
+            if env_config.GLOBAL_WORKING_DIR_ENABLED
+            else str(Path.cwd() / "figures")
+        )
+
+        os.makedirs(output_dir_path, exist_ok=True)
+
+    with tempfile.TemporaryDirectory() as temp_dir:
+        if is_image:
+            if file is None:
+                image_paths = [filename]
+            else:
+                if isinstance(file, bytes):
+                    file_data = file
+                else:
+                    file.seek(0)
+                    file_data = file.read()
+
+                tmp_file_path = os.path.join(temp_dir, "tmp_file")
+                with open(tmp_file_path, "wb") as tmp_file:
+                    tmp_file.write(file_data)
+                image_paths = [tmp_file_path]
+        else:
+            _image_paths = convert_pdf_to_image(
+                filename,
+                file,
+                pdf_image_dpi,
+                output_folder=temp_dir,
+                path_only=True,
+                password=password,
+            )
+            image_paths = cast(List[str], _image_paths)
+
+        figure_number = 0
+        for el in elements:
+            if el.category != element_category_to_save:
+                continue
+
+            coordinates = el.metadata.coordinates
+            if not coordinates or not coordinates.points:
+                continue
+
+            points = coordinates.points
+            x1, y1 = points[0]
+            x2, y2 = points[2]
+            h_padding = env_config.EXTRACT_IMAGE_BLOCK_CROP_HORIZONTAL_PAD
+            v_padding = env_config.EXTRACT_IMAGE_BLOCK_CROP_VERTICAL_PAD
+            padded_bbox = cast(
+                Tuple[int, int, int, int], pad_bbox((x1, y1, x2, y2), (h_padding, v_padding))
+            )
+
+            # The page number in the metadata may have been offset
+            # by starting_page_number. Make sure we use the right
+            # value for indexing!
+            assert el.metadata.page_number
+            metadata_page_number = el.metadata.page_number
+            page_index = metadata_page_number - starting_page_number
+
+            figure_number += 1
+            try:
+                image_path = image_paths[page_index]
+                image = Image.open(image_path)
+                cropped_image = image.crop(padded_bbox)
+
+                # PNG images with transparency need to be converted before saving
+                if cropped_image.mode == "RGBA":
+                    cropped_image = cropped_image.convert("RGB")
+
+                if extract_image_block_to_payload:
+                    buffered = BytesIO()
+                    cropped_image.save(buffered, format="JPEG")
+                    img_base64 = base64.b64encode(buffered.getvalue())
+                    img_base64_str = img_base64.decode()
+                    el.metadata.image_base64 = img_base64_str
+                    el.metadata.image_mime_type = "image/jpeg"
+                else:
+                    basename = "table" if el.category == ElementType.TABLE else "figure"
+                    assert output_dir_path
+                    output_f_path = os.path.join(
+                        output_dir_path,
+                        f"{basename}-{metadata_page_number}-{figure_number}.jpg",
+                    )
+                    write_image(cropped_image, output_f_path)
+                    # add image path to element metadata
+                    el.metadata.image_path = output_f_path
+            except (ValueError, IOError):
+                logger.warning("Image Extraction Error: Skipping the failed image", exc_info=True)
+
+
+def check_element_types_to_extract(
+    extract_image_block_types: Optional[List[str]],
+) -> List[str]:
+    """Check and normalize the provided list of element types to extract."""
+
+    if extract_image_block_types is None:
+        return []
+
+    if not isinstance(extract_image_block_types, list):
+        raise TypeError(
+            "The extract_image_block_types parameter must be a list of element types as strings, "
+            "ex. ['Table', 'Image']",
+        )
+
+    available_element_types = {e_type.lower(): e_type for e_type in ElementType.to_dict().values()}
+    normalized_extract_image_block_types = []
+    for el_type in extract_image_block_types:
+        normalized_el_type = available_element_types.get(
+            el_type.lower(), el_type.lower().capitalize()
+        )
+        if normalized_el_type not in available_element_types.values():
+            logger.warning(f"The requested type ({el_type}) doesn't match any available type")
+        normalized_extract_image_block_types.append(normalized_el_type)
+
+    return normalized_extract_image_block_types
+
+
+def valid_text(text: str) -> bool:
+    """a helper that determines if the text is valid ascii text"""
+    if not text:
+        return False
+    return "(cid:" not in text
+
+
+def cid_ratio(text: str) -> float:
+    """Gets ratio of unknown 'cid' characters extracted from text to all characters."""
+    if not is_cid_present(text):
+        return 0.0
+    cid_pattern = r"\(cid\:(\d+)\)"
+    unmatched, n_cid = re.subn(cid_pattern, "", text)
+    total = n_cid + len(unmatched)
+    return n_cid / total
+
+
+def is_cid_present(text: str) -> bool:
+    """Checks if a cid code is present in a text selection."""
+    if len(text) < len("(cid:x)"):
+        return False
+    return text.find("(cid:") != -1
+
+
+def annotate_layout_elements_with_image(
+    inferred_page_layout: "PageLayout",
+    extracted_page_layout: Optional["PageLayout"],
+    output_dir_path: str,
+    output_f_basename: str,
+    page_number: int,
+):
+    """
+     Annotates a page image with both inferred and extracted layout elements.
+
+    This function takes the layout elements of a single page, either extracted from or inferred
+    for the document, and annotates them on the page image. It creates two separate annotated
+    images, one for each set of layout elements: 'inferred' and 'extracted'.
+    These annotated images are saved to a specified directory.
+    """
+
+    layout_map = {"inferred": {"layout": inferred_page_layout, "color": "blue"}}
+    if extracted_page_layout:
+        layout_map["extracted"] = {"layout": extracted_page_layout, "color": "green"}
+
+    for label, layout_data in layout_map.items():
+        page_layout = layout_data.get("layout")
+        color = layout_data.get("color")
+
+        img = page_layout.annotate(colors=color)
+        output_f_path = os.path.join(
+            output_dir_path, f"{output_f_basename}_{page_number}_{label}.jpg"
+        )
+        write_image(img, output_f_path)
+        print(f"output_image_path: {output_f_path}")
+
+
+def annotate_layout_elements(
+    inferred_document_layout: "DocumentLayout",
+    extracted_layout: List["TextRegion"],
+    filename: str,
+    output_dir_path: str,
+    pdf_image_dpi: int,
+    is_image: bool = False,
+) -> None:
+    """
+    Annotates layout elements on images extracted from a PDF or an image file.
+
+    This function processes a given document (PDF or image) and annotates layout elements based
+    on the inferred and extracted layout information.
+    It handles both PDF documents and standalone image files. For PDFs, it converts each page
+    into an image, whereas for image files, it processes the single image.
+    """
+
+    from unstructured_inference.inference.layout import PageLayout
+
+    output_f_basename = os.path.splitext(os.path.basename(filename))[0]
+    images = []
+    try:
+        if is_image:
+            with Image.open(filename) as img:
+                img = img.convert("RGB")
+                images.append(img)
+
+                extracted_page_layout = None
+                if extracted_layout:
+                    extracted_page_layout = PageLayout(
+                        number=1,
+                        image=img,
+                    )
+                    extracted_page_layout.elements = extracted_layout[0]
+
+                inferred_page_layout = inferred_document_layout.pages[0]
+                inferred_page_layout.image = img
+
+                annotate_layout_elements_with_image(
+                    inferred_page_layout=inferred_document_layout.pages[0],
+                    extracted_page_layout=extracted_page_layout,
+                    output_dir_path=output_dir_path,
+                    output_f_basename=output_f_basename,
+                    page_number=1,
+                )
+        else:
+            with tempfile.TemporaryDirectory() as temp_dir:
+                _image_paths = convert_pdf_to_image(
+                    filename,
+                    dpi=pdf_image_dpi,
+                    output_folder=temp_dir,
+                    path_only=True,
+                )
+                image_paths = cast(List[str], _image_paths)
+                for i, image_path in enumerate(image_paths):
+                    with Image.open(image_path) as img:
+                        page_number = i + 1
+
+                        extracted_page_layout = None
+                        if extracted_layout:
+                            extracted_page_layout = PageLayout(
+                                number=page_number,
+                                image=img,
+                            )
+                            extracted_page_layout.elements = extracted_layout[i]
+
+                        inferred_page_layout = inferred_document_layout.pages[i]
+                        inferred_page_layout.image = img
+
+                        annotate_layout_elements_with_image(
+                            inferred_page_layout=inferred_document_layout.pages[i],
+                            extracted_page_layout=extracted_page_layout,
+                            output_dir_path=output_dir_path,
+                            output_f_basename=output_f_basename,
+                            page_number=page_number,
+                        )
+    except Exception as e:
+        if os.path.isdir(filename) or os.path.isfile(filename):
+            raise e
+        else:
+            raise FileNotFoundError(f'File "{filename}" not found!') from e
+
+
+def convert_pdf_to_images(
+    filename: str = "",
+    file: Optional[bytes | IO[bytes]] = None,
+    chunk_size: int = 10,
+    password: Optional[str] = None,
+) -> Iterator[Image.Image]:
+    # Convert a PDF in small chunks of pages at a time (e.g. 1-10, 11-20... and so on)
+    exactly_one(filename=filename, file=file)
+    if file is not None:
+        f_bytes = convert_to_bytes(file)
+        info = pdf2image.pdfinfo_from_bytes(f_bytes, userpw=password)
+    else:
+        f_bytes = None
+        info = pdf2image.pdfinfo_from_path(filename, userpw=password)
+
+    total_pages = info["Pages"]
+    for start_page in range(1, total_pages + 1, chunk_size):
+        end_page = min(start_page + chunk_size - 1, total_pages)
+        try:
+            chunk_images = render_pdf_to_image(
+                filename=filename if f_bytes is None else None,
+                file=f_bytes,
+                dpi=env_config.PDF_RENDER_DPI,
+                first_page=start_page,
+                last_page=end_page,
+                password=password,
+                pdf_render_max_pixels_per_page=env_config.PDF_RENDER_MAX_PIXELS_PER_PAGE,
+            )
+        except PdfRenderTooLargeError as exc:
+            raise UnprocessableEntityError(str(exc)) from exc
+        chunk_images = cast(List[Image.Image], chunk_images)
+
+        for image in chunk_images:
+            yield image
+
+
+def remove_control_characters(text: str) -> str:
+    """Removes control characters from text."""
+
+    # Replace newline character with a space
+    text = text.replace("\t", " ").replace("\n", " ")
+    # Remove other control characters
+    out_text = "".join(c for c in text if unicodedata.category(c)[0] != "C")
+    return out_text

unstructured/partition/pdf_image/pdfminer_utils.py

@@ -0,0 +1,536 @@
+import os
+import re
+import tempfile
+import zlib
+from typing import BinaryIO, List, Mapping, Optional, Tuple, Union
+
+from pdfminer import settings as pdfminer_settings
+from pdfminer.cmapdb import CMap, CMapDB
+from pdfminer.converter import PDFPageAggregator
+from pdfminer.layout import LAParams, LTChar, LTContainer, LTImage, LTItem, LTTextLine
+from pdfminer.pdffont import PDFCIDFont, PDFFontError
+from pdfminer.pdfinterp import LITERAL_FONT, PDFPageInterpreter, PDFResourceManager
+from pdfminer.pdfpage import PDFPage
+from pdfminer.pdftypes import (
+    LITERALS_ASCII85_DECODE,
+    LITERALS_ASCIIHEX_DECODE,
+    LITERALS_FLATE_DECODE,
+    PDFStream,
+    resolve1,
+)
+from pdfminer.psexceptions import PSSyntaxError
+from pdfminer.psparser import literal_name
+from pydantic import BaseModel
+
+from unstructured.logger import logger
+from unstructured.partition.utils.config import env_config
+from unstructured.utils import requires_dependencies
+
+_RE_CIDRANGE_BLOCK = re.compile(rb"begincidrange\s+(.*?)\s+endcidrange", re.DOTALL)
+_RE_CIDRANGE_ENTRY = re.compile(rb"<([0-9A-Fa-f]+)>\s*<([0-9A-Fa-f]+)>\s+(\d+)")
+_RE_CIDCHAR_BLOCK = re.compile(rb"begincidchar\s+(.*?)\s+endcidchar", re.DOTALL)
+_RE_CIDCHAR_ENTRY = re.compile(rb"<([0-9A-Fa-f]+)>\s+(\d+)")
+_RE_WMODE = re.compile(rb"/WMode\s+(\d+)")
+
+# Cap on decompressed CMap stream size to bound regex/parse cost before any mapping
+# cap kicks in. Real-world embedded CMaps are typically a few hundred bytes; even
+# large CJK CMaps are well under 100 KB.
+_MAX_CMAP_STREAM_BYTES = 1_000_000
+
+# Cap on total code-to-CID mappings to prevent malicious PDFs from causing excessive
+# memory/CPU usage via huge begincidrange spans. 131072 covers all real-world fonts
+# (single-byte: max 256, double-byte CJK: typically ~20-30K glyphs).
+_MAX_CODE2CID_MAPPINGS = 131072
+
+
+def _parse_embedded_cmap_stream(data: bytes) -> CMap:
+    """Parse an embedded CMap stream into a CMap with a populated code2cid mapping.
+
+    pdfminer.six does not parse embedded Encoding CMap streams for CIDFonts — it only
+    looks up the CMap name in its predefined database. When a PDF (e.g. produced by
+    Prince XML) embeds a custom CMap as a stream, pdfminer silently falls back to an
+    empty CMap and all text using that font is lost.
+
+    This function parses the begincidrange/begincidchar sections from the raw CMap
+    stream and builds the code2cid dict that CMap.decode() uses. It also extracts
+    WMode (writing mode: 0=horizontal, 1=vertical) when present.
+    """
+    if len(data) > _MAX_CMAP_STREAM_BYTES:
+        logger.warning(
+            "Embedded CMap stream too large (%d bytes, limit %d), skipping",
+            len(data),
+            _MAX_CMAP_STREAM_BYTES,
+        )
+        return CMap()
+
+    code2cid: dict[int, object] = {}
+    total_mappings = 0
+
+    for match in _RE_CIDRANGE_BLOCK.finditer(data):
+        entries = _RE_CIDRANGE_ENTRY.findall(match.group(1))
+        for start_hex, end_hex, cid_str in entries:
+            start_bytes = bytes.fromhex(start_hex.decode("ascii"))
+            end_bytes = bytes.fromhex(end_hex.decode("ascii"))
+            start_cid = int(cid_str)
+            code_len = len(start_bytes)
+            start_val = int.from_bytes(start_bytes, "big")
+            end_val = int.from_bytes(end_bytes, "big")
+
+            if end_val < start_val:
+                continue
+            range_size = end_val - start_val + 1
+            if total_mappings + range_size > _MAX_CODE2CID_MAPPINGS:
+                logger.warning(
+                    "Embedded CMap would exceed %d mappings; discarding partial CMap",
+                    _MAX_CODE2CID_MAPPINGS,
+                )
+                return CMap()
+            for i in range(range_size):
+                code_val = start_val + i
+                cid = start_cid + i
+                if code_len == 1:
+                    code2cid[code_val] = cid
+                else:
+                    code_bytes = code_val.to_bytes(code_len, "big")
+                    d = code2cid
+                    for b in code_bytes[:-1]:
+                        if b not in d:
+                            d[b] = {}
+                        d = d[b]  # type: ignore[assignment]
+                    d[code_bytes[-1]] = cid
+            total_mappings += range_size
+
+    for match in _RE_CIDCHAR_BLOCK.finditer(data):
+        entries = _RE_CIDCHAR_ENTRY.findall(match.group(1))
+        for code_hex, cid_str in entries:
+            if total_mappings >= _MAX_CODE2CID_MAPPINGS:
+                logger.warning(
+                    "Embedded CMap exceeded %d mappings; discarding partial CMap",
+                    _MAX_CODE2CID_MAPPINGS,
+                )
+                return CMap()
+            code_bytes = bytes.fromhex(code_hex.decode("ascii"))
+            cid = int(cid_str)
+            if len(code_bytes) == 1:
+                code2cid[code_bytes[0]] = cid
+            else:
+                d = code2cid
+                for b in code_bytes[:-1]:
+                    if b not in d:
+                        d[b] = {}
+                    d = d[b]  # type: ignore[assignment]
+                d[code_bytes[-1]] = cid
+            total_mappings += 1
+
+    cmap = CMap()
+    cmap.code2cid = code2cid
+
+    wmode_match = _RE_WMODE.search(data)
+    if wmode_match and int(wmode_match.group(1)) != 0:
+        cmap.attrs["WMode"] = int(wmode_match.group(1))
+
+    return cmap
+
+
+def _flate_decode_with_limit(data: bytes, limit: int) -> Optional[bytes]:
+    """Decompress Flate data with a hard output size limit.
+
+    Returns None if the decompressed output would exceed `limit` bytes,
+    without fully materializing the result.
+    """
+    decomp = zlib.decompressobj()
+    out = bytearray()
+    try:
+        out.extend(decomp.decompress(data, limit + 1))
+        if len(out) > limit or decomp.unconsumed_tail:
+            return None
+        out.extend(decomp.flush(limit + 1 - len(out)))
+        if len(out) > limit:
+            return None
+    except zlib.error:
+        return None
+    return bytes(out)
+
+
+def _decode_pdfstream_with_limit(stream: PDFStream, max_decoded_bytes: int) -> Optional[bytes]:
+    """Decode a PDFStream with a hard output size limit, without mutating the stream.
+
+    Unlike PDFStream.get_data(), this never assigns to stream.data or stream.rawdata.
+    Returns None if the decoded output exceeds the limit or uses an unsupported filter.
+    """
+    raw = stream.get_rawdata()
+    if raw is None:
+        # Stream was already decoded elsewhere; check the cached result.
+        data = stream.data
+        if data is None or len(data) > max_decoded_bytes:
+            return None
+        return data
+
+    data = raw
+
+    if stream.decipher:
+        if stream.objid is None or stream.genno is None:
+            logger.debug("Encrypted CMap stream missing objid/genno; skipping")
+            return None
+        data = stream.decipher(stream.objid, stream.genno, data, stream.attrs)
+
+    for filt, params in stream.get_filters():
+        if filt in LITERALS_FLATE_DECODE:
+            result = _flate_decode_with_limit(data, max_decoded_bytes)
+            if result is None:
+                logger.warning(
+                    "Embedded CMap stream exceeded %d bytes during decompression; skipping",
+                    max_decoded_bytes,
+                )
+                return None
+            data = result
+        elif filt in LITERALS_ASCII85_DECODE:
+            from pdfminer.ascii85 import ascii85decode
+
+            data = ascii85decode(data)
+        elif filt in LITERALS_ASCIIHEX_DECODE:
+            from pdfminer.ascii85 import asciihexdecode
+
+            data = asciihexdecode(data)
+        else:
+            logger.debug("Unsupported embedded CMap filter %r; skipping", filt)
+            return None
+
+        if len(data) > max_decoded_bytes:
+            logger.warning(
+                "Embedded CMap stream exceeded %d bytes after filter; skipping",
+                max_decoded_bytes,
+            )
+            return None
+
+    return data
+
+
+class CustomPDFCIDFont(PDFCIDFont):
+    """A CIDFont subclass that handles embedded Encoding CMap streams at construction time.
+
+    pdfminer.six's PDFCIDFont.get_cmap_from_spec only looks up CMap names in its
+    predefined database. When a PDF embeds a custom CMap as a stream, pdfminer falls
+    back to an empty CMap and all text is lost.
+
+    This subclass overrides get_cmap_from_spec to parse the embedded stream when the
+    name lookup fails. Because this runs during __init__, all constructor-time state
+    (vertical mode, widths, displacements) is derived from the correct CMap.
+    """
+
+    def get_cmap_from_spec(self, spec: Mapping, strict: bool):
+        cmap_name = self._get_cmap_name(spec, strict)
+        try:
+            return CMapDB.get_cmap(cmap_name)
+        except CMapDB.CMapNotFound as e:
+            encoding = resolve1(spec.get("Encoding"))
+            if isinstance(encoding, PDFStream):
+                try:
+                    data = _decode_pdfstream_with_limit(encoding, _MAX_CMAP_STREAM_BYTES)
+                    if data is not None:
+                        cmap = _parse_embedded_cmap_stream(data)
+                        if cmap.code2cid:
+                            logger.debug(
+                                "Parsed embedded CMap stream %r (%d code mappings)",
+                                cmap_name,
+                                len(cmap.code2cid),
+                            )
+                            return cmap
+                except (
+                    ValueError,
+                    KeyError,
+                    TypeError,
+                    UnicodeDecodeError,
+                    zlib.error,
+                    PSSyntaxError,
+                ) as exc:
+                    logger.warning(
+                        "Failed to parse embedded CMap stream %r: %s",
+                        cmap_name,
+                        exc,
+                        exc_info=True,
+                    )
+            if strict:
+                raise PDFFontError(e) from e
+            return CMap()
+
+
+class CustomPDFResourceManager(PDFResourceManager):
+    """A resource manager that uses CustomPDFCIDFont for CID font construction.
+
+    This ensures embedded CMap streams are resolved during font construction,
+    not after, so all constructor-time state (WMode, widths) is correct.
+    """
+
+    def get_font(self, objid, spec):
+        subtype = literal_name(spec["Subtype"]) if "Subtype" in spec else "Type1"
+
+        if subtype in ("CIDFontType0", "CIDFontType2"):
+            if objid and objid in self._cached_fonts:
+                return self._cached_fonts[objid]
+            if pdfminer_settings.STRICT and spec.get("Type") is not LITERAL_FONT:
+                raise PDFFontError("Type is not /Font")
+            font = CustomPDFCIDFont(self, spec)
+            if objid and self.caching:
+                self._cached_fonts[objid] = font
+            return font
+
+        return super().get_font(objid, spec)
+
+
+class CustomPDFPageInterpreter(PDFPageInterpreter):
+    """a custom pdfminer page interpreter that adds character render mode information to LTChar
+    object as `rendermode` attribute. This is intended to be used to detect invisible text."""
+
+    def _patch_current_chars_with_render_mode(self, start: int):
+        """Add render_mode to LTChar objects added since index `start`."""
+        cur_item = getattr(self.device, "cur_item", None)
+        if not cur_item:
+            return
+        render_mode = self.textstate.render
+        for obj in getattr(cur_item, "_objs", ())[start:]:
+            if isinstance(obj, LTChar):
+                obj.rendermode = render_mode
+
+    def do_TJ(self, seq):
+        start = len(getattr(getattr(self.device, "cur_item", None), "_objs", ()))
+        super().do_TJ(seq)
+        self._patch_current_chars_with_render_mode(start)
+
+
+class PDFMinerConfig(BaseModel):
+    line_overlap: Optional[float] = None
+    word_margin: Optional[float] = None
+    line_margin: Optional[float] = None
+    char_margin: Optional[float] = None
+    detect_vertical: Optional[bool] = None
+
+
+def init_pdfminer(pdfminer_config: Optional[PDFMinerConfig] = None):
+    rsrcmgr = CustomPDFResourceManager()
+
+    laparams_kwargs = pdfminer_config.model_dump(exclude_none=True) if pdfminer_config else {}
+    laparams = LAParams(**laparams_kwargs)
+
+    device = PDFPageAggregator(rsrcmgr, laparams=laparams)
+    interpreter = CustomPDFPageInterpreter(rsrcmgr, device)
+
+    return device, interpreter
+
+
+def extract_image_objects(parent_object: LTItem) -> List[LTImage]:
+    """Recursively extracts image objects from a given parent object in a PDF document."""
+    objects = []
+
+    if isinstance(parent_object, LTImage):
+        objects.append(parent_object)
+    elif isinstance(parent_object, LTContainer):
+        for child in parent_object:
+            objects.extend(extract_image_objects(child))
+
+    return objects
+
+
+def extract_text_objects(parent_object: LTItem) -> List[LTTextLine]:
+    """Recursively extracts text objects from a given parent object in a PDF document."""
+    objects = []
+
+    if isinstance(parent_object, LTTextLine):
+        objects.append(parent_object)
+    elif isinstance(parent_object, LTContainer):
+        for child in parent_object:
+            objects.extend(extract_text_objects(child))
+
+    return objects
+
+
+def rect_to_bbox(
+    rect: Tuple[float, float, float, float],
+    height: float,
+) -> Tuple[float, float, float, float]:
+    """
+    Converts a PDF rectangle coordinates (x1, y1, x2, y2) to a bounding box in the specified
+    coordinate system where the vertical axis is measured from the top of the page.
+
+    Args:
+        rect (Tuple[float, float, float, float]): A tuple representing a PDF rectangle
+            coordinates (x1, y1, x2, y2).
+        height (float): The height of the page in the specified coordinate system.
+
+    Returns:
+        Tuple[float, float, float, float]: A tuple representing the bounding box coordinates
+        (x1, y1, x2, y2) with the y-coordinates adjusted to be measured from the top of the page.
+    """
+    x1, y2, x2, y1 = rect
+    y1 = height - y1
+    y2 = height - y2
+    return (x1, y1, x2, y2)
+
+
+def _is_duplicate_char(char1: LTChar, char2: LTChar, threshold: float) -> bool:
+    """Detect if two characters are duplicates caused by fake bold rendering.
+
+    Some PDF generators create bold text by rendering the same character twice at slightly
+    offset positions. This function detects such duplicates by checking if two characters
+    have the same text content and overlapping bounding boxes.
+
+    Key insight: Fake-bold duplicates OVERLAP significantly, while legitimate consecutive
+    identical letters (like "ll" in "skills") are ADJACENT with minimal/no overlap.
+
+    Args:
+        char1: First LTChar object.
+        char2: Second LTChar object.
+        threshold: Maximum pixel distance to consider as duplicate.
+
+    Returns:
+        True if char2 appears to be a duplicate of char1.
+    """
+    # Must be the same character
+    if char1.get_text() != char2.get_text():
+        return False
+
+    # Calculate horizontal and vertical distances between character origins
+    x_diff = abs(char1.x0 - char2.x0)
+    y_diff = abs(char1.y0 - char2.y0)
+
+    # Characters must be very close in position
+    if x_diff >= threshold or y_diff >= threshold:
+        return False
+
+    # Additional check: Calculate bounding box overlap to distinguish
+    # fake-bold (high overlap) from legitimate doubles (low/no overlap)
+
+    # Get character widths and heights
+    char1_width = char1.x1 - char1.x0
+    char2_width = char2.x1 - char2.x0
+
+    # Calculate horizontal overlap
+    overlap_x_start = max(char1.x0, char2.x0)
+    overlap_x_end = min(char1.x1, char2.x1)
+    horizontal_overlap = max(0, overlap_x_end - overlap_x_start)
+
+    # Calculate overlap percentage relative to character width
+    avg_width = (char1_width + char2_width) / 2
+    overlap_ratio = horizontal_overlap / avg_width if avg_width > 0 else 0
+
+    # Fake-bold duplicates typically have >70% overlap
+    # Legitimate consecutive letters have <30% overlap (or none)
+    # Use configurable threshold (default 50%) to be conservative
+    overlap_ratio_threshold = env_config.PDF_CHAR_OVERLAP_RATIO_THRESHOLD
+    return overlap_ratio > overlap_ratio_threshold
+
+
+def deduplicate_chars_in_text_line(text_line: LTTextLine, threshold: float) -> str:
+    """Extract text from an LTTextLine with duplicate characters removed.
+
+    Some PDFs create bold text by rendering each character twice at slightly offset
+    positions. This function removes such duplicates by keeping only the first instance
+    when two identical characters appear at nearly the same position.
+
+    Args:
+        text_line: An LTTextLine object containing characters to extract.
+        threshold: Maximum pixel distance to consider characters as duplicates.
+                   Set to 0 to disable deduplication.
+
+    Returns:
+        The extracted text with duplicate characters removed.
+    """
+    if threshold <= 0:
+        return text_line.get_text()
+
+    # Build deduplicated text while preserving non-LTChar items (like LTAnno for spaces)
+    result_parts: List[str] = []
+    last_ltchar: Optional[LTChar] = None
+
+    for item in text_line:
+        if isinstance(item, LTChar):
+            # Check if this is a duplicate of the last LTChar
+            if last_ltchar is not None and _is_duplicate_char(last_ltchar, item, threshold):
+                # Skip this duplicate character
+                continue
+            last_ltchar = item
+            result_parts.append(item.get_text())
+        else:
+            # Non-LTChar items (e.g., LTAnno for spaces) - keep as-is
+            if hasattr(item, "get_text"):
+                result_parts.append(item.get_text())
+
+    return "".join(result_parts)
+
+
+def get_text_with_deduplication(
+    text_obj: Union[LTTextLine, LTContainer, LTItem],
+    threshold: float,
+) -> str:
+    """Get text from a text object with optional character deduplication.
+
+    This is the main entry point for extracting text with fake-bold deduplication.
+    It handles LTTextLine objects and recursively processes containers.
+
+    Args:
+        text_obj: An LTTextLine, LTContainer, or other LTItem object.
+        threshold: Maximum pixel distance to consider characters as duplicates.
+                   Set to 0 to disable deduplication.
+
+    Returns:
+        The extracted text with duplicate characters removed.
+    """
+    if isinstance(text_obj, LTTextLine):
+        return deduplicate_chars_in_text_line(text_obj, threshold)
+    elif isinstance(text_obj, LTContainer):
+        parts: List[str] = []
+        for child in text_obj:
+            if isinstance(child, LTTextLine):
+                parts.append(deduplicate_chars_in_text_line(child, threshold))
+            elif hasattr(child, "get_text"):
+                parts.append(child.get_text())
+        return "".join(parts)
+    elif hasattr(text_obj, "get_text"):
+        return text_obj.get_text()
+    return ""
+
+
+@requires_dependencies(["pikepdf", "pypdf"])
+def open_pdfminer_pages_generator(
+    fp: BinaryIO, password: Optional[str] = None, pdfminer_config: Optional[PDFMinerConfig] = None
+):
+    """Open PDF pages using PDFMiner, handling and repairing invalid dictionary constructs."""
+
+    import pikepdf
+
+    from unstructured.partition.pdf_image.pypdf_utils import get_page_data
+
+    device, interpreter = init_pdfminer(pdfminer_config=pdfminer_config)
+    with tempfile.TemporaryDirectory() as tmp_dir_path:
+        tmp_file_path = os.path.join(tmp_dir_path, "tmp_file")
+        try:
+            pages = PDFPage.get_pages(fp, password=password or "")
+            # Detect invalid dictionary construct for entire PDF
+            for i, page in enumerate(pages):
+                try:
+                    # Detect invalid dictionary construct for one page
+                    interpreter.process_page(page)
+                    page_layout = device.get_result()
+                except PSSyntaxError:
+                    logger.info("Detected invalid dictionary construct for PDFminer")
+                    logger.info(f"Repairing the PDF page {i + 1} ...")
+                    # find the error page from binary data fp
+                    error_page_data = get_page_data(fp, page_number=i)
+                    # repair the error page with pikepdf
+                    with pikepdf.Pdf.open(error_page_data) as pdf:
+                        pdf.save(tmp_file_path)
+                    page = next(PDFPage.get_pages(open(tmp_file_path, "rb")))  # noqa: SIM115
+                    interpreter.process_page(page)
+                    page_layout = device.get_result()
+                yield page, page_layout
+        except PSSyntaxError:
+            logger.info("Detected invalid dictionary construct for PDFminer")
+            logger.info("Repairing the PDF document ...")
+            # repair the entire doc with pikepdf
+            with pikepdf.Pdf.open(fp) as pdf:
+                pdf.save(tmp_file_path)
+            pages = PDFPage.get_pages(open(tmp_file_path, "rb"))  # noqa: SIM115
+            for page in pages:
+                interpreter.process_page(page)
+                page_layout = device.get_result()
+                yield page, page_layout