chore: import upstream snapshot with attribution

test_unstructured/benchmarks/test_benchmark_standardize_quotes.py

@@ -0,0 +1,26 @@
+from unstructured.metrics.text_extraction import standardize_quotes
+
+SAMPLE_TEXTS = [
+    "She said \u201cHello\u201d and then whispered \u2018Goodbye\u2019 before leaving.",
+    "\u201eTo be, or not to be, that is the question\u201d - Shakespeare\u2019s famous quote.",
+    "\u00abWhen he said \u201clife is beautiful,\u201d I believed him\u00bb wrote Maria.",
+    "\u275dDo you remember when we first met?\u275e she asked with a smile.",
+    "\u301dThe meeting starts at 10:00, don\u2019t be late!\u301f announced the manager.",
+    '\u300cHe told me "This is important" yesterday\u300d, she explained.',
+    "\u300eThe sun was setting. The birds were singing. It was peaceful.\u300f",
+    "\ufe42Meeting #123 @ 15:00 - Don\u2019t forget!\ufe41",
+    "\u300cHello\u300d, \u275dWorld\u275e, \"Test\", 'Example', \u201eQuote\u201d, \u00abFinal\u00bb",  # noqa: E501
+    "It\u2019s John\u2019s book, isn\u2019t it?",
+    '\u2039Testing the system\u2019s capability for "quoted" text\u203a',
+    "\u275bFirst sentence. Second sentence. Third sentence.\u275c",
+    "\u300cChapter 1\u300d: \u275dThe Beginning\u275e - \u201eA new story\u201d begins \u00abtoday\u00bb.",  # noqa: E501
+]
+
+
+def run_standardize_quotes():
+    for text in SAMPLE_TEXTS:
+        standardize_quotes(text)
+
+
+def test_benchmark_standardize_quotes(benchmark):
+    benchmark(run_standardize_quotes)

test_unstructured/chunking/test_basic.py

@@ -0,0 +1,308 @@
+"""Test suite for the `unstructured.chunking.basic` module.
+
+That module implements the baseline chunking strategy. The baseline strategy has all behaviors
+shared by all chunking strategies and no extra rules like perserve section or page boundaries.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from test_unstructured.unit_utils import FixtureRequest, Mock, function_mock
+from unstructured.chunking.basic import chunk_elements
+from unstructured.documents.elements import (
+    CompositeElement,
+    ElementMetadata,
+    Table,
+    TableChunk,
+    Text,
+    Title,
+)
+from unstructured.partition.docx import partition_docx
+
+
+def test_it_chunks_a_document_when_basic_chunking_strategy_is_specified_on_partition_function():
+    """Basic chunking can be combined with partitioning, exercising the decorator."""
+    filename = "example-docs/handbook-1p.docx"
+
+    chunks = partition_docx(filename, chunking_strategy="basic")
+
+    assert chunks == [
+        CompositeElement(
+            "US Trustee Handbook\n\nCHAPTER 1\n\nINTRODUCTION\n\nCHAPTER 1 – INTRODUCTION"
+            "\n\nA. PURPOSE"
+        ),
+        CompositeElement(
+            "The United States Trustee appoints and supervises standing trustees and monitors and"
+            " supervises cases under chapter 13 of title 11 of the United States Code. 28 U.S.C."
+            " § 586(b). The Handbook, issued as part of our duties under 28 U.S.C. § 586,"
+            " establishes or clarifies the position of the United States Trustee Program (Program)"
+            " on the duties owed by a standing trustee to the debtors, creditors, other parties in"
+            " interest, and the United States Trustee. The Handbook does not present a full and"
+        ),
+        CompositeElement(
+            "complete statement of the law; it should not be used as a substitute for legal"
+            " research and analysis. The standing trustee must be familiar with relevant"
+            " provisions of the Bankruptcy Code, Federal Rules of Bankruptcy Procedure (Rules),"
+            " any local bankruptcy rules, and case law. 11 U.S.C. § 321, 28 U.S.C. § 586,"
+            " 28 C.F.R. § 58.6(a)(3). Standing trustees are encouraged to follow Practice Tips"
+            " identified in this Handbook but these are not considered mandatory."
+        ),
+        CompositeElement(
+            "Nothing in this Handbook should be construed to excuse the standing trustee from"
+            " complying with all duties imposed by the Bankruptcy Code and Rules, local rules, and"
+            " orders of the court. The standing trustee should notify the United States Trustee"
+            " whenever the provision of the Handbook conflicts with the local rules or orders of"
+            " the court. The standing trustee is accountable for all duties set forth in this"
+            " Handbook, but need not personally perform any duty unless otherwise indicated. All"
+        ),
+        CompositeElement(
+            "statutory references in this Handbook refer to the Bankruptcy Code, 11 U.S.C. § 101"
+            " et seq., unless otherwise indicated."
+        ),
+        CompositeElement(
+            "This Handbook does not create additional rights against the standing trustee or"
+            " United States Trustee in favor of other parties.\n\nB. ROLE OF THE UNITED STATES"
+            " TRUSTEE"
+        ),
+        CompositeElement(
+            "The Bankruptcy Reform Act of 1978 removed the bankruptcy judge from the"
+            " responsibilities for daytoday administration of cases. Debtors, creditors, and"
+            " third parties with adverse interests to the trustee were concerned that the court,"
+            " which previously appointed and supervised the trustee, would not impartially"
+            " adjudicate their rights as adversaries of that trustee. To address these concerns,"
+            " judicial and administrative functions within the bankruptcy system were bifurcated."
+        ),
+        CompositeElement(
+            "Many administrative functions formerly performed by the court were placed within the"
+            " Department of Justice through the creation of the Program. Among the administrative"
+            " functions assigned to the United States Trustee were the appointment and supervision"
+            " of chapter 13 trustees./ This Handbook is issued under the authority of the"
+            " Program’s enabling statutes.\n\nC. STATUTORY DUTIES OF A STANDING TRUSTEE"
+        ),
+        CompositeElement(
+            "The standing trustee has a fiduciary responsibility to the bankruptcy estate. The"
+            " standing trustee is more than a mere disbursing agent. The standing trustee must"
+            " be personally involved in the trustee operation. If the standing trustee is or"
+            " becomes unable to perform the duties and responsibilities of a standing trustee,"
+            " the standing trustee must immediately advise the United States Trustee."
+            " 28 U.S.C. § 586(b), 28 C.F.R. § 58.4(b) referencing 28 C.F.R. § 58.3(b)."
+        ),
+        CompositeElement(
+            "Although this Handbook is not intended to be a complete statutory reference, the"
+            " standing trustee’s primary statutory duties are set forth in 11 U.S.C. § 1302, which"
+            " incorporates by reference some of the duties of chapter 7 trustees found in"
+            " 11 U.S.C. § 704. These duties include, but are not limited to, the"
+            " following:\n\nCopyright"
+        ),
+    ]
+
+
+def test_it_chunks_elements_when_the_user_already_has_them():
+    elements = [
+        Title("Introduction"),
+        Text(
+            # --------------------------------------------------------- 64 -v
+            "Lorem ipsum dolor sit amet consectetur adipiscing elit. In rhoncus ipsum sed lectus"
+            " porta volutpat.",
+        ),
+    ]
+
+    chunks = chunk_elements(elements, max_characters=64)
+
+    assert chunks == [
+        CompositeElement("Introduction"),
+        # -- splits on even word boundary, not mid-"rhoncus" --
+        CompositeElement("Lorem ipsum dolor sit amet consectetur adipiscing elit. In"),
+        CompositeElement("rhoncus ipsum sed lectus porta volutpat."),
+    ]
+
+
+def test_it_includes_original_elements_as_metadata_when_requested():
+    element = Title("Introduction")
+    element_2 = Text("Lorem ipsum dolor sit amet consectetur adipiscing elit.")
+    element_3 = Text("In rhoncus ipsum sed lectus porta volutpat.")
+
+    chunks = chunk_elements(
+        [element, element_2, element_3], max_characters=70, include_orig_elements=True
+    )
+
+    assert len(chunks) == 2
+    chunk = chunks[0]
+    assert chunk == CompositeElement(
+        "Introduction\n\nLorem ipsum dolor sit amet consectetur adipiscing elit."
+    )
+    assert chunk.metadata.orig_elements == [element, element_2]
+    # --
+    chunk = chunks[1]
+    assert chunk == CompositeElement("In rhoncus ipsum sed lectus porta volutpat.")
+    assert chunk.metadata.orig_elements == [element_3]
+
+
+def test_it_repeats_table_headers_by_default_but_can_opt_out():
+    table_html = (
+        "<table>"
+        "<thead>"
+        "<tr><th>Header A</th><th>Header B</th></tr>"
+        "<tr><th>Subhead A</th><th>Subhead B</th></tr>"
+        "</thead>"
+        "<tbody>"
+        "<tr><td>Body 1</td><td>Alpha</td></tr>"
+        "<tr><td>Body 2</td><td>Bravo</td></tr>"
+        "<tr><td>Body 3</td><td>Charlie</td></tr>"
+        "<tr><td>Body 4</td><td>Delta</td></tr>"
+        "</tbody>"
+        "</table>"
+    )
+    table_text = (
+        "Header A Header B\n"
+        "Subhead A Subhead B\n"
+        "Body 1 Alpha\n"
+        "Body 2 Bravo\n"
+        "Body 3 Charlie\n"
+        "Body 4 Delta"
+    )
+    table = Table(table_text, metadata=ElementMetadata(text_as_html=table_html))
+
+    repeated_header_chunks = chunk_elements([table], max_characters=55)
+    opt_out_chunks = chunk_elements([table], max_characters=55, repeat_table_headers=False)
+
+    assert len(repeated_header_chunks) == 4
+    assert all(isinstance(chunk, TableChunk) for chunk in repeated_header_chunks)
+    assert [chunk.text for chunk in repeated_header_chunks] == [
+        "Header A Header B Subhead A Subhead B Body 1 Alpha",
+        "Header A Header B Subhead A Subhead B Body 2 Bravo",
+        "Header A Header B Subhead A Subhead B Body 3 Charlie",
+        "Header A Header B Subhead A Subhead B Body 4 Delta",
+    ]
+    assert [chunk.text for chunk in opt_out_chunks] == [
+        "Header A Header B Subhead A Subhead B Body 1 Alpha",
+        "Body 2 Bravo Body 3 Charlie Body 4 Delta",
+    ]
+
+
+def test_skip_table_chunking_passes_oversized_table_through_unchanged():
+    table_text = "cell " * 200  # 1000 chars, well above max_characters=100
+    table = Table(table_text.strip())
+    text_before = Text("Hello world")
+    text_after = Text("Goodbye world")
+
+    chunks = chunk_elements(
+        [text_before, table, text_after],
+        max_characters=100,
+        skip_table_chunking=True,
+    )
+
+    assert len(chunks) == 3
+    assert isinstance(chunks[0], CompositeElement)
+    assert isinstance(chunks[1], Table)
+    assert isinstance(chunks[2], CompositeElement)
+    # -- table text is unchanged --
+    assert chunks[1].text == table_text.strip()
+
+
+def test_skip_table_chunking_does_not_affect_text_element_chunking():
+    long_text = Text("word " * 200)
+    table = Table("small table")
+
+    chunks = chunk_elements(
+        [long_text, table],
+        max_characters=100,
+        skip_table_chunking=True,
+    )
+
+    # -- long text element is still split, table is still isolated --
+    text_chunks = [c for c in chunks if isinstance(c, CompositeElement)]
+    table_chunks = [c for c in chunks if isinstance(c, Table)]
+    assert len(text_chunks) > 1
+    assert len(table_chunks) == 1
+    assert table_chunks[0].text == "small table"
+
+
+# ------------------------------------------------------------------------------------------------
+# UNIT TESTS
+# ------------------------------------------------------------------------------------------------
+
+
+class Describe_chunk_elements:
+    """Unit-test suite for `unstructured.chunking.basic.chunk_elements()` function."""
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"include_orig_elements": True}, True),
+            ({"include_orig_elements": False}, False),
+            ({"include_orig_elements": None}, True),
+            ({}, True),
+        ],
+    )
+    def it_supports_the_include_orig_elements_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_elements_: Mock
+    ):
+        # -- this line would raise if "include_orig_elements" was not an available parameter on
+        # -- `chunk_elements()`.
+        chunk_elements([], **kwargs)
+
+        _, opts = _chunk_elements_.call_args.args
+        assert opts.include_orig_elements is expected_value
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"repeat_table_headers": True}, True),
+            ({"repeat_table_headers": False}, False),
+            ({"repeat_table_headers": None}, True),
+            ({}, True),
+        ],
+    )
+    def it_supports_the_repeat_table_headers_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_elements_: Mock
+    ):
+        # -- this line would raise if "repeat_table_headers" was not an available parameter on
+        # -- `chunk_elements()`.
+        chunk_elements([], **kwargs)
+
+        _, opts = _chunk_elements_.call_args.args
+        assert opts.repeat_table_headers is expected_value
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"skip_table_chunking": True}, True),
+            ({"skip_table_chunking": False}, False),
+            ({"skip_table_chunking": None}, False),
+            ({}, False),
+        ],
+    )
+    def it_supports_the_skip_table_chunking_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_elements_: Mock
+    ):
+        chunk_elements([], **kwargs)
+
+        _, opts = _chunk_elements_.call_args.args
+        assert opts.skip_table_chunking is expected_value
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"isolate_table": True}, True),
+            ({"isolate_table": False}, False),
+            ({"isolate_table": None}, True),
+            ({}, True),
+        ],
+    )
+    def it_supports_the_isolate_table_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_elements_: Mock
+    ):
+        chunk_elements([], **kwargs)
+
+        _, opts = _chunk_elements_.call_args.args
+        assert opts.isolate_table is expected_value
+
+    # -- fixtures --------------------------------------------------------------------------------
+
+    @pytest.fixture()
+    def _chunk_elements_(self, request: FixtureRequest):
+        return function_mock(request, "unstructured.chunking.basic._chunk_elements")

test_unstructured/chunking/test_dispatch.py

@@ -0,0 +1,92 @@
+# pyright: reportPrivateUsage=false
+
+"""Unit-test suite for the `unstructured.chunking.dispatch` module."""
+
+from __future__ import annotations
+
+from typing import Any, Iterable, Optional
+
+import pytest
+
+from unstructured.chunking import add_chunking_strategy, register_chunking_strategy
+from unstructured.chunking.dispatch import _ChunkerSpec, chunk
+from unstructured.documents.elements import CompositeElement, Element, Text
+
+
+class Describe_add_chunking_strategy:
+    """Unit-test suite for `unstructured.chunking.add_chunking_strategy()` decorator."""
+
+    def it_dispatches_the_partitioned_elements_to_the_indicated_chunker(self):
+        decorated_partitioner = add_chunking_strategy(partition_this)
+
+        chunks = decorated_partitioner(chunking_strategy="basic")
+
+        assert chunks == [CompositeElement("Lorem ipsum.\n\nSit amet.")]
+
+    def but_it_skips_dispatch_when_no_chunking_strategy_is_specified(self):
+        decorated_partitioner = add_chunking_strategy(partition_this)
+
+        elements = decorated_partitioner()
+
+        assert elements == [Text("Lorem ipsum."), Text("Sit amet.")]
+
+
+class Describe_chunk:
+    """Unit-test suite for `unstructured.chunking.dispatch.chunk()` function."""
+
+    def it_dispatches_to_the_chunker_registered_for_the_chunking_strategy(self):
+        register_chunking_strategy("by_something_else", chunk_by_something_else)
+        kwargs = {
+            "max_characters": 750,
+            # -- unused kwargs shouldn't cause a problem; in general `kwargs` will contain all
+            # -- keyword arguments used in the partitioning call.
+            "foo": "bar",
+        }
+
+        chunks = chunk([Text("Lorem"), Text("Ipsum")], "by_something_else", **kwargs)
+
+        assert chunks == [
+            CompositeElement("chunked 2 elements with `(max_characters=750, whizbang=None)`")
+        ]
+
+    def it_raises_when_the_requested_chunking_strategy_is_not_registered(self):
+        with pytest.raises(
+            ValueError,
+            match="unrecognized chunking strategy 'foobar'",
+        ):
+            chunk(elements=[], chunking_strategy="foobar")
+
+
+class Describe_ChunkerSpec:
+    """Unit-test suite for `unstructured.chunking.dispatch._ChunkerSpec` objects."""
+
+    def it_provides_access_to_the_chunking_function(self):
+        spec = _ChunkerSpec(chunk_by_something_else)
+        assert spec.chunker is chunk_by_something_else
+
+    def it_knows_which_keyword_args_the_chunking_function_can_accept(self):
+        spec = _ChunkerSpec(chunk_by_something_else)
+        assert spec.kw_arg_names == ("max_characters", "whizbang")
+
+
+# -- MODULE-LEVEL FIXTURES -----------------------------------------------------------------------
+
+
+def chunk_by_something_else(
+    elements: Iterable[Element],
+    max_characters: Optional[int] = None,
+    whizbang: Optional[float] = None,
+) -> list[Element]:
+    """A "fake" minimal chunker suitable for use in tests."""
+    els = list(elements)
+    return [
+        CompositeElement(
+            f"chunked {len(els)} elements with"
+            f" `(max_characters={max_characters}, whizbang={whizbang})`"
+        )
+    ]
+
+
+def partition_this(**kwargs: Any) -> list[Element]:
+    """A fake partitioner."""
+    return [Text("Lorem ipsum."), Text("Sit amet.")]

test_unstructured/chunking/test_html_output.py

@@ -0,0 +1,90 @@
+from functools import partial
+
+import pytest
+
+from unstructured.chunking.basic import chunk_elements
+from unstructured.chunking.title import chunk_by_title
+from unstructured.documents.elements import ElementMetadata, NarrativeText, Text, Title
+
+
+@pytest.fixture(params=[chunk_elements, partial(chunk_by_title, combine_text_under_n_chars=0)])
+def chunking_fn(request):
+    return request.param
+
+
+def test_combining_html_metadata_when_multiple_elements_in_composite_element(chunking_fn):
+    metadata_1 = '<h1 class="Title" id="1">Header </h1>'
+    metadata_2 = '<time class="CalendarDate" id="2">Date: October 30, 2023 </time>'
+    metadata_3 = (
+        '<form class="Form" id="3"> '
+        '<label class="FormField" for="company-name" id="4">Form field name </label>'
+        '<input class="FormFieldValue" id="5" value="Example value" />'
+        "</form>"
+    )
+    combined_metadata = " ".join([metadata_1, metadata_2, metadata_3])
+
+    elements = [
+        Title(text="Header", metadata=ElementMetadata(text_as_html=metadata_1)),
+        Text(text="Date: October 30, 2023", metadata=ElementMetadata(text_as_html=metadata_2)),
+        Text(
+            text="Form field name Example value", metadata=ElementMetadata(text_as_html=metadata_3)
+        ),
+    ]
+    chunks = chunking_fn(elements)
+    assert len(chunks) == 1
+    assert chunks[0].metadata.text_as_html == combined_metadata
+
+
+def test_combining_html_metadata_with_nested_relationship_between_elements(chunking_fn):
+    """
+    Ground truth
+    <Document>
+        <Page>
+            <Section>
+                <p>First</p>
+                <p>Second</p>
+            </Section>
+        </Page>
+    </Document>
+    Elements: Document, Page, Section, Paragraph, Paragraph
+    Chunk 1: Document, Page, Section, Paragraph
+
+    Chunk 2:
+        Paragraph
+    """
+
+    metadata_1 = '<div class="Section" id="1" />'
+    metadata_2 = '<p class="Paragraph" id="2">First </p>'
+    metadata_3 = '<p class="Paragraph" id="3">Second </p>'
+
+    elements = [
+        Text(text="", metadata=ElementMetadata(text_as_html=metadata_1)),
+        NarrativeText(
+            text="First", metadata=ElementMetadata(text_as_html=metadata_2, parent_id="1")
+        ),
+        NarrativeText(
+            text="Second", metadata=ElementMetadata(text_as_html=metadata_3, parent_id="1")
+        ),
+    ]
+    chunks = chunking_fn(elements, max_characters=6)
+    assert len(chunks) == 2
+    assert chunks[0].text == "First"
+    assert chunks[1].text == "Second"
+
+    assert chunks[0].metadata.text_as_html == metadata_1 + " " + metadata_2
+    assert chunks[1].metadata.text_as_html == metadata_3
+
+
+def test_html_metadata_exist_in_both_element_when_text_is_split(chunking_fn):
+    """Mimic behaviour of elements with non-html metadata"""
+    metadata_1 = '<h1 class="Title" id="1">Header </h1>'
+    elements = [
+        Title(text="Header", metadata=ElementMetadata(text_as_html=metadata_1)),
+    ]
+    chunks = chunking_fn(elements, max_characters=3)
+    assert len(chunks) == 2
+
+    assert chunks[0].text == "Hea"
+    assert chunks[1].text == "der"
+    assert chunks[0].metadata.text_as_html == '<h1 class="Title" id="1">Header </h1>'
+    assert chunks[1].metadata.text_as_html == '<h1 class="Title" id="1">Header </h1>'

test_unstructured/chunking/test_table_isolation.py

@@ -0,0 +1,357 @@
+# pyright: reportPrivateUsage=false
+
+"""Regression and characterization tests for table isolation during chunking (issue #3921).
+
+`Table` (and `TableChunk`, which subclasses `Table`) must never share a pre-chunk with unrelated
+text elements, so downstream logic can emit standalone table chunks instead of `CompositeElement`
+wrapping mixed content.
+"""
+
+from __future__ import annotations
+
+from typing import Callable
+
+import pytest
+
+from unstructured.chunking.base import (
+    ChunkingOptions,
+    PreChunk,
+    PreChunkBuilder,
+    PreChunkCombiner,
+    PreChunker,
+)
+from unstructured.chunking.basic import chunk_elements
+from unstructured.chunking.title import chunk_by_title
+from unstructured.documents.elements import (
+    CompositeElement,
+    ElementMetadata,
+    Table,
+    TableChunk,
+    Text,
+    Title,
+)
+
+
+class DescribeTableIsolationPreChunkBuilder:
+    """`PreChunkBuilder` must keep every table-family element in its own pre-chunk."""
+
+    @pytest.mark.parametrize(
+        ("preamble", "make_table"),
+        [
+            (Text("Short preamble."), lambda: Table("H\nC")),
+            (Text("Short preamble."), lambda: TableChunk(text="x", metadata=ElementMetadata())),
+        ],
+    )
+    def it_refuses_to_append_a_table_after_any_other_element(
+        self, preamble: Text, make_table: Callable[[], Table]
+    ):
+        builder = PreChunkBuilder(opts=ChunkingOptions(max_characters=500))
+        builder.add_element(preamble)
+
+        assert not builder.will_fit(make_table())
+
+    @pytest.mark.parametrize(
+        "trailing",
+        [
+            Text("Follow-up paragraph."),
+            Title("Next section"),
+        ],
+    )
+    def it_refuses_to_append_a_non_table_after_a_table(self, trailing: Text | Title):
+        builder = PreChunkBuilder(opts=ChunkingOptions(max_characters=500))
+        builder.add_element(Table("Heading\nCell"))
+
+        assert not builder.will_fit(trailing)
+
+    def it_allows_only_a_table_when_the_builder_is_empty(self):
+        builder = PreChunkBuilder(opts=ChunkingOptions())
+        assert builder.will_fit(Table("Heading\nCell text"))
+
+    def it_allows_text_after_flush_even_if_previous_pre_chunk_was_a_table(self):
+        opts = ChunkingOptions(max_characters=200)
+        builder = PreChunkBuilder(opts=opts)
+        builder.add_element(Table("Heading\nCell text"))
+        list(builder.flush())  # clears builder state
+
+        assert builder.will_fit(Text("Fresh start after table pre-chunk."))
+
+
+class DescribeTableIsolationPreChunkStream:
+    """End-to-end pre-chunk segmentation from `PreChunker.iter_pre_chunks`."""
+
+    def it_emits_a_table_only_pre_chunk_between_text_blocks(self):
+        elements = [
+            Title("Section A"),
+            Text("Narrative before the table."),
+            Table("Col1\nCell A"),
+            Text("Narrative after the table."),
+        ]
+        # -- `new_after_n_chars=0` forces one element per pre-chunk (deterministic layout) --
+        opts = ChunkingOptions(max_characters=500, new_after_n_chars=0)
+
+        pre_chunks = list(PreChunker.iter_pre_chunks(elements, opts=opts))
+
+        assert len(pre_chunks) == 4
+        assert pre_chunks[0]._elements == [Title("Section A")]
+        assert pre_chunks[1]._elements == [Text("Narrative before the table.")]
+        assert pre_chunks[2]._elements == [Table("Col1\nCell A")]
+        assert pre_chunks[3]._elements == [Text("Narrative after the table.")]
+
+    def it_emits_one_pre_chunk_per_table_when_multiple_tables_are_adjacent(self):
+        elements = [
+            Table("T1\nA"),
+            Table("T2\nB"),
+            Text("Closing text."),
+        ]
+        opts = ChunkingOptions(max_characters=500)
+
+        pre_chunks = list(PreChunker.iter_pre_chunks(elements, opts=opts))
+
+        assert len(pre_chunks) == 3
+        assert pre_chunks[0]._elements == [Table("T1\nA")]
+        assert pre_chunks[1]._elements == [Table("T2\nB")]
+        assert pre_chunks[2]._elements == [Text("Closing text.")]
+
+
+class DescribeTableIsolationPreChunkCombiner:
+    """`PreChunkCombiner` must not stitch table pre-chunks onto text neighbors."""
+
+    def it_keeps_a_table_pre_chunk_separate_when_combining_is_enabled(self):
+        opts = ChunkingOptions(max_characters=500, combine_text_under_n_chars=500)
+        stream = [
+            PreChunk([Text("Hello world.")], overlap_prefix="", opts=opts),
+            PreChunk([Table("H\nC")], overlap_prefix="", opts=opts),
+            PreChunk([Text("Goodbye world.")], overlap_prefix="", opts=opts),
+        ]
+
+        combined = list(PreChunkCombiner(stream, opts=opts).iter_combined_pre_chunks())
+
+        assert len(combined) == 3
+        assert combined[0]._elements == [Text("Hello world.")]
+        assert combined[1]._elements == [Table("H\nC")]
+        assert combined[2]._elements == [Text("Goodbye world.")]
+
+
+class DescribeTableIsolationOrderingGuarantees:
+    """Invariants that should hold for any future refactor of table isolation."""
+
+    def it_preserves_global_element_order_in_pre_chunks(self):
+        elements = [
+            Text("alpha"),
+            Table("T\n1"),
+            Text("beta"),
+            Table("T\n2"),
+            Title("gamma"),
+        ]
+        opts = ChunkingOptions(max_characters=500, new_after_n_chars=0)
+        flat = [e for pc in PreChunker.iter_pre_chunks(elements, opts=opts) for e in pc._elements]
+
+        assert flat == elements
+
+    def it_preserves_global_element_order_in_chunk_elements_output(self):
+        elements = [
+            Title("Intro"),
+            Text("Body before."),
+            Table("K\nV"),
+            Text("Body after."),
+        ]
+        chunks = chunk_elements(elements, max_characters=500, new_after_n_chars=0)
+        # -- flatten chunk element categories in stream order (table chunks are atomic) --
+        categories = [c.category for c in chunks]
+
+        assert categories == [
+            "CompositeElement",
+            "CompositeElement",
+            "Table",
+            "CompositeElement",
+        ]
+
+
+class DescribeTableIsolationChunkElements:
+    """`chunk_elements` (basic strategy) should emit `Table`/`TableChunk`, not mixed composites."""
+
+    def it_does_not_wrap_a_table_and_surrounding_text_in_one_composite_element(self):
+        elements = [
+            Title("Report"),
+            Text("Short intro."),
+            Table("Key\nValue"),
+            Text("Short outro."),
+        ]
+        chunks = chunk_elements(
+            elements,
+            max_characters=500,
+            new_after_n_chars=0,
+        )
+
+        assert len(chunks) == 4
+        assert isinstance(chunks[0], CompositeElement)
+        assert isinstance(chunks[1], CompositeElement)
+        assert isinstance(chunks[2], Table)
+        assert isinstance(chunks[3], CompositeElement)
+        assert "Key" in chunks[2].text or "Value" in chunks[2].text
+
+    def it_yields_distinct_chunks_for_two_tables_in_a_row(self):
+        elements = [
+            Table("T1\nA"),
+            Table("T2\nB"),
+        ]
+        chunks = chunk_elements(elements, max_characters=500)
+
+        assert len(chunks) == 2
+        assert all(isinstance(c, Table) for c in chunks)
+
+    def it_still_isolates_a_table_even_when_the_window_is_very_large(self):
+        """Regression: isolation is a semantic rule, not a size heuristic."""
+        elements = [
+            Text("x"),
+            Table("tiny"),
+            Text("y"),
+        ]
+        chunks = chunk_elements(elements, max_characters=50_000, new_after_n_chars=10_000)
+
+        table_chunks = [c for c in chunks if isinstance(c, Table)]
+        composite_chunks = [c for c in chunks if isinstance(c, CompositeElement)]
+
+        assert len(table_chunks) == 1
+        assert len(composite_chunks) == 2
+        assert "tiny" in table_chunks[0].text
+
+    def it_never_produces_a_composite_element_that_lists_a_table_in_orig_elements(
+        self,
+    ):
+        """`CompositeElement` chunks come from `_Chunker`, not `_TableChunker`."""
+        elements = [
+            Text("preamble"),
+            Table("H\nC"),
+            Text("post"),
+        ]
+        chunks = chunk_elements(
+            elements,
+            max_characters=400,
+            new_after_n_chars=0,
+            include_orig_elements=True,
+        )
+
+        composites = [c for c in chunks if isinstance(c, CompositeElement)]
+        for comp in composites:
+            orig = comp.metadata.orig_elements or []
+            assert not any(isinstance(e, Table) for e in orig)
+
+
+class DescribeTableIsolationDisabled:
+    """When `isolate_table=False`, the pre-#4307 behavior is restored."""
+
+    def it_lets_a_table_share_a_pre_chunk_with_adjacent_text_when_disabled(self):
+        opts = ChunkingOptions(max_characters=500, isolate_table=False)
+        builder = PreChunkBuilder(opts=opts)
+        builder.add_element(Text("Short preamble."))
+
+        assert builder.will_fit(Table("Heading\nCell text"))
+
+    def it_lets_text_follow_a_table_in_the_same_pre_chunk_when_disabled(self):
+        opts = ChunkingOptions(max_characters=500, isolate_table=False)
+        builder = PreChunkBuilder(opts=opts)
+        builder.add_element(Table("Heading\nCell text"))
+
+        assert builder.will_fit(Text("Follow-up paragraph."))
+
+    def it_combines_a_table_pre_chunk_with_text_neighbors_when_disabled(self):
+        opts = ChunkingOptions(
+            max_characters=500, combine_text_under_n_chars=500, isolate_table=False
+        )
+        stream = [
+            PreChunk([Text("Hello world.")], overlap_prefix="", opts=opts),
+            PreChunk([Table("H\nC")], overlap_prefix="", opts=opts),
+            PreChunk([Text("Goodbye world.")], overlap_prefix="", opts=opts),
+        ]
+
+        combined = list(PreChunkCombiner(stream, opts=opts).iter_combined_pre_chunks())
+
+        assert len(combined) == 1
+        assert combined[0]._elements == [
+            Text("Hello world."),
+            Table("H\nC"),
+            Text("Goodbye world."),
+        ]
+
+    def it_wraps_a_table_into_a_composite_element_with_neighbors_when_disabled(self):
+        """End-to-end: opting out collapses tiny table + text into one CompositeElement."""
+        elements = [
+            Text("preamble"),
+            Table("H\nC"),
+            Text("post"),
+        ]
+        chunks = chunk_elements(
+            elements,
+            max_characters=500,
+            isolate_table=False,
+        )
+
+        assert len(chunks) == 1
+        assert isinstance(chunks[0], CompositeElement)
+        text = chunks[0].text
+        assert "preamble" in text
+        assert "post" in text
+
+
+class DescribeTableIsolationOverlapAll:
+    """With overlap_all=True, overlap must not cross table / narrative boundaries."""
+
+    def it_does_not_prefix_table_chunk_with_prior_text_overlap(self):
+        """Regression: pre-chunk overlap_tail must not become table's overlap_prefix."""
+        elements = [Text("Alpha beta gamma delta."), Table("H\nC")]
+        chunks = chunk_elements(
+            elements,
+            max_characters=500,
+            new_after_n_chars=0,
+            overlap=5,
+            overlap_all=True,
+        )
+
+        table_chunks = [c for c in chunks if isinstance(c, Table)]
+        assert len(table_chunks) == 1
+        t = table_chunks[0].text or ""
+        assert "Alpha" not in t
+        assert "elta" not in t  # tail of "delta" leaked in buggy overlap
+
+    def it_does_not_prefix_text_after_table_with_table_overlap(self):
+        elements = [Table("H\nC"), Text("Omega sigma tau upsilon.")]
+        chunks = chunk_elements(
+            elements,
+            max_characters=500,
+            new_after_n_chars=0,
+            overlap=5,
+            overlap_all=True,
+        )
+
+        composites = [c for c in chunks if isinstance(c, CompositeElement)]
+        assert len(composites) == 1
+        assert composites[0].text.startswith("Omega")
+        assert "H" not in composites[0].text[:20]
+
+    def it_chunk_by_title_respects_same_overlap_boundaries(self):
+        elements = [
+            Title("Section"),
+            Text("Alpha beta gamma delta."),
+            Table("H\nC"),
+            Text("Omega sigma tau upsilon."),
+        ]
+        chunks = chunk_by_title(
+            elements,
+            max_characters=500,
+            new_after_n_chars=0,
+            overlap=5,
+            overlap_all=True,
+            combine_text_under_n_chars=0,
+        )
+
+        table_chunks = [c for c in chunks if isinstance(c, Table)]
+        assert len(table_chunks) == 1
+        assert "Alpha" not in (table_chunks[0].text or "")
+        assert "elta" not in (table_chunks[0].text or "")
+
+        omega_composites = [
+            c for c in chunks if isinstance(c, CompositeElement) and "Omega" in (c.text or "")
+        ]
+        assert len(omega_composites) == 1
+        assert omega_composites[0].text.startswith("Omega")

test_unstructured/chunking/test_title.py

@@ -0,0 +1,890 @@
+# pyright: reportPrivateUsage=false
+
+"""Test suite for the `unstructured.chunking.title` module."""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+import pytest
+
+from test_unstructured.unit_utils import FixtureRequest, Mock, function_mock, input_path
+from unstructured.chunking.base import CHUNK_MULTI_PAGE_DEFAULT
+from unstructured.chunking.title import _ByTitleChunkingOptions, chunk_by_title
+from unstructured.documents.coordinates import CoordinateSystem
+from unstructured.documents.elements import (
+    CheckBox,
+    CompositeElement,
+    CoordinatesMetadata,
+    Element,
+    ElementMetadata,
+    ListItem,
+    Table,
+    TableChunk,
+    Text,
+    Title,
+)
+from unstructured.partition.html import partition_html
+from unstructured.staging.base import elements_from_json
+
+# ================================================================================================
+# INTEGRATION-TESTS
+# ================================================================================================
+# These test `chunk_by_title()` as an integrated whole, calling `chunk_by_title()` and inspecting
+# the outputs.
+# ================================================================================================
+
+
+def test_it_chunks_text_followed_by_table_together_when_both_fit():
+    elements = elements_from_json(input_path("chunking/title_table_200.json"))
+
+    chunks = chunk_by_title(elements, combine_text_under_n_chars=0)
+
+    assert len(chunks) == 2
+    assert isinstance(chunks[0], CompositeElement)
+    assert isinstance(chunks[1], Table)
+
+
+def test_it_chunks_table_followed_by_text_together_when_both_fit():
+    elements = elements_from_json(input_path("chunking/table_text_200.json"))
+
+    # -- disable chunk combining so we test pre-chunking behavior, not chunk-combining --
+    chunks = chunk_by_title(elements, combine_text_under_n_chars=0)
+
+    assert len(chunks) == 2
+    assert isinstance(chunks[0], Table)
+    assert isinstance(chunks[1], CompositeElement)
+
+
+def test_it_splits_oversized_table():
+    elements = elements_from_json(input_path("chunking/table_2000.json"))
+
+    chunks = chunk_by_title(elements)
+
+    assert len(chunks) == 5
+    assert all(isinstance(chunk, TableChunk) for chunk in chunks)
+
+
+def test_skip_table_chunking_passes_oversized_table_through_unchanged():
+    elements = elements_from_json(input_path("chunking/table_2000.json"))
+
+    chunks = chunk_by_title(elements, skip_table_chunking=True)
+
+    assert len(chunks) == 1
+    assert isinstance(chunks[0], Table)
+
+
+def test_skip_table_chunking_does_not_combine_table_with_adjacent_text():
+    table_text = "cell " * 200
+    table = Table(table_text.strip())
+    text_before = Text("Hello world")
+    text_after = Text("Goodbye world")
+
+    chunks = chunk_by_title(
+        [text_before, table, text_after],
+        max_characters=5000,
+        combine_text_under_n_chars=5000,
+        skip_table_chunking=True,
+    )
+
+    assert isinstance(chunks[0], CompositeElement)
+    assert isinstance(chunks[1], Table)
+    assert isinstance(chunks[2], CompositeElement)
+    assert chunks[1].text == table_text.strip()
+
+
+def test_it_repeats_table_headers_by_default_but_can_opt_out():
+    table_html = (
+        "<table>"
+        "<thead>"
+        "<tr><th>Header A</th><th>Header B</th></tr>"
+        "<tr><th>Subhead A</th><th>Subhead B</th></tr>"
+        "</thead>"
+        "<tbody>"
+        "<tr><td>Body 1</td><td>Alpha</td></tr>"
+        "<tr><td>Body 2</td><td>Bravo</td></tr>"
+        "<tr><td>Body 3</td><td>Charlie</td></tr>"
+        "<tr><td>Body 4</td><td>Delta</td></tr>"
+        "</tbody>"
+        "</table>"
+    )
+    table_text = (
+        "Header A Header B\n"
+        "Subhead A Subhead B\n"
+        "Body 1 Alpha\n"
+        "Body 2 Bravo\n"
+        "Body 3 Charlie\n"
+        "Body 4 Delta"
+    )
+    table = Table(table_text, metadata=ElementMetadata(text_as_html=table_html))
+
+    repeated_header_chunks = chunk_by_title(
+        [table], combine_text_under_n_chars=0, max_characters=55
+    )
+    opt_out_chunks = chunk_by_title(
+        [table],
+        combine_text_under_n_chars=0,
+        max_characters=55,
+        repeat_table_headers=False,
+    )
+
+    assert len(repeated_header_chunks) == 4
+    assert all(isinstance(chunk, TableChunk) for chunk in repeated_header_chunks)
+    assert [chunk.text for chunk in repeated_header_chunks] == [
+        "Header A Header B Subhead A Subhead B Body 1 Alpha",
+        "Header A Header B Subhead A Subhead B Body 2 Bravo",
+        "Header A Header B Subhead A Subhead B Body 3 Charlie",
+        "Header A Header B Subhead A Subhead B Body 4 Delta",
+    ]
+    assert [chunk.text for chunk in opt_out_chunks] == [
+        "Header A Header B Subhead A Subhead B Body 1 Alpha",
+        "Body 2 Bravo Body 3 Charlie Body 4 Delta",
+    ]
+
+
+def test_it_starts_new_chunk_for_table_after_full_text_chunk():
+    elements = elements_from_json(input_path("chunking/long_text_table_200.json"))
+
+    chunks = chunk_by_title(elements, max_characters=250)
+
+    assert len(chunks) == 2
+    assert [type(chunk) for chunk in chunks] == [CompositeElement, Table]
+
+
+def test_it_starts_new_chunk_for_text_after_full_table_chunk():
+    elements = elements_from_json(input_path("chunking/full_table_long_text_250.json"))
+
+    chunks = chunk_by_title(elements, max_characters=250)
+
+    assert len(chunks) == 2
+    assert [type(chunk) for chunk in chunks] == [Table, CompositeElement]
+
+
+def test_it_splits_a_large_text_element_into_multiple_chunks():
+    elements: list[Element] = [
+        Title("Introduction"),
+        Text(
+            "Lorem ipsum dolor sit amet consectetur adipiscing elit. In rhoncus ipsum sed lectus"
+            " porta volutpat.",
+        ),
+    ]
+
+    chunks = chunk_by_title(elements, max_characters=50)
+
+    assert chunks == [
+        CompositeElement("Introduction"),
+        CompositeElement("Lorem ipsum dolor sit amet consectetur adipiscing"),
+        CompositeElement("elit. In rhoncus ipsum sed lectus porta volutpat."),
+    ]
+
+
+def test_it_splits_elements_by_title_and_table():
+    elements: list[Element] = [
+        Title("A Great Day"),
+        Text("Today is a great day."),
+        Text("It is sunny outside."),
+        Table("Heading\nCell text"),
+        Title("An Okay Day"),
+        Text("Today is an okay day."),
+        Text("It is rainy outside."),
+        Title("A Bad Day"),
+        Text("Today is a bad day."),
+        Text("It is storming outside."),
+        CheckBox(),
+    ]
+
+    chunks = chunk_by_title(elements, combine_text_under_n_chars=0, include_orig_elements=True)
+
+    assert len(chunks) == 4
+    # --
+    chunk = chunks[0]
+    assert isinstance(chunk, CompositeElement)
+    assert chunk.metadata.orig_elements == [
+        Title("A Great Day"),
+        Text("Today is a great day."),
+        Text("It is sunny outside."),
+    ]
+    # --
+    chunk = chunks[1]
+    assert isinstance(chunk, Table)
+    assert len(chunk.metadata.orig_elements) == 1
+    assert isinstance(chunk.metadata.orig_elements[0], Table)
+    assert chunk.metadata.orig_elements[0].text == "Heading\nCell text"
+    # --
+    chunk = chunks[2]
+    assert isinstance(chunk, CompositeElement)
+    assert chunk.metadata.orig_elements == [
+        Title("An Okay Day"),
+        Text("Today is an okay day."),
+        Text("It is rainy outside."),
+    ]
+    # --
+    chunk = chunks[3]
+    assert isinstance(chunk, CompositeElement)
+    assert chunk.metadata.orig_elements == [
+        Title("A Bad Day"),
+        Text("Today is a bad day."),
+        Text("It is storming outside."),
+        CheckBox(),
+    ]
+
+
+def test_chunk_by_title():
+    elements: list[Element] = [
+        Title("A Great Day", metadata=ElementMetadata(emphasized_text_contents=["Day"])),
+        Text("Today is a great day.", metadata=ElementMetadata(emphasized_text_contents=["day"])),
+        Text("It is sunny outside."),
+        Table("Heading\nCell text"),
+        Title("An Okay Day"),
+        Text("Today is an okay day."),
+        Text("It is rainy outside."),
+        Title("A Bad Day"),
+        Text("Today is a bad day."),
+        Text("It is storming outside."),
+        CheckBox(),
+    ]
+
+    chunks = chunk_by_title(elements, combine_text_under_n_chars=0, include_orig_elements=False)
+
+    assert len(chunks) == 4
+    assert chunks[0] == CompositeElement(
+        "A Great Day\n\nToday is a great day.\n\nIt is sunny outside."
+    )
+    assert isinstance(chunks[1], Table)
+    assert chunks[1].text == "Heading\nCell text"
+    assert chunks[2] == CompositeElement(
+        "An Okay Day\n\nToday is an okay day.\n\nIt is rainy outside."
+    )
+    assert chunks[3] == CompositeElement(
+        "A Bad Day\n\nToday is a bad day.\n\nIt is storming outside."
+    )
+    assert chunks[0].metadata == ElementMetadata(emphasized_text_contents=["Day", "day"])
+
+
+def test_chunk_by_title_separates_by_page_number():
+    elements: list[Element] = [
+        Title("A Great Day", metadata=ElementMetadata(page_number=1)),
+        Text("Today is a great day.", metadata=ElementMetadata(page_number=2)),
+        Text("It is sunny outside.", metadata=ElementMetadata(page_number=2)),
+        Table("Heading\nCell text"),
+        Title("An Okay Day"),
+        Text("Today is an okay day."),
+        Text("It is rainy outside."),
+        Title("A Bad Day"),
+        Text("Today is a bad day."),
+        Text("It is storming outside."),
+        CheckBox(),
+    ]
+    chunks = chunk_by_title(elements, multipage_sections=False, combine_text_under_n_chars=0)
+
+    assert len(chunks) == 5
+    assert chunks[0] == CompositeElement("A Great Day")
+    assert chunks[1] == CompositeElement("Today is a great day.\n\nIt is sunny outside.")
+    assert isinstance(chunks[2], Table)
+    assert chunks[2].text == "Heading\nCell text"
+    assert chunks[3] == CompositeElement(
+        "An Okay Day\n\nToday is an okay day.\n\nIt is rainy outside."
+    )
+    assert chunks[4] == CompositeElement(
+        "A Bad Day\n\nToday is a bad day.\n\nIt is storming outside."
+    )
+
+
+def test_chuck_by_title_respects_multipage():
+    elements: list[Element] = [
+        Title("A Great Day", metadata=ElementMetadata(page_number=1)),
+        Text("Today is a great day.", metadata=ElementMetadata(page_number=2)),
+        Text("It is sunny outside.", metadata=ElementMetadata(page_number=2)),
+        Table("Heading\nCell text"),
+        Title("An Okay Day"),
+        Text("Today is an okay day."),
+        Text("It is rainy outside."),
+        Title("A Bad Day"),
+        Text("Today is a bad day."),
+        Text("It is storming outside."),
+        CheckBox(),
+    ]
+    chunks = chunk_by_title(elements, multipage_sections=True, combine_text_under_n_chars=0)
+    assert len(chunks) == 4
+    assert chunks[0] == CompositeElement(
+        "A Great Day\n\nToday is a great day.\n\nIt is sunny outside."
+    )
+    assert isinstance(chunks[1], Table)
+    assert chunks[1].text == "Heading\nCell text"
+    assert chunks[2] == CompositeElement(
+        "An Okay Day\n\nToday is an okay day.\n\nIt is rainy outside."
+    )
+    assert chunks[3] == CompositeElement(
+        "A Bad Day\n\nToday is a bad day.\n\nIt is storming outside."
+    )
+
+
+def test_chunk_by_title_groups_across_pages():
+    elements: list[Element] = [
+        Title("A Great Day", metadata=ElementMetadata(page_number=1)),
+        Text("Today is a great day.", metadata=ElementMetadata(page_number=2)),
+        Text("It is sunny outside.", metadata=ElementMetadata(page_number=2)),
+        Table("Heading\nCell text"),
+        Title("An Okay Day"),
+        Text("Today is an okay day."),
+        Text("It is rainy outside."),
+        Title("A Bad Day"),
+        Text("Today is a bad day."),
+        Text("It is storming outside."),
+        CheckBox(),
+    ]
+    chunks = chunk_by_title(elements, multipage_sections=True, combine_text_under_n_chars=0)
+
+    assert len(chunks) == 4
+    assert chunks[0] == CompositeElement(
+        "A Great Day\n\nToday is a great day.\n\nIt is sunny outside."
+    )
+    assert isinstance(chunks[1], Table)
+    assert chunks[1].text == "Heading\nCell text"
+    assert chunks[2] == CompositeElement(
+        "An Okay Day\n\nToday is an okay day.\n\nIt is rainy outside."
+    )
+    assert chunks[3] == CompositeElement(
+        "A Bad Day\n\nToday is a bad day.\n\nIt is storming outside."
+    )
+
+
+def test_add_chunking_strategy_on_partition_html():
+    filename = "example-docs/example-10k-1p.html"
+    chunk_elements = partition_html(filename, chunking_strategy="by_title")
+    elements = partition_html(filename)
+    chunks = chunk_by_title(elements)
+    assert chunk_elements != elements
+    assert chunk_elements == chunks
+
+
+def test_add_chunking_strategy_respects_max_characters():
+    filename = "example-docs/example-10k-1p.html"
+    chunk_elements = partition_html(
+        filename,
+        chunking_strategy="by_title",
+        combine_text_under_n_chars=0,
+        new_after_n_chars=50,
+        max_characters=100,
+    )
+    elements = partition_html(filename)
+    chunks = chunk_by_title(
+        elements,
+        combine_text_under_n_chars=0,
+        new_after_n_chars=50,
+        max_characters=100,
+    )
+
+    for chunk in chunks:
+        assert isinstance(chunk, Text)
+        assert len(chunk.text) <= 100
+    for chunk_element in chunk_elements:
+        assert isinstance(chunk_element, Text)
+        assert len(chunk_element.text) <= 100
+    assert chunk_elements != elements
+    assert chunk_elements == chunks
+
+
+def test_add_chunking_strategy_forwards_repeat_table_headers():
+    filename = "example-docs/example-10k-1p.html"
+    chunk_elements = partition_html(
+        filename,
+        chunking_strategy="by_title",
+        repeat_table_headers=False,
+    )
+    elements = partition_html(filename)
+    chunks = chunk_by_title(elements, repeat_table_headers=False)
+
+    assert chunk_elements != elements
+    assert chunk_elements == chunks
+
+
+def test_chunk_by_title_drops_detection_class_prob():
+    elements: list[Element] = [
+        Title(
+            "A Great Day",
+            metadata=ElementMetadata(
+                detection_class_prob=0.5,
+            ),
+        ),
+        Text(
+            "Today is a great day.",
+            metadata=ElementMetadata(
+                detection_class_prob=0.62,
+            ),
+        ),
+        Text(
+            "It is sunny outside.",
+            metadata=ElementMetadata(
+                detection_class_prob=0.73,
+            ),
+        ),
+        Title(
+            "An Okay Day",
+            metadata=ElementMetadata(
+                detection_class_prob=0.84,
+            ),
+        ),
+        Text(
+            "Today is an okay day.",
+            metadata=ElementMetadata(
+                detection_class_prob=0.95,
+            ),
+        ),
+    ]
+    chunks = chunk_by_title(elements, combine_text_under_n_chars=0)
+    assert str(chunks[0]) == str(
+        CompositeElement("A Great Day\n\nToday is a great day.\n\nIt is sunny outside."),
+    )
+    assert str(chunks[1]) == str(CompositeElement("An Okay Day\n\nToday is an okay day."))
+
+
+def test_chunk_by_title_drops_extra_metadata():
+    elements: list[Element] = [
+        Title(
+            "A Great Day",
+            metadata=ElementMetadata(
+                coordinates=CoordinatesMetadata(
+                    points=(
+                        (0.1, 0.1),
+                        (0.2, 0.1),
+                        (0.1, 0.2),
+                        (0.2, 0.2),
+                    ),
+                    system=CoordinateSystem(width=0.1, height=0.1),
+                ),
+            ),
+        ),
+        Text(
+            "Today is a great day.",
+            metadata=ElementMetadata(
+                coordinates=CoordinatesMetadata(
+                    points=(
+                        (0.2, 0.2),
+                        (0.3, 0.2),
+                        (0.2, 0.3),
+                        (0.3, 0.3),
+                    ),
+                    system=CoordinateSystem(width=0.2, height=0.2),
+                ),
+            ),
+        ),
+        Text(
+            "It is sunny outside.",
+            metadata=ElementMetadata(
+                coordinates=CoordinatesMetadata(
+                    points=(
+                        (0.3, 0.3),
+                        (0.4, 0.3),
+                        (0.3, 0.4),
+                        (0.4, 0.4),
+                    ),
+                    system=CoordinateSystem(width=0.3, height=0.3),
+                ),
+            ),
+        ),
+        Title(
+            "An Okay Day",
+            metadata=ElementMetadata(
+                coordinates=CoordinatesMetadata(
+                    points=(
+                        (0.3, 0.3),
+                        (0.4, 0.3),
+                        (0.3, 0.4),
+                        (0.4, 0.4),
+                    ),
+                    system=CoordinateSystem(width=0.3, height=0.3),
+                ),
+            ),
+        ),
+        Text(
+            "Today is an okay day.",
+            metadata=ElementMetadata(
+                coordinates=CoordinatesMetadata(
+                    points=(
+                        (0.4, 0.4),
+                        (0.5, 0.4),
+                        (0.4, 0.5),
+                        (0.5, 0.5),
+                    ),
+                    system=CoordinateSystem(width=0.4, height=0.4),
+                ),
+            ),
+        ),
+    ]
+
+    chunks = chunk_by_title(elements, combine_text_under_n_chars=0)
+
+    assert str(chunks[0]) == str(
+        CompositeElement("A Great Day\n\nToday is a great day.\n\nIt is sunny outside."),
+    )
+
+    assert str(chunks[1]) == str(CompositeElement("An Okay Day\n\nToday is an okay day."))
+
+
+def test_it_considers_separator_length_when_pre_chunking():
+    """PreChunker includes length of separators when computing remaining space."""
+    elements: list[Element] = [
+        Title("Chunking Priorities"),  # 19 chars
+        ListItem("Divide text into manageable chunks"),  # 34 chars
+        ListItem("Preserve semantic boundaries"),  # 28 chars
+        ListItem("Minimize mid-text chunk-splitting"),  # 33 chars
+    ]  # 114 chars total but 120 chars with separators
+
+    chunks = chunk_by_title(elements, max_characters=115)
+
+    assert chunks == [
+        CompositeElement(
+            "Chunking Priorities"
+            "\n\nDivide text into manageable chunks"
+            "\n\nPreserve semantic boundaries",
+        ),
+        CompositeElement("Minimize mid-text chunk-splitting"),
+    ]
+
+
+# ================================================================================================
+# UNIT-TESTS
+# ================================================================================================
+# These test individual components in isolation so can exercise all edge cases while still
+# performing well.
+# ================================================================================================
+
+
+class Describe_chunk_by_title:
+    """Unit-test suite for `unstructured.chunking.title.chunk_by_title()` function."""
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"include_orig_elements": True}, True),
+            ({"include_orig_elements": False}, False),
+            ({"include_orig_elements": None}, True),
+            ({}, True),
+        ],
+    )
+    def it_supports_the_include_orig_elements_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_by_title_: Mock
+    ):
+        # -- this line would raise if "include_orig_elements" was not an available parameter on
+        # -- `chunk_by_title()`.
+        chunk_by_title([], **kwargs)
+
+        _, opts = _chunk_by_title_.call_args.args
+        assert opts.include_orig_elements is expected_value
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"repeat_table_headers": True}, True),
+            ({"repeat_table_headers": False}, False),
+            ({"repeat_table_headers": None}, True),
+            ({}, True),
+        ],
+    )
+    def it_supports_the_repeat_table_headers_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_by_title_: Mock
+    ):
+        # -- this line would raise if "repeat_table_headers" was not an available parameter on
+        # -- `chunk_by_title()`.
+        chunk_by_title([], **kwargs)
+
+        _, opts = _chunk_by_title_.call_args.args
+        assert opts.repeat_table_headers is expected_value
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"skip_table_chunking": True}, True),
+            ({"skip_table_chunking": False}, False),
+            ({"skip_table_chunking": None}, False),
+            ({}, False),
+        ],
+    )
+    def it_supports_the_skip_table_chunking_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_by_title_: Mock
+    ):
+        chunk_by_title([], **kwargs)
+
+        _, opts = _chunk_by_title_.call_args.args
+        assert opts.skip_table_chunking is expected_value
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"isolate_table": True}, True),
+            ({"isolate_table": False}, False),
+            ({"isolate_table": None}, True),
+            ({}, True),
+        ],
+    )
+    def it_supports_the_isolate_table_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_by_title_: Mock
+    ):
+        chunk_by_title([], **kwargs)
+
+        _, opts = _chunk_by_title_.call_args.args
+        assert opts.isolate_table is expected_value
+
+    # -- fixtures --------------------------------------------------------------------------------
+
+    @pytest.fixture()
+    def _chunk_by_title_(self, request: FixtureRequest):
+        return function_mock(request, "unstructured.chunking.title._chunk_by_title")
+
+
+class Describe_ByTitleChunkingOptions:
+    """Unit-test suite for `unstructured.chunking.title._ByTitleChunkingOptions` objects."""
+
+    @pytest.mark.parametrize("n_chars", [-1, -42])
+    def it_rejects_combine_text_under_n_chars_for_n_less_than_zero(self, n_chars: int):
+        with pytest.raises(
+            ValueError,
+            match=f"'combine_text_under_n_chars' argument must be >= 0, got {n_chars}",
+        ):
+            _ByTitleChunkingOptions.new(combine_text_under_n_chars=n_chars)
+
+    def it_accepts_0_for_combine_text_under_n_chars_to_disable_chunk_combining(self):
+        """Specifying `combine_text_under_n_chars=0` is how a caller disables chunk-combining."""
+        opts = _ByTitleChunkingOptions(combine_text_under_n_chars=0)
+        assert opts.combine_text_under_n_chars == 0
+
+    def it_does_not_complain_when_specifying_combine_text_under_n_chars_by_itself(self):
+        """Caller can specify `combine_text_under_n_chars` arg without specifying other options."""
+        try:
+            opts = _ByTitleChunkingOptions(combine_text_under_n_chars=50)
+        except ValueError:
+            pytest.fail("did not accept `combine_text_under_n_chars` as option by itself")
+
+        assert opts.combine_text_under_n_chars == 50
+
+    @pytest.mark.parametrize(
+        ("combine_text_under_n_chars", "max_characters", "expected_hard_max"),
+        [(600, None, 500), (600, 450, 450)],
+    )
+    def it_rejects_combine_text_under_n_chars_greater_than_maxchars(
+        self, combine_text_under_n_chars: int, max_characters: Optional[int], expected_hard_max: int
+    ):
+        """`combine_text_under_n_chars` > `max_characters` can produce behavior confusing to users.
+
+        The behavior is no different from `combine_text_under_n_chars == max_characters`, but if
+        `max_characters` is left to default (500) and `combine_text_under_n_chars` is set to a
+        larger number like 1500 then it can look like chunk-combining isn't working.
+        """
+        with pytest.raises(
+            ValueError,
+            match=(
+                "'combine_text_under_n_chars' argument must not exceed `max_characters` value,"
+                f" got {combine_text_under_n_chars} > {expected_hard_max}"
+            ),
+        ):
+            _ByTitleChunkingOptions.new(
+                max_characters=max_characters, combine_text_under_n_chars=combine_text_under_n_chars
+            )
+
+    def it_does_not_complain_when_specifying_new_after_n_chars_by_itself(self):
+        """Caller can specify `new_after_n_chars` arg without specifying any other options."""
+        try:
+            opts = _ByTitleChunkingOptions.new(new_after_n_chars=200)
+        except ValueError:
+            pytest.fail("did not accept `new_after_n_chars` as option by itself")
+
+        assert opts.soft_max == 200
+
+    @pytest.mark.parametrize(
+        ("multipage_sections", "expected_value"),
+        [(True, True), (False, False), (None, CHUNK_MULTI_PAGE_DEFAULT)],
+    )
+    def it_knows_whether_to_break_chunks_on_page_boundaries(
+        self, multipage_sections: bool, expected_value: bool
+    ):
+        opts = _ByTitleChunkingOptions(multipage_sections=multipage_sections)
+        assert opts.multipage_sections is expected_value
+
+
+# ================================================================================================
+# TOKEN-BASED CHUNKING INTEGRATION TESTS
+# ================================================================================================
+
+
+class DescribeTokenBasedChunking:
+    """Integration tests for token-based chunking with chunk_by_title()."""
+
+    @pytest.fixture
+    def _tiktoken_installed(self):
+        """Skip test if tiktoken is not installed."""
+        pytest.importorskip("tiktoken")
+
+    def it_chunks_elements_by_token_count(self, _tiktoken_installed: None):
+        """Test that chunk_by_title works with max_tokens parameter and respects token limits."""
+        import tiktoken
+
+        enc = tiktoken.get_encoding("cl100k_base")
+        elements: list[Element] = [
+            Title("Introduction"),
+            Text("This is a test document with some text content."),
+            Text("Here is more content that should be chunked appropriately."),
+        ]
+
+        chunks = chunk_by_title(
+            elements,
+            max_tokens=20,
+            tokenizer="cl100k_base",
+            combine_text_under_n_chars=0,
+        )
+
+        # -- verify chunks were created --
+        assert len(chunks) >= 1
+        assert all(isinstance(chunk, (CompositeElement, Table, TableChunk)) for chunk in chunks)
+
+        # -- verify each chunk respects the token limit --
+        for chunk in chunks:
+            token_count = len(enc.encode(chunk.text))
+            assert token_count <= 20, f"Chunk exceeded token limit: {token_count} tokens"
+
+        # -- verify the first chunk contains expected content --
+        assert "Introduction" in chunks[0].text
+
+    def it_respects_new_after_n_tokens_soft_limit(self, _tiktoken_installed: None):
+        """Test that new_after_n_tokens creates smaller chunks with correct content."""
+        import tiktoken
+
+        enc = tiktoken.get_encoding("cl100k_base")
+        elements: list[Element] = [
+            Title("Section 1"),
+            Text("Some text for section one."),
+            Text("More text for section one."),
+            Title("Section 2"),
+            Text("Text for section two."),
+        ]
+
+        # -- with a low soft limit, each section should become its own chunk --
+        chunks = chunk_by_title(
+            elements,
+            max_tokens=100,
+            new_after_n_tokens=10,
+            tokenizer="cl100k_base",
+            combine_text_under_n_chars=0,
+        )
+
+        # -- verify we get multiple chunks --
+        assert len(chunks) >= 2
+
+        # -- verify each chunk respects the hard limit --
+        for chunk in chunks:
+            token_count = len(enc.encode(chunk.text))
+            assert token_count <= 100, f"Chunk exceeded token limit: {token_count} tokens"
+
+        # -- verify content is distributed across chunks --
+        all_text = " ".join(chunk.text for chunk in chunks)
+        assert "Section 1" in all_text
+        assert "Section 2" in all_text
+
+    def it_rejects_max_tokens_and_max_characters_together(self, _tiktoken_installed: None):
+        """Test that specifying both max_tokens and max_characters raises ValueError."""
+        elements: list[Element] = [Text("Some text")]
+
+        with pytest.raises(ValueError, match="mutually exclusive"):
+            chunk_by_title(
+                elements,
+                max_tokens=100,
+                max_characters=500,
+                tokenizer="cl100k_base",
+            )
+
+    def it_rejects_max_tokens_without_tokenizer(self, _tiktoken_installed: None):
+        """Test that max_tokens requires tokenizer to be specified."""
+        elements: list[Element] = [Text("Some text")]
+
+        with pytest.raises(ValueError, match="'tokenizer' is required"):
+            chunk_by_title(elements, max_tokens=100)
+
+    def it_accepts_model_name_as_tokenizer(self, _tiktoken_installed: None):
+        """Test that model names like 'gpt-4' work as tokenizer."""
+        import tiktoken
+
+        # -- gpt-4 uses cl100k_base encoding --
+        enc = tiktoken.encoding_for_model("gpt-4")
+        elements: list[Element] = [
+            Title("Test"),
+            Text("Some test content."),
+        ]
+
+        chunks = chunk_by_title(
+            elements,
+            max_tokens=50,
+            tokenizer="gpt-4",
+            combine_text_under_n_chars=0,
+        )
+
+        assert len(chunks) >= 1
+
+        # -- verify token counts are within limit --
+        for chunk in chunks:
+            token_count = len(enc.encode(chunk.text))
+            assert token_count <= 50, f"Chunk exceeded token limit: {token_count} tokens"
+
+        # -- verify content --
+        assert chunks[0].text == "Test\n\nSome test content."
+
+    def it_splits_oversized_element_respecting_token_limit(self, _tiktoken_installed: None):
+        """Test that oversized elements are split correctly by token count."""
+        import tiktoken
+
+        enc = tiktoken.get_encoding("cl100k_base")
+        # -- create a text that will definitely need to be split (>10 tokens) --
+        long_text = "The quick brown fox jumps over the lazy dog. " * 10
+        elements: list[Element] = [Text(long_text)]
+
+        chunks = chunk_by_title(
+            elements,
+            max_tokens=15,
+            tokenizer="cl100k_base",
+            combine_text_under_n_chars=0,
+        )
+
+        # -- should produce multiple chunks --
+        assert len(chunks) > 1
+
+        # -- each chunk should respect the token limit --
+        for chunk in chunks:
+            token_count = len(enc.encode(chunk.text))
+            assert token_count <= 15, f"Chunk exceeded token limit: {token_count} tokens"
+
+    def it_applies_token_based_overlap_in_split_chunks(self, _tiktoken_installed: None):
+        """Test that overlap in token mode is measured in tokens, not characters."""
+        import tiktoken
+
+        enc = tiktoken.get_encoding("cl100k_base")
+        # -- create text that will be split into multiple chunks --
+        long_text = "Apple banana cherry date elderberry fig grape honeydew. " * 5
+        elements: list[Element] = [Text(long_text)]
+
+        chunks = chunk_by_title(
+            elements,
+            max_tokens=12,
+            tokenizer="cl100k_base",
+            overlap=3,  # 3 tokens of overlap
+            combine_text_under_n_chars=0,
+        )
+
+        # -- should produce multiple chunks --
+        assert len(chunks) >= 2
+
+        # -- verify overlap: second chunk should start with tokens from end of first --
+        if len(chunks) >= 2:
+            # -- the overlap tail from chunk 0 should appear at start of chunk 1 --
+            first_chunk_text = chunks[0].text
+            second_chunk_text = chunks[1].text
+
+            # -- the overlap should be approximately 3 tokens worth of text --
+            # -- verify second chunk starts with content from end of first chunk --
+            # -- (the overlap mechanism prepends tail to remainder) --
+            assert len(second_chunk_text) > 0
+            # -- verify some content from first chunk appears at start of second --
+            first_chunk_words = first_chunk_text.split()
+            assert any(word in second_chunk_text for word in first_chunk_words[-3:])
+            # -- verify each chunk respects limits --
+            for chunk in chunks:
+                token_count = len(enc.encode(chunk.text))
+                assert token_count <= 12, f"Chunk exceeded token limit: {token_count} tokens"

test_unstructured/cleaners/test_core.py

@@ -0,0 +1,305 @@
+import re
+
+import pytest
+
+from unstructured.cleaners import core
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        (
+            "\x88This text contains non-ascii characters!\x88",
+            "This text contains non-ascii characters!",
+        ),
+        ("\x93A lovely quote!\x94", "A lovely quote!"),
+        ("● An excellent point! ●●●", " An excellent point! "),
+        ("Item\xa01A", "Item1A"),
+        ("Our dog's bowl.", "Our dog's bowl."),
+        ("5 w=E2=80=99s", "5 w=E2=80=99s"),
+    ],
+)
+def test_clean_non_ascii_chars(text, expected):
+    assert core.clean_non_ascii_chars(text) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("● An excellent point!", "An excellent point!"),
+        ("● An excellent point! ●●●", "An excellent point! ●●●"),
+        ("An excellent point!", "An excellent point!"),
+        ("Morse code! ●●●", "Morse code! ●●●"),
+        ("– An EN DASH bullet point!", "An EN DASH bullet point!"),
+        ("\u2013 Another EN DASH bullet!", "Another EN DASH bullet!"),
+        ("Text with – inside", "Text with – inside"),
+    ],
+)
+def test_clean_bullets(text, expected):
+    assert core.clean_bullets(text=text) == expected
+    assert core.clean(text=text, bullets=True) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("1. Introduction:", "Introduction:"),
+        ("a. Introduction:", "Introduction:"),
+        ("20.3 Morse code ●●●", "Morse code ●●●"),
+        ("5.3.1 Convolutional Networks ", "Convolutional Networks"),
+        ("D.b.C Recurrent Neural Networks", "Recurrent Neural Networks"),
+        ("2.b.1 Recurrent Neural Networks", "Recurrent Neural Networks"),
+        ("eins. Neural Networks", "eins. Neural Networks"),
+        ("bb.c Feed Forward Neural Networks", "Feed Forward Neural Networks"),
+        ("aaa.ccc Metrics", "aaa.ccc Metrics"),
+        (" version = 3.8", " version = 3.8"),
+        ("1 2. 3 4", "1 2. 3 4"),
+        ("1) 2. 3 4", "1) 2. 3 4"),
+        ("2,3. Morse code 3. ●●●", "2,3. Morse code 3. ●●●"),
+        ("1..2.3 four", "1..2.3 four"),
+        ("Fig. 2: The relationship", "Fig. 2: The relationship"),
+        ("23 is everywhere", "23 is everywhere"),
+    ],
+)
+def test_clean_ordered_bullets(text, expected):
+    assert core.clean_ordered_bullets(text=text) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("The æther is a classic element.", "The aether is a classic element."),
+        ("In old texts, Æsop's fables are", "In old texts, AEsop's fables are"),
+        ("The buffer zone is there.", "The buffer zone is there."),
+        ("The file was found in the system.", "The file was found in the system."),
+        ("She had a flower in her hair.", "She had a flower in her hair."),
+        ("The coffin was placed in the grave.", "The coffin was placed in the grave."),
+        ("The buffle zone was clearly marked.", "The buffle zone was clearly marked."),
+        ("The craſtsman worked with dedication.", "The craftsman worked with dedication."),
+        ("The symbol ʪ is very rare.", "The symbol ls is very rare."),
+        ("The word 'cœur' means 'heart' in French.", "The word 'coeur' means 'heart' in French."),
+        ("The word 'Œuvre' refers to the works", "The word 'OEuvre' refers to the works"),
+        ("The ȹ symbol is used in some contexts.", "The qp symbol is used in some contexts."),
+        ("The postman delivers mail daily.", "The postman delivers mail daily."),
+        (
+            "The symbol ʦ can be found in certain alphabets.",
+            "The symbol ts can be found in certain alphabets.",
+        ),
+    ],
+)
+def test_clean_ligatures(text, expected):
+    assert core.clean_ligatures(text=text) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("\x93A lovely quote!\x94", "“A lovely quote!”"),
+        ("\x91A lovely quote!\x92", "‘A lovely quote!’"),
+        ("Our dog's bowl.", "Our dog's bowl."),
+    ],
+)
+def test_replace_unicode_quotes(text, expected):
+    assert core.replace_unicode_quotes(text=text) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [("5 w=E2=80=99s", "5 w’s")],
+)
+def test_replace_mime_encodings(text, expected):
+    assert core.replace_mime_encodings(text=text) == expected
+
+
+def test_replace_mime_encodings_works_with_different_encodings():
+    text = "5 w=E2=80-99s=E2=80-92"
+    assert core.replace_mime_encodings(text=text, encoding="latin-1") == "5 wâ\x80-99sâ\x80-92"
+
+
+def test_replace_mime_encodings_works_with_right_to_left_encodings():
+    text = "=EE=E0=E9=E4"
+    assert core.replace_mime_encodings(text=text, encoding="iso-8859-8") == "מאיה"
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("“A lovely quote!”", "A lovely quote"),
+        ("‘A lovely quote!’", "A lovely quote"),
+        ("'()[]{};:'\",.?/\\-_", ""),
+    ],
+)
+def test_remove_punctuation(text, expected):
+    assert core.remove_punctuation(text) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("RISK\n\nFACTORS", "RISK FACTORS"),
+        ("Item\xa01A", "Item 1A"),
+        ("  Risk factors ", "Risk factors"),
+        ("Risk   factors ", "Risk factors"),
+    ],
+)
+def test_clean_extra_whitespace(text, expected):
+    assert core.clean_extra_whitespace(text) == expected
+    assert core.clean(text=text, extra_whitespace=True) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("Risk-factors", "Risk factors"),
+        ("Risk – factors", "Risk   factors"),
+        ("Risk\u2013factors", "Risk factors"),
+        ("Risk factors-\u2013", "Risk factors"),
+    ],
+)
+def test_clean_dashes(text, expected):
+    assert core.clean_dashes(text) == expected
+    assert core.clean(text=text, dashes=True) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("Item 1A:", "Item 1A"),
+        ("Item 1A;", "Item 1A"),
+        ("Item 1A.", "Item 1A"),
+        ("Item 1A,", "Item 1A"),
+        ("Item, 1A: ", "Item, 1A"),
+    ],
+)
+def test_clean_trailing_punctuation(text, expected):
+    assert core.clean_trailing_punctuation(text) == expected
+    assert core.clean(text=text, trailing_punctuation=True) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "pattern", "ignore_case", "strip", "expected"),
+    [
+        ("SUMMARY: A great SUMMARY", r"(SUMMARY|DESC):", False, True, "A great SUMMARY"),
+        ("DESC: A great SUMMARY", r"(SUMMARY|DESC):", False, True, "A great SUMMARY"),
+        ("SUMMARY: A great SUMMARY", r"(SUMMARY|DESC):", False, False, " A great SUMMARY"),
+        ("summary: A great SUMMARY", r"(SUMMARY|DESC):", True, True, "A great SUMMARY"),
+    ],
+)
+def test_clean_prefix(text, pattern, ignore_case, strip, expected):
+    assert core.clean_prefix(text, pattern, ignore_case, strip) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "pattern", "ignore_case", "strip", "expected"),
+    [
+        ("The END! END", r"(END|STOP)", False, True, "The END!"),
+        ("The END! STOP", r"(END|STOP)", False, True, "The END!"),
+        ("The END! END", r"(END|STOP)", False, False, "The END! "),
+        ("The END! end", r"(END|STOP)", True, True, "The END!"),
+    ],
+)
+def test_clean_postfix(text, pattern, ignore_case, strip, expected):
+    assert core.clean_postfix(text, pattern, ignore_case, strip) == expected
+
+
+def test_group_broken_paragraphs():
+    text = """The big red fox
+is walking down the lane.
+
+At the end of the lane
+the fox met a friendly bear."""
+
+    assert (
+        core.group_broken_paragraphs(text)
+        == """The big red fox is walking down the lane.
+
+At the end of the lane the fox met a friendly bear."""
+    )
+
+
+def test_group_broken_paragraphs_non_default_settings():
+    text = """The big red fox
+
+is walking down the lane.
+
+
+At the end of the lane
+
+the fox met a friendly bear."""
+
+    para_split_re = re.compile(r"(\s*\n\s*){3}")
+
+    clean_text = core.group_broken_paragraphs(text, paragraph_split=para_split_re)
+    assert (
+        clean_text
+        == """The big red fox is walking down the lane.
+
+At the end of the lane the fox met a friendly bear."""
+    )
+
+
+def test_group_broken_paragraphs_with_bullets():
+    text = """○The big red fox
+is walking down the lane.
+
+○At the end of the lane
+the fox met a friendly bear."""
+    assert core.group_bullet_paragraph(text) == [
+        "○The big red fox is walking down the lane. ",
+        "○At the end of the lane the fox met a friendly bear.",
+    ]
+
+
+def test_group_bullet_paragraph_with_e_bullets():
+    text = """e The big red fox
+is walking down the lane.
+
+e At the end of the lane
+the fox met a friendly bear."""
+    assert core.group_bullet_paragraph(text) == [
+        "· The big red fox is walking down the lane. ",
+        "· At the end of the lane the fox met a friendly bear.",
+    ]
+
+
+@pytest.mark.parametrize(
+    # NOTE(yuming): Tests combined cleaners
+    (
+        "text",
+        "extra_whitespace",
+        "dashes",
+        "bullets",
+        "lowercase",
+        "trailing_punctuation",
+        "expected",
+    ),
+    [
+        ("  Risk-factors ", True, True, False, False, False, "Risk factors"),
+        ("● Point!  ●●● ", True, False, True, False, False, "Point! ●●●"),
+        ("Risk- factors ", True, False, False, True, False, "risk- factors"),
+        ("Risk   factors: ", True, False, False, False, True, "Risk factors"),
+        ("● Risk-factors●●● ", False, True, True, False, False, "Risk factors●●●"),
+        ("Risk-factors ", False, True, False, True, False, "risk factors"),
+        ("Risk-factors: ", False, True, False, False, True, "Risk factors"),
+        ("● Point! ●●● ", False, False, True, True, False, "point! ●●●"),
+        ("● Point! ●●●: ", False, False, True, False, True, "Point! ●●●"),
+        ("Risk factors: ", False, False, False, True, True, "risk factors"),
+    ],
+)
+def test_clean(text, extra_whitespace, dashes, bullets, lowercase, trailing_punctuation, expected):
+    assert (
+        core.clean(
+            text=text,
+            extra_whitespace=extra_whitespace,
+            dashes=dashes,
+            bullets=bullets,
+            trailing_punctuation=trailing_punctuation,
+            lowercase=lowercase,
+        )
+        == expected
+    )
+
+
+def test_bytes_string_to_string():
+    text = "\xe6\xaf\x8f\xe6\x97\xa5\xe6\x96\xb0\xe9\x97\xbb"
+    assert core.bytes_string_to_string(text, "utf-8") == "每日新闻"

test_unstructured/cleaners/test_extract.py

@@ -0,0 +1,156 @@
+import datetime
+
+import pytest
+
+from unstructured.cleaners import extract
+
+EMAIL_META_DATA_INPUT = """from ABC.DEF.local ([ba23::58b5:2236:45g2:88h2]) by
+    \n ABC.DEF.local ([68.183.71.12]) with mapi id\
+    n 32.88.5467.123; Fri, 26 Mar 2021 11:04:09 +1200"""
+
+
+def test_get_indexed_match_raises_with_bad_index():
+    with pytest.raises(ValueError):
+        extract._get_indexed_match("BLAH BLAH BLAH", "BLAH", -1)
+
+
+def test_get_indexed_match_raises_with_index_too_high():
+    with pytest.raises(ValueError):
+        extract._get_indexed_match("BLAH BLAH BLAH", "BLAH", 4)
+
+
+def test_extract_text_before():
+    text = "Teacher: BLAH BLAH BLAH; Student: BLAH BLAH BLAH!"
+    assert extract.extract_text_before(text, "BLAH", 1) == "Teacher: BLAH"
+
+
+def test_extract_text_after():
+    text = "Teacher: BLAH BLAH BLAH; Student: BLAH BLAH BLAH!"
+    assert extract.extract_text_after(text, "BLAH;", 0) == "Student: BLAH BLAH BLAH!"
+
+
+def test_extract_email_address():
+    text = "Im Rabn <Im.Rabn@npf.gov.nr>"
+    assert extract.extract_email_address(text) == ["im.rabn@npf.gov.nr"]
+
+
+def test_extract_ip_address():
+    assert extract.extract_ip_address(EMAIL_META_DATA_INPUT) == [
+        "ba23::58b5:2236:45g2:88h2",
+        "68.183.71.12",
+    ]
+
+
+def test_extract_ip_address_name():
+    assert extract.extract_ip_address_name(EMAIL_META_DATA_INPUT) == [
+        "ABC.DEF.local",
+        "ABC.DEF.local",
+    ]
+
+
+def test_extract_mapi_id():
+    assert extract.extract_mapi_id(EMAIL_META_DATA_INPUT) == ["32.88.5467.123"]
+
+
+def test_extract_datetimetz():
+    assert extract.extract_datetimetz(EMAIL_META_DATA_INPUT) == datetime.datetime(
+        2021,
+        3,
+        26,
+        11,
+        4,
+        9,
+        tzinfo=datetime.timezone(datetime.timedelta(seconds=43200)),
+    )
+
+
+def test_extract_datetimetz_works_with_no_date():
+    assert extract.extract_datetimetz("NO DATE HERE") is None
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("215-867-5309", "215-867-5309"),
+        ("Phone Number: +1 215.867.5309", "+1 215.867.5309"),
+        ("Phone Number: Just Kidding", ""),
+    ],
+)
+def test_extract_us_phone_number(text, expected):
+    phone_number = extract.extract_us_phone_number(text)
+    assert phone_number == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("1. Introduction:", ("1", None, None)),
+        ("a. Introduction:", ("a", None, None)),
+        ("20.3 Morse code ●●●", ("20", "3", None)),
+        ("5.3.1 Convolutional Networks ", ("5", "3", "1")),
+        ("D.b.C Recurrent Neural Networks", ("D", "b", "C")),
+        ("2.b.1 Recurrent Neural Networks", ("2", "b", "1")),
+        ("eins. Neural Networks", (None, None, None)),
+        ("bb.c Feed Forward Neural Networks", ("bb", "c", None)),
+        ("aaa.ccc Metrics", (None, None, None)),
+        (" version = 3.8", (None, None, None)),
+        ("1 2. 3 4", (None, None, None)),
+        ("1) 2. 3 4", (None, None, None)),
+        ("2,3. Morse code 3. ●●●", (None, None, None)),
+        ("1..2.3 four", (None, None, None)),
+        ("Fig. 2: The relationship", (None, None, None)),
+        ("23 is everywhere", (None, None, None)),
+    ],
+)
+def test_extract_ordered_bullets(text, expected):
+    assert extract.extract_ordered_bullets(text=text) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        (
+            "https://my-image.jpg",
+            (["https://my-image.jpg"]),
+        ),
+        (
+            "https://my-image.png with some text",
+            (["https://my-image.png"]),
+        ),
+        (
+            "https://my-image/with/some/path.png",
+            (["https://my-image/with/some/path.png"]),
+        ),
+        (
+            "some text https://my-image.jpg with another http://my-image.bmp",
+            (["https://my-image.jpg", "http://my-image.bmp"]),
+        ),
+        (
+            "http://not-an-image.com",
+            ([]),
+        ),
+        (
+            "some text",
+            ([]),
+        ),
+        (
+            "some text https://my-image.JPG with another http://my-image.BMP",
+            (["https://my-image.JPG", "http://my-image.BMP"]),
+        ),
+        (
+            "http://my-path-with-CAPS/my-image.JPG",
+            (["http://my-path-with-CAPS/my-image.JPG"]),
+        ),
+        (
+            "http://my-path/my%20image.JPG",
+            (["http://my-path/my%20image.JPG"]),
+        ),
+        # url with reference #
+        (
+            "https://my-image.jpg#ref",
+            (["https://my-image.jpg"]),
+        ),
+    ],
+)
+def test_extract_image_urls_from_html(text, expected):
+    assert extract.extract_image_urls_from_html(text=text) == expected

test_unstructured/cleaners/test_translate.py

@@ -0,0 +1,64 @@
+import os
+
+import pytest
+
+from unstructured.cleaners import translate
+
+IS_CI = os.getenv("CI") == "true"
+
+
+def test_get_opus_mt_model_name():
+    model_name = translate._get_opus_mt_model_name("ru", "en")
+    assert model_name == "Helsinki-NLP/opus-mt-ru-en"
+
+
+@pytest.mark.parametrize("code", ["way-too-long", "a", "", None])
+def test_validate_language_code(code):
+    with pytest.raises(ValueError):
+        translate._validate_language_code(code)
+
+
+def test_translate_returns_same_text_if_dest_is_same():
+    text = "This is already in English!"
+    assert translate.translate_text(text, "en", "en") == text
+
+
+def test_translate_returns_same_text_text_is_empty():
+    text = "      "
+    assert translate.translate_text(text) == text
+
+
+@pytest.mark.skipif(IS_CI, reason="Skipping this test in CI pipeline")
+def test_translate_with_language_specified():
+    text = "Ich bin ein Berliner!"
+    assert translate.translate_text(text, "de") == "I'm a Berliner!"
+
+
+@pytest.mark.skipif(IS_CI, reason="Skipping this test in CI pipeline")
+def test_translate_with_no_language_specified():
+    text = "Ich bin ein Berliner!"
+    assert translate.translate_text(text) == "I'm a Berliner!"
+
+
+@pytest.mark.skipif(IS_CI, reason="Skipping this test in CI pipeline")
+def test_translate_raises_with_bad_language():
+    text = "Ich bin ein Berliner!"
+    with pytest.raises(ValueError):
+        translate.translate_text(text, "zz")
+
+
+@pytest.mark.skipif(IS_CI, reason="Skipping this test in CI pipeline")
+def test_tranlate_works_with_russian():
+    text = "Я тоже можно переводать русский язык!"
+    assert translate.translate_text(text) == "I can also translate Russian!"
+
+
+@pytest.mark.skipif(IS_CI, reason="Skipping this test in CI pipeline")
+def test_translate_works_with_chinese():
+    text = "網站有中、英文版本"
+    translate.translate_text(text) == "Website available in Chinese and English"
+
+
+def translate_works_with_arabic():
+    text = "مرحباً بكم في متجرنا"
+    translate.translate_text(text) == "Welcome to our store."

test_unstructured/common/test_html_table.py

@@ -0,0 +1,301 @@
+# pyright: reportPrivateUsage=false
+
+"""Unit-test suite for the `unstructured.common.html_table` module."""
+
+from __future__ import annotations
+
+import pytest
+from lxml.html import fragment_fromstring
+
+from unstructured.common.html_table import (
+    HtmlCell,
+    HtmlRow,
+    HtmlTable,
+    htmlify_matrix_of_cell_texts,
+)
+
+
+class Describe_htmlify_matrix_of_cell_texts:
+    """Unit-test suite for `unstructured.common.html_table.htmlify_matrix_of_cell_texts()`."""
+
+    def test_htmlify_matrix_handles_empty_cells(self):
+        assert htmlify_matrix_of_cell_texts([["cell1", "", "cell3"], ["", "cell5", ""]]) == (
+            "<table>"
+            "<tr><td>cell1</td><td/><td>cell3</td></tr>"
+            "<tr><td/><td>cell5</td><td/></tr>"
+            "</table>"
+        )
+
+    def test_htmlify_matrix_handles_special_characters(self):
+        assert htmlify_matrix_of_cell_texts([['<>&"', "newline\n"]]) == (
+            "<table><tr><td>&lt;&gt;&amp;&quot;</td><td>newline<br/></td></tr></table>"
+        )
+
+    def test_htmlify_matrix_handles_multiple_rows_and_cells(self):
+        assert htmlify_matrix_of_cell_texts([["cell1", "cell2"], ["cell3", "cell4"]]) == (
+            "<table>"
+            "<tr><td>cell1</td><td>cell2</td></tr>"
+            "<tr><td>cell3</td><td>cell4</td></tr>"
+            "</table>"
+        )
+
+    def test_htmlify_matrix_handles_empty_matrix(self):
+        assert htmlify_matrix_of_cell_texts([]) == ""
+
+
+class DescribeHtmlTable:
+    """Unit-test suite for `unstructured.common.html_table.HtmlTable`."""
+
+    def it_can_construct_from_html_text(self):
+        html_table = HtmlTable.from_html_text("<table><tr><td>foobar</td></tr></table>")
+
+        assert isinstance(html_table, HtmlTable)
+        assert html_table._table.tag == "table"
+
+    @pytest.mark.parametrize(
+        "html_text",
+        [
+            "<table><tr><td>foobar</td></tr></table>",
+            "<body><table><tr><td>foobar</td></tr></table></body>",
+            "<html><body><table><tr><td>foobar</td></tr></table></body></html>",
+        ],
+    )
+    def it_can_find_a_table_wrapped_in_an_html_or_body_element(self, html_text: str):
+        html_table = HtmlTable.from_html_text(html_text)
+
+        assert isinstance(html_table, HtmlTable)
+        assert html_table._table.tag == "table"
+
+    def but_it_raises_when_no_table_element_is_present_in_the_html(self):
+        with pytest.raises(ValueError, match="`html_text` contains no `<table>` element"):
+            HtmlTable.from_html_text("<html><body><tr><td>foobar</td></tr></body></html>")
+
+    def it_removes_any_attributes_present_on_the_table_element(self):
+        html_table = HtmlTable.from_html_text(
+            '<table border="1", class="foobar"><tr><td>foobar</td></tr></table>',
+        )
+        assert html_table.html == "<table><tr><td>foobar</td></tr></table>"
+
+    def but_it_preserves_colspan_and_rowspan_as_structural_cell_attributes(self):
+        html_table = HtmlTable.from_html_text(
+            "<table>"
+            "<tr><th colspan='2' class='hdr' style='x'>A</th>"
+            "<th rowspan='2' id='foo'>B</th></tr>"
+            "<tr><td colspan='2' rowspan='3' data-k='v'>C</td><td>D</td></tr>"
+            "</table>"
+        )
+        table = fragment_fromstring(html_table.html)
+
+        # -- colspan/rowspan survive compactification --
+        assert table.xpath("./tr[1]/td[1]/@colspan") == ["2"]
+        assert table.xpath("./tr[1]/td[2]/@rowspan") == ["2"]
+        assert table.xpath("./tr[2]/td[1]/@colspan") == ["2"]
+        assert table.xpath("./tr[2]/td[1]/@rowspan") == ["3"]
+        # -- cosmetic / arbitrary attributes are still stripped --
+        assert table.xpath("./tr[1]/td[1]/@class") == []
+        assert table.xpath("./tr[1]/td[1]/@style") == []
+        assert table.xpath("./tr[1]/td[2]/@id") == []
+        assert table.xpath("./tr[2]/td[1]/@data-k") == []
+
+    @pytest.mark.parametrize(
+        "html_text",
+        [
+            "<table><thead><tr><td>foobar</td></tr></thead></table>",
+            "<table><thead><tr><td>foobar</td></tr></thead><tbody></tbody></table>",
+            "<table><tbody><tr><td>foobar</td></tr></tbody><tfoot></tfoot></table>",
+        ],
+    )
+    def it_removes_any_thead_tbody_or_tfoot_elements_present_within_the_table_element(
+        self, html_text: str
+    ):
+        html_table = HtmlTable.from_html_text(html_text)
+        assert html_table.html == "<table><tr><td>foobar</td></tr></table>"
+
+    def it_changes_any_th_elements_to_td_elements_for_cell_element_uniformity(self):
+        html_table = HtmlTable.from_html_text(
+            "<table>  <tr><th>a</th><th/><th>b</th></tr>  <tr><td/><td>c</td><td/></tr></table>"
+        )
+        assert html_table.html == (
+            "<table><tr><td>a</td><td/><td>b</td></tr><tr><td/><td>c</td><td/></tr></table>"
+        )
+
+    def it_removes_any_extra_whitespace_between_elements_and_normalizes_whitespace_in_text(self):
+        html_table = HtmlTable.from_html_text(
+            "\n  <table>\n  <tr>\n    <td>\tabc   def\nghi </td>\n  </tr>\n</table>\n  ",
+        )
+        assert html_table.html == "<table><tr><td>abc def ghi</td></tr></table>"
+
+    def it_preserves_tail_text_in_mixed_content_cells(self):
+        """Tail text (between inline children) carries real content and must not be dropped."""
+        html_table = HtmlTable.from_html_text(
+            "<table><tr>"
+            "<td><b>foo</b> bar <b>baz</b></td>"
+            "<td><b>x</b><br/>y<br/>z</td>"
+            "</tr></table>"
+        )
+        assert html_table.html == (
+            "<table><tr>"
+            "<td><b>foo</b> bar <b>baz</b></td>"
+            "<td><b>x</b><br/>y<br/>z</td>"
+            "</tr></table>"
+        )
+
+    def and_it_normalizes_whitespace_within_tail_text(self):
+        """Tail whitespace runs are collapsed, but leading/trailing spaces are kept."""
+        html_table = HtmlTable.from_html_text(
+            "<table><tr><td><b>a</b>   b\n  c  <b>d</b></td></tr></table>"
+        )
+        assert html_table.html == "<table><tr><td><b>a</b> b c <b>d</b></td></tr></table>"
+
+    def it_can_serialize_the_table_element_to_str_html_text(self):
+        table = fragment_fromstring("<table><tr><td>foobar</td></tr></table>")
+        html_table = HtmlTable(table)
+
+        assert html_table.html == "<table><tr><td>foobar</td></tr></table>"
+
+    def it_can_iterate_the_rows_in_the_table(self):
+        html_table = HtmlTable.from_html_text(
+            "<table>"
+            "  <tr><td>abc</td><td>def</td><td>ghi</td></tr>"
+            "  <tr><td>jkl</td><td>mno</td><td>pqr</td></tr>"
+            "  <tr><td>stu</td><td>vwx</td><td>yz</td></tr>"
+            "</table>"
+        )
+
+        row_iter = html_table.iter_rows()
+
+        row = next(row_iter)
+        assert isinstance(row, HtmlRow)
+        assert row.html == "<tr><td>abc</td><td>def</td><td>ghi</td></tr>"
+        # --
+        row = next(row_iter)
+        assert isinstance(row, HtmlRow)
+        assert row.html == "<tr><td>jkl</td><td>mno</td><td>pqr</td></tr>"
+        # --
+        row = next(row_iter)
+        assert isinstance(row, HtmlRow)
+        assert row.html == "<tr><td>stu</td><td>vwx</td><td>yz</td></tr>"
+        # --
+        with pytest.raises(StopIteration):
+            next(row_iter)
+
+    def it_preserves_row_header_semantics_when_iterating_rows(self):
+        html_table = HtmlTable.from_html_text(
+            "<table>"
+            "  <thead><tr><td>head-from-thead</td></tr></thead>"
+            "  <tbody>"
+            "    <tr><th>head-from-th</th></tr>"
+            "    <tr><td>body</td></tr>"
+            "  </tbody>"
+            "</table>"
+        )
+
+        assert [row.is_header for row in html_table.iter_rows()] == [True, True, False]
+
+    def and_it_preserves_source_row_html_before_compactification(self):
+        html_table = HtmlTable.from_html_text(
+            "<table>"
+            "  <thead><tr data-row='header'><th scope='col'>Header</th></tr></thead>"
+            "  <tbody><tr><td class='body-cell'>Body</td></tr></tbody>"
+            "</table>"
+        )
+        rows = list(html_table.iter_rows())
+        header_row = fragment_fromstring(rows[0].source_html or "<tr/>")
+        body_row = fragment_fromstring(rows[1].source_html or "<tr/>")
+
+        assert header_row.xpath("./@data-row") == ["header"]
+        assert header_row.xpath("./th/@scope") == ["col"]
+        assert body_row.xpath("./td/@class") == ["body-cell"]
+
+        # -- compactified row HTML contract remains unchanged --
+        assert rows[0].html == "<tr><td>Header</td></tr>"
+        assert rows[1].html == "<tr><td>Body</td></tr>"
+
+    def it_provides_access_to_the_clear_concatenated_text_of_the_table(self):
+        html_table = HtmlTable.from_html_text(
+            "<table>"
+            "  <tr><th> a\n b  c  </th><th/><th>def</th></tr>"
+            "  <tr><td>gh \ti</td><td/><td>\n jk l </td></tr>"
+            "  <tr><td/><td> m n op\n</td><td/></tr>"
+            "</table>"
+        )
+        assert html_table.text == "a b c def gh i jk l m n op"
+
+
+class DescribeHtmlRow:
+    """Unit-test suite for `unstructured.common.html_table.HtmlRow`."""
+
+    def it_can_serialize_the_row_to_html(self):
+        assert HtmlRow(fragment_fromstring("<tr><td>a</td><td>b</td><td/></tr>")).html == (
+            "<tr><td>a</td><td>b</td><td/></tr>"
+        )
+
+    def it_can_iterate_the_cells_in_the_row(self):
+        row = HtmlRow(fragment_fromstring("<tr><td>a</td><td>b</td><td/></tr>"))
+
+        cell_iter = row.iter_cells()
+
+        cell = next(cell_iter)
+        assert isinstance(cell, HtmlCell)
+        assert cell.html == "<td>a</td>"
+        # --
+        cell = next(cell_iter)
+        assert isinstance(cell, HtmlCell)
+        assert cell.html == "<td>b</td>"
+        # --
+        cell = next(cell_iter)
+        assert isinstance(cell, HtmlCell)
+        assert cell.html == "<td/>"
+        # --
+        with pytest.raises(StopIteration):
+            next(cell_iter)
+
+    def it_can_iterate_the_texts_of_the_cells_in_the_row(self):
+        row = HtmlRow(fragment_fromstring("<tr><td>a</td><td>b</td><td/></tr>"))
+
+        text_iter = row.iter_cell_texts()
+
+        assert next(text_iter) == "a"
+        assert next(text_iter) == "b"
+        with pytest.raises(StopIteration):
+            next(text_iter)
+
+    def and_it_includes_descendant_inline_text_in_cell_texts(self):
+        row = HtmlRow(
+            fragment_fromstring(
+                "<tr>"
+                "<td>ID</td>"
+                "<td><a href='#'>Category Link</a></td>"
+                "<td><span>  Extra  spacing </span></td>"
+                "</tr>"
+            )
+        )
+
+        assert list(row.iter_cell_texts()) == ["ID", "Category Link", "Extra spacing"]
+
+    def it_knows_when_it_represents_a_header_row(self):
+        assert HtmlRow(fragment_fromstring("<tr><td>a</td></tr>")).is_header is False
+        assert HtmlRow(fragment_fromstring("<tr><td>a</td></tr>"), is_header=True).is_header is True
+
+
+class DescribeHtmlCell:
+    """Unit-test suite for `unstructured.common.html_table.HtmlCell`."""
+
+    def it_can_serialize_the_cell_to_html(self):
+        assert HtmlCell(fragment_fromstring("<td>a b c</td>")).html == "<td>a b c</td>"
+
+    def and_it_preserves_nested_markup_when_serializing_nonempty_cells(self):
+        assert HtmlCell(fragment_fromstring("<td><a href='#'>Category Link</a></td>")).html == (
+            '<td><a href="#">Category Link</a></td>'
+        )
+
+    @pytest.mark.parametrize(
+        ("cell_html", "expected_value"),
+        [
+            ("<td>  Lorem ipsum  </td>", "Lorem ipsum"),
+            ("<td><a href='#'>Category Link</a></td>", "Category Link"),
+            ("<td/>", ""),
+        ],
+    )
+    def it_knows_the_text_in_the_cell(self, cell_html: str, expected_value: str):
+        assert HtmlCell(fragment_fromstring(cell_html)).text == expected_value

test_unstructured/documents/html_files/example.html

@@ -0,0 +1,55 @@
+<body class="Document">
+ <div class="Page" data-page-number="1">
+  <header class="Header">
+   <h1 class="Title">
+    Header
+   </h1>
+   <time class="CalendarDate">
+    Date: October 30, 2023
+   </time>
+  </header>
+  <form class="Form">
+   <label class="FormField" for="company-name">
+    From field name
+   </label>
+   <input class="FormFieldValue" value="Example value"/>
+  </form>
+  <section class="Section">
+   <table class="Table">
+    <thead class="TableHeader">
+     <tr class="TableRow">
+      <th class="TableCellHeader">
+       Description
+      </th>
+      <th class="TableCellHeader">
+       Row header
+      </th>
+     </tr>
+    </thead>
+    <tbody class="TableBody">
+     <tr class="TableRow">
+      <td class="TableCell">
+       Value description
+      </td>
+      <td class="TableCell">
+       <span class="Currency">
+        50 $
+       </span>
+       <span class="Measurement">
+        (1.32 %)
+       </span>
+      </td>
+     </tr>
+    </tbody>
+   </table>
+  </section>
+  <section class="Section">
+   <h2 class="Subtitle">
+    2. Subtitle
+   </h2>
+   <p class="NarrativeText">
+    Paragraph text
+   </p>
+  </section>
+ </div>
+</body>

test_unstructured/documents/html_files/example_with_alternative_text.html

@@ -0,0 +1,8 @@
+<body class="Document">
+ <div class="Page" data-page-number="1">
+   <header class="Header">
+    <img alt="New York logo" class="Logo"/>
+    <img alt="A line graph showing the comparison of 5 year cumulative total return for stocks" class="Image"/>
+   </header>
+ </div>
+</body>

test_unstructured/documents/html_files/example_with_inline_fields.html

@@ -0,0 +1,18 @@
+<body class="Document">
+ <div class="Page" data-page-number="1">
+  <header class="Header">
+   <p class="NarrativeText">
+    Table of Contents
+   </p>
+   <address class="Address">
+    68 Prince Street Palmdale, CA 93550
+   </address>
+   <a class="Hyperlink">
+    www.google.com
+   </a>
+   <span class="UncategorizedText">
+    More text
+   </span>
+  </header>
+ </div>
+</body>

test_unstructured/documents/html_files/three_tables.html

@@ -0,0 +1,149 @@
+<body class="Document">
+ <table class="Table">
+  <tr>
+   <th>
+    Header 1
+   </th>
+   <th>
+    Header 2
+   </th>
+  </tr>
+  <tr>
+   <td>
+    Row 1, Cell 1
+   </td>
+   <td>
+    Row 1, Cell 2
+   </td>
+  </tr>
+  <tr>
+   <td>
+    Row 2, Cell 1
+   </td>
+   <td>
+    Row 2, Cell 2
+   </td>
+  </tr>
+ </table>
+ <table>
+  <tr>
+   <th colspan="3">
+    Big Table Header
+   </th>
+  </tr>
+  <tr>
+   <td rowspan="2">
+    Merged Cell 1
+   </td>
+   <td>
+    Cell 2
+   </td>
+   <td>
+    Cell 3
+   </td>
+  </tr>
+  <tr>
+   <td colspan="2">
+    Merged Cell 4 and 5
+   </td>
+  </tr>
+  <tr>
+   <td>
+    Cell 6
+   </td>
+   <td>
+    Cell 7
+   </td>
+   <td>
+    Cell 8
+   </td>
+  </tr>
+  <tr>
+   <td>
+    Cell 9
+   </td>
+   <td colspan="2">
+    A cell with a lot of text. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.
+   </td>
+  </tr>
+  <tr>
+   <td>
+    Cell 10
+   </td>
+   <td>
+    Cell 11
+   </td>
+   <td>
+    Cell 12
+   </td>
+  </tr>
+ </table>
+ <table class="TableOfContents">
+  <tr>
+   <th>
+    Chapter
+   </th>
+   <th>
+    Title
+   </th>
+   <th>
+    Page
+   </th>
+  </tr>
+  <tr>
+   <td>
+    1
+   </td>
+   <td>
+    Introduction
+   </td>
+   <td>
+    1
+   </td>
+  </tr>
+  <tr>
+   <td>
+    2
+   </td>
+   <td>
+    Getting Started
+   </td>
+   <td>
+    5
+   </td>
+  </tr>
+  <tr>
+   <td>
+    3
+   </td>
+   <td>
+    Basic Concepts
+   </td>
+   <td>
+    12
+   </td>
+  </tr>
+  <tr>
+   <td>
+    4
+   </td>
+   <td>
+    Advanced Topics
+   </td>
+   <td>
+    25
+   </td>
+  </tr>
+  <tr>
+   <td>
+    5
+   </td>
+   <td>
+    Conclusion
+   </td>
+   <td>
+    40
+   </td>
+  </tr>
+ </table>
+</body>

test_unstructured/documents/test_coordinates.py

@@ -0,0 +1,73 @@
+import pytest
+
+from unstructured.documents.coordinates import (
+    CoordinateSystem,
+    Orientation,
+    RelativeCoordinateSystem,
+    convert_coordinate,
+)
+
+
+@pytest.mark.parametrize(
+    ("old_t", "old_t_max", "new_t_max", "t_orientation", "expected"),
+    [(0, 7, 5, 1, 0), (7, 7, 5, 1, 5), (0, 7, 5, -1, 5), (7, 7, 5, -1, 0)],
+)
+def test_convert_coordinate(old_t, old_t_max, new_t_max, t_orientation, expected):
+    assert convert_coordinate(old_t, old_t_max, new_t_max, t_orientation) == expected
+
+
+@pytest.mark.parametrize(
+    ("width", "height", "orientation", "x", "y", "expected_x", "expected_y"),
+    [
+        (100, 300, Orientation.CARTESIAN, 0.8, 0.4, 80, 120),
+        (100, 300, Orientation.SCREEN, 0.8, 0.6, 80, 120),
+    ],
+)
+def test_convert_from_relative(width, height, orientation, x, y, expected_x, expected_y):
+    coord1 = CoordinateSystem(width, height)
+    coord1.orientation = orientation
+    assert coord1.convert_from_relative(x, y) == (expected_x, expected_y)
+
+
+@pytest.mark.parametrize(
+    ("width", "height", "orientation", "x", "y", "expected_x", "expected_y"),
+    [
+        (100, 300, Orientation.CARTESIAN, 80, 120, 0.8, 0.4),
+        (100, 300, Orientation.SCREEN, 80, 120, 0.8, 0.6),
+    ],
+)
+def test_convert_to_relative(width, height, orientation, x, y, expected_x, expected_y):
+    coord1 = CoordinateSystem(width, height)
+    coord1.orientation = orientation
+    assert coord1.convert_to_relative(x, y) == (expected_x, expected_y)
+
+
+@pytest.mark.parametrize(
+    ("orientation1", "orientation2", "x", "y", "expected_x", "expected_y"),
+    [
+        (Orientation.CARTESIAN, Orientation.CARTESIAN, 80, 120, 800, 1200),
+        (Orientation.CARTESIAN, Orientation.SCREEN, 80, 120, 800, 800),
+        (Orientation.SCREEN, Orientation.CARTESIAN, 80, 120, 800, 800),
+        (Orientation.SCREEN, Orientation.SCREEN, 80, 120, 800, 1200),
+    ],
+)
+def test_convert_to_new_system(orientation1, orientation2, x, y, expected_x, expected_y):
+    coord1 = CoordinateSystem(width=100, height=200)
+    coord1.orientation = orientation1
+    coord2 = CoordinateSystem(width=1000, height=2000)
+    coord2.orientation = orientation2
+    assert coord1.convert_coordinates_to_new_system(coord2, x, y) == (expected_x, expected_y)
+
+
+@pytest.mark.parametrize(
+    ("width", "height", "orientation", "x", "y", "expected_x", "expected_y"),
+    [
+        (100, 300, Orientation.CARTESIAN, 80, 120, 0.8, 0.4),
+        (100, 300, Orientation.SCREEN, 80, 120, 0.8, 0.6),
+    ],
+)
+def test_relative_system(width, height, orientation, x, y, expected_x, expected_y):
+    coord1 = CoordinateSystem(width, height)
+    coord1.orientation = orientation
+    coord2 = RelativeCoordinateSystem()
+    assert coord1.convert_coordinates_to_new_system(coord2, x, y) == (expected_x, expected_y)

test_unstructured/documents/test_elements.py

@@ -0,0 +1,776 @@
+# pyright: reportPrivateUsage=false
+
+"""Test-suite for `unstructured.documents.elements` module."""
+
+from __future__ import annotations
+
+import copy
+import io
+import json
+import pathlib
+from functools import partial
+
+import pytest
+
+from test_unstructured.unit_utils import assign_hash_ids, example_doc_path
+from unstructured.cleaners.core import clean_bullets, clean_prefix
+from unstructured.documents.coordinates import (
+    CoordinateSystem,
+    Orientation,
+    RelativeCoordinateSystem,
+)
+from unstructured.documents.elements import (
+    CheckBox,
+    ConsolidationStrategy,
+    CoordinatesMetadata,
+    DataSourceMetadata,
+    Element,
+    ElementMetadata,
+    Points,
+    Text,
+    Title,
+    assign_and_map_hash_ids,
+)
+from unstructured.partition.json import partition_json
+from unstructured.staging.base import elements_from_base64_gzipped_json
+
+
+@pytest.mark.parametrize("element", [Element(), Text(text=""), CheckBox()])
+def test_Element_autoassigns_a_UUID_then_becomes_an_idempotent_and_deterministic_hash(
+    element: Element,
+):
+    # -- element self-assigns itself a UUID --
+    assert isinstance(element.id, str)
+    assert len(element.id) == 36
+    assert element.id.count("-") == 4
+
+    expected_hash = "5336294a19f32ff03ef80066fbc3e0f7"
+    # -- calling `.id_to_hash()` changes the element's id-type to hash --
+    assert element.id_to_hash(0) == expected_hash
+    assert element.id == expected_hash
+
+    # -- `.id_to_hash()` is idempotent --
+    assert element.id_to_hash(0) == expected_hash
+    assert element.id == expected_hash
+
+
+def test_Text_is_JSON_serializable():
+    # -- This shold run without an error --
+    json.dumps(Text(text="hello there!", element_id=None).to_dict())
+
+
+@pytest.mark.parametrize(
+    "element",
+    [
+        Element(),
+        Text(text=""),  # -- element_id should be implicitly None --
+        Text(text="", element_id=None),  # -- setting explicitly to None --
+        CheckBox(),
+    ],
+)
+def test_Element_self_assigns_itself_a_UUID_id(element: Element):
+    assert isinstance(element.id, str)
+    assert len(element.id) == 36
+    assert element.id.count("-") == 4
+
+
+def test_text_element_apply_cleaners():
+    text_element = Text(text="[1] A Textbook on Crocodile Habitats")
+
+    text_element.apply(partial(clean_prefix, pattern=r"\[\d{1,2}\]"))
+    assert str(text_element) == "A Textbook on Crocodile Habitats"
+
+
+def test_text_element_apply_multiple_cleaners():
+    cleaners = [partial(clean_prefix, pattern=r"\[\d{1,2}\]"), partial(clean_bullets)]
+    text_element = Text(text="[1] \u2022 A Textbook on Crocodile Habitats")
+    text_element.apply(*cleaners)
+    assert str(text_element) == "A Textbook on Crocodile Habitats"
+
+
+def test_non_text_elements_are_serializable_to_text():
+    element = CheckBox()
+    assert hasattr(element, "text")
+    assert element.text is not None
+    assert element.text == ""
+    assert str(element) == ""
+
+
+def test_apply_raises_if_func_does_not_produce_string():
+    def bad_cleaner(s: str):
+        return 1
+
+    text_element = Text(text="[1] A Textbook on Crocodile Habitats")
+
+    with pytest.raises(ValueError, match="Cleaner produced a non-string output."):
+        text_element.apply(bad_cleaner)  # pyright: ignore[reportArgumentType]
+
+
+@pytest.mark.parametrize(
+    ("coordinates", "orientation1", "orientation2", "expected_coords"),
+    [
+        (
+            ((1, 2), (1, 4), (3, 4), (3, 2)),
+            Orientation.CARTESIAN,
+            Orientation.CARTESIAN,
+            ((10, 20), (10, 40), (30, 40), (30, 20)),
+        ),
+        (
+            ((1, 2), (1, 4), (3, 4), (3, 2)),
+            Orientation.CARTESIAN,
+            Orientation.SCREEN,
+            ((10, 1980), (10, 1960), (30, 1960), (30, 1980)),
+        ),
+        (
+            ((1, 2), (1, 4), (3, 4), (3, 2)),
+            Orientation.SCREEN,
+            Orientation.CARTESIAN,
+            ((10, 1980), (10, 1960), (30, 1960), (30, 1980)),
+        ),
+        (
+            ((1, 2), (1, 4), (3, 4), (3, 2)),
+            Orientation.SCREEN,
+            Orientation.SCREEN,
+            ((10, 20), (10, 40), (30, 40), (30, 20)),
+        ),
+    ],
+)
+def test_convert_coordinates_to_new_system(
+    coordinates: Points,
+    orientation1: Orientation,
+    orientation2: Orientation,
+    expected_coords: Points,
+):
+    coord1 = CoordinateSystem(100, 200)
+    coord1.orientation = orientation1
+    coord2 = CoordinateSystem(1000, 2000)
+    coord2.orientation = orientation2
+    element = Element(coordinates=coordinates, coordinate_system=coord1)
+
+    new_coords = element.convert_coordinates_to_new_system(coord2)
+
+    assert new_coords is not None
+    for new_coord, expected in zip(new_coords, expected_coords):
+        assert new_coord == pytest.approx(expected)  # pyright: ignore[reportUnknownMemberType]
+    element.convert_coordinates_to_new_system(coord2, in_place=True)
+    assert element.metadata.coordinates is not None
+    assert element.metadata.coordinates.points is not None
+    for new_coord, expected in zip(element.metadata.coordinates.points, expected_coords):
+        assert new_coord == pytest.approx(expected)  # pyright: ignore[reportUnknownMemberType]
+    assert element.metadata.coordinates.system == coord2
+
+
+def test_convert_coordinate_to_new_system_none():
+    element = Element(coordinates=None, coordinate_system=None)
+    coord = CoordinateSystem(100, 200)
+    coord.orientation = Orientation.SCREEN
+    assert element.convert_coordinates_to_new_system(coord) is None
+
+
+def test_element_constructor_coordinates_all_present():
+    coordinates = ((1, 2), (1, 4), (3, 4), (3, 2))
+    coordinate_system = RelativeCoordinateSystem()
+    element = Element(coordinates=coordinates, coordinate_system=coordinate_system)
+    expected_coordinates_metadata = CoordinatesMetadata(
+        points=coordinates,
+        system=coordinate_system,
+    )
+    assert element.metadata.coordinates == expected_coordinates_metadata
+
+
+def test_element_constructor_coordinates_points_absent():
+    with pytest.raises(ValueError) as exc_info:
+        Element(coordinate_system=RelativeCoordinateSystem())
+    assert (
+        str(exc_info.value)
+        == "Coordinates points should not exist without coordinates system and vice versa."
+    )
+
+
+def test_element_constructor_coordinates_system_absent():
+    with pytest.raises(ValueError) as exc_info:
+        Element(coordinates=((1, 2), (1, 4), (3, 4), (3, 2)))
+    assert (
+        str(exc_info.value)
+        == "Coordinates points should not exist without coordinates system and vice versa."
+    )
+
+
+def test_coordinate_metadata_serdes():
+    coordinates = ((1, 2), (1, 4), (3, 4), (3, 2))
+    coordinate_system = RelativeCoordinateSystem()
+    coordinates_metadata = CoordinatesMetadata(points=coordinates, system=coordinate_system)
+    expected_schema = {
+        "layout_height": 1,
+        "layout_width": 1,
+        "points": ((1, 2), (1, 4), (3, 4), (3, 2)),
+        "system": "RelativeCoordinateSystem",
+    }
+    coordinates_metadata_dict = coordinates_metadata.to_dict()
+    assert coordinates_metadata_dict == expected_schema
+    assert CoordinatesMetadata.from_dict(coordinates_metadata_dict) == coordinates_metadata
+
+
+def test_element_to_dict():
+    coordinates = ((1, 2), (1, 4), (3, 4), (3, 2))
+    coordinate_system = RelativeCoordinateSystem()
+    element = Element(
+        element_id="awt32t1",
+        coordinates=coordinates,
+        coordinate_system=coordinate_system,
+    )
+
+    assert element.to_dict() == {
+        "metadata": {
+            "coordinates": {
+                "layout_height": 1,
+                "layout_width": 1,
+                "points": ((1, 2), (1, 4), (3, 4), (3, 2)),
+                "system": "RelativeCoordinateSystem",
+            },
+        },
+        "type": None,
+        "text": "",
+        "element_id": "awt32t1",
+    }
+
+
+class DescribeElementMetadata:
+    """Unit-test suite for `unstructured.documents.elements.ElementMetadata`."""
+
+    # -- It can be constructed with known keyword arguments. In particular, including a non-known
+    # -- keyword argument produces a type-error at development time and raises an exception at
+    # -- runtime. This catches typos before they reach production.
+
+    def it_detects_unknown_constructor_args_at_both_development_time_and_runtime(self):
+        with pytest.raises(TypeError, match="got an unexpected keyword argument 'file_name'"):
+            ElementMetadata(file_name="memo.docx")  # pyright: ignore[reportCallIssue]
+
+    @pytest.mark.parametrize(
+        "file_path",
+        [
+            pathlib.Path("documents/docx") / "memos" / "memo-2023-11-10.docx",
+            "documents/docx/memos/memo-2023-11-10.docx",
+        ],
+    )
+    def it_accommodates_either_a_pathlib_Path_or_str_for_its_filename_arg(
+        self, file_path: pathlib.Path | str
+    ):
+        meta = ElementMetadata(filename=file_path)
+
+        assert meta.file_directory == "documents/docx/memos"
+        assert meta.filename == "memo-2023-11-10.docx"
+
+    def it_leaves_both_filename_and_file_directory_None_when_neither_is_specified(self):
+        meta = ElementMetadata()
+
+        assert meta.file_directory is None
+        assert meta.filename is None
+
+    @pytest.mark.parametrize("file_path", [pathlib.Path("memo.docx"), "memo.docx"])
+    def and_it_leaves_file_directory_None_when_not_specified_and_filename_is_not_a_path(
+        self, file_path: pathlib.Path | str
+    ):
+        meta = ElementMetadata(filename=file_path)
+
+        assert meta.file_directory is None
+        assert meta.filename == "memo.docx"
+
+    def and_it_splits_off_directory_path_from_its_filename_arg_when_it_is_a_file_path(self):
+        meta = ElementMetadata(filename="documents/docx/memo-2023-11-11.docx")
+
+        assert meta.file_directory == "documents/docx"
+        assert meta.filename == "memo-2023-11-11.docx"
+
+    def but_it_prefers_a_specified_file_directory_when_filename_also_contains_a_path(self):
+        meta = ElementMetadata(filename="tmp/staging/memo.docx", file_directory="documents/docx")
+
+        assert meta.file_directory == "documents/docx"
+        assert meta.filename == "memo.docx"
+
+    # -- It knows the types of its known members so type-checking support is available. --
+
+    def it_knows_the_types_of_its_known_members_so_type_checking_support_is_available(self):
+        ElementMetadata(
+            category_depth="2",  # pyright: ignore[reportArgumentType]
+            file_directory=True,  # pyright: ignore[reportArgumentType]
+            text_as_html=42,  # pyright: ignore[reportArgumentType]
+        )
+        # -- it does not check types at runtime however (choosing to avoid validation overhead) --
+
+    # -- It only stores a field's value when it is not None. --
+
+    def it_returns_the_value_of_an_attribute_it_has(self):
+        meta = ElementMetadata(url="https://google.com")
+        assert "url" in meta.__dict__
+        assert meta.url == "https://google.com"
+
+    def and_it_returns_None_for_a_known_attribute_it_does_not_have(self):
+        meta = ElementMetadata()
+        assert "url" not in meta.__dict__
+        assert meta.url is None
+
+    def but_it_raises_AttributeError_for_an_unknown_attribute_it_does_not_have(self):
+        meta = ElementMetadata()
+        assert "coefficient" not in meta.__dict__
+        with pytest.raises(AttributeError, match="object has no attribute 'coefficient'"):
+            meta.coefficient
+
+    def it_stores_a_non_None_field_value_when_assigned(self):
+        meta = ElementMetadata()
+        assert "file_directory" not in meta.__dict__
+        meta.file_directory = "tmp/"
+        assert "file_directory" in meta.__dict__
+        assert meta.file_directory == "tmp/"
+
+    def it_removes_a_field_when_None_is_assigned_to_it(self):
+        meta = ElementMetadata(file_directory="tmp/")
+        assert "file_directory" in meta.__dict__
+        assert meta.file_directory == "tmp/"
+
+        meta.file_directory = None
+        assert "file_directory" not in meta.__dict__
+        assert meta.file_directory is None
+
+    # -- It can serialize itself to a dict -------------------------------------------------------
+
+    def it_can_serialize_itself_to_a_dict(self):
+        meta = ElementMetadata(
+            category_depth=1,
+            file_directory="tmp/",
+            page_number=2,
+            text_as_html="<table></table>",
+            url="https://google.com",
+        )
+        assert meta.to_dict() == {
+            "category_depth": 1,
+            "file_directory": "tmp/",
+            "page_number": 2,
+            "text_as_html": "<table></table>",
+            "url": "https://google.com",
+        }
+
+    def and_it_serializes_a_coordinates_sub_object_to_a_dict_when_it_is_present(self):
+        meta = ElementMetadata(
+            category_depth=1,
+            coordinates=CoordinatesMetadata(
+                points=((2, 2), (1, 4), (3, 4), (3, 2)),
+                system=RelativeCoordinateSystem(),
+            ),
+            page_number=2,
+        )
+        assert meta.to_dict() == {
+            "category_depth": 1,
+            "coordinates": {
+                "layout_height": 1,
+                "layout_width": 1,
+                "points": ((2, 2), (1, 4), (3, 4), (3, 2)),
+                "system": "RelativeCoordinateSystem",
+            },
+            "page_number": 2,
+        }
+
+    def and_it_serializes_a_data_source_sub_object_to_a_dict_when_it_is_present(self):
+        meta = ElementMetadata(
+            category_depth=1,
+            data_source=DataSourceMetadata(
+                url="https://www.nih.gov/about-nih/who-we-are/nih-director",
+                date_created="2023-11-09",
+            ),
+            page_number=2,
+        )
+        assert meta.to_dict() == {
+            "category_depth": 1,
+            "data_source": {
+                "url": "https://www.nih.gov/about-nih/who-we-are/nih-director",
+                "date_created": "2023-11-09",
+            },
+            "page_number": 2,
+        }
+
+    def and_it_round_trips_an_enrichment_origins_dict_of_lists_through_a_dict(self):
+        enrichment_origins = {
+            "text": [
+                {"type": "enrichment_foo", "provider": "provider_a", "model": "model_x"},
+                {"type": "enrichment_bar", "provider": "provider_a", "model": "model_x"},
+            ],
+            "embeddings": [
+                {"type": "enrichment_baz", "provider": "provider_b", "model": "model_y"},
+            ],
+        }
+        meta = ElementMetadata(enrichment_origins=enrichment_origins)
+
+        # -- it serializes verbatim (no special-casing needed) --
+        assert meta.to_dict()["enrichment_origins"] == enrichment_origins
+        # -- and rehydrates to an equal value --
+        assert ElementMetadata.from_dict(meta.to_dict()).enrichment_origins == enrichment_origins
+
+    def and_it_serializes_an_orig_elements_sub_object_to_base64_when_it_is_present(self):
+        elements = assign_hash_ids([Title("Lorem"), Text("Lorem Ipsum")])
+        meta = ElementMetadata(
+            category_depth=1,
+            orig_elements=elements,
+            page_number=2,
+        )
+
+        meta_dict = meta.to_dict()
+
+        assert meta_dict["category_depth"] == 1
+        assert meta_dict["page_number"] == 2
+        # Verify the orig_elements value is a base64 string that round-trips correctly.
+        # We don't compare the exact compressed bytes because zlib output varies across
+        # implementations (e.g. standard zlib vs zlib-ng).
+        assert isinstance(meta_dict["orig_elements"], str)
+        restored = elements_from_base64_gzipped_json(meta_dict["orig_elements"])
+        assert len(restored) == 2
+        assert restored[0].text == "Lorem"
+        assert restored[1].text == "Lorem Ipsum"
+
+    def but_unlike_in_ElementMetadata_unknown_fields_in_sub_objects_are_ignored(self):
+        """Metadata sub-objects ignore fields they do not explicitly define.
+
+        This is _not_ the case for ElementMetadata itself where an non-known field is welcomed as a
+        user-defined ad-hoc metadata field.
+        """
+        element_metadata = {
+            "new_field": "hello",
+            "data_source": {
+                "new_field": "world",
+            },
+            "coordinates": {
+                "new_field": "foo",
+            },
+        }
+
+        metadata = ElementMetadata.from_dict(element_metadata)
+        metadata_dict = metadata.to_dict()
+
+        assert "new_field" in metadata_dict
+        assert "new_field" not in metadata_dict["coordinates"]
+        assert "new_field" not in metadata_dict["data_source"]
+
+    # -- It can deserialize itself from a dict ---------------------------------------------------
+
+    def it_can_deserialize_itself_from_a_dict(self):
+        meta_dict = {
+            "category_depth": 1,
+            "coefficient": 0.58,
+            "coordinates": {
+                "layout_height": 4,
+                "layout_width": 2,
+                "points": ((1, 2), (1, 4), (3, 4), (3, 2)),
+                "system": "RelativeCoordinateSystem",
+            },
+            "data_source": {
+                "url": "https://www.nih.gov/about-nih/who-we-are/nih-director",
+                "date_created": "2023-11-09",
+            },
+            "languages": ["eng"],
+        }
+
+        meta = ElementMetadata.from_dict(meta_dict)
+
+        # -- known fields present in dict are present in meta --
+        assert meta.category_depth == 1
+
+        # -- known sub-object fields present in dict are present in meta --
+        assert meta.coordinates == CoordinatesMetadata(
+            points=((1, 2), (1, 4), (3, 4), (3, 2)),
+            system=RelativeCoordinateSystem(),
+        )
+        assert meta.data_source == DataSourceMetadata(
+            url="https://www.nih.gov/about-nih/who-we-are/nih-director",
+            date_created="2023-11-09",
+        )
+
+        # -- known fields absent from dict report None but are not present in meta --
+        assert meta.file_directory is None
+        assert "file_directory" not in meta.__dict__
+
+        # -- non-known fields present in dict are present in meta (we have no way to tell whether
+        # -- they are "ad-hoc" or not because we lack indication of user-intent)
+        assert meta.coefficient == 0.58
+
+        # -- ad-hoc fields absent from dict raise on attempted access --
+        with pytest.raises(AttributeError, match="ntMetadata' object has no attribute 'quotient'"):
+            meta.quotient
+
+        # -- but that can be worked around by end-user --
+        assert (meta.quotient if hasattr(meta, "quotient") else None) is None
+
+        # -- mutating a mutable (collection) field does not affect the original value --
+        assert isinstance(meta.languages, list)
+        assert meta.languages == ["eng"]
+        meta.languages.append("spa")
+        assert meta.languages == ["eng", "spa"]
+        assert meta_dict["languages"] == ["eng"]
+
+    # -- It allows downstream users to add an arbitrary new member by assignment. ----------------
+
+    def it_allows_an_end_user_to_add_an_arbitrary_field(self):
+        meta = ElementMetadata()
+        meta.foobar = 7
+        assert "foobar" in meta.__dict__
+        assert meta.foobar == 7
+
+    def and_fields_so_added_appear_in_the_metadata_JSON(self):
+        meta = ElementMetadata()
+        meta.foobar = 7
+        assert meta.to_dict() == {"foobar": 7}
+
+    def and_it_removes_an_end_user_field_when_it_is_assigned_None(self):
+        meta = ElementMetadata()
+        meta.foobar = 7
+        assert "foobar" in meta.__dict__
+        meta.foobar = None
+        assert "foobar" not in meta.__dict__
+        with pytest.raises(
+            AttributeError, match="'ElementMetadata' object has no attribute 'foobar'"
+        ):
+            meta.foobar
+
+    # -- It can update itself from another instance ----------------------------------------------
+
+    def it_can_update_itself_from_another_instance(self):
+        meta = ElementMetadata(category_depth=1, page_number=1)
+        meta.coefficient = 0.58
+        meta.stem_length = 18
+        other = ElementMetadata(file_directory="tmp/", page_number=2)
+        other.quotient = 1.4
+        other.stem_length = 20
+
+        meta.update(other)
+
+        # -- known-fields present on self but not other are unchanged --
+        assert meta.category_depth == 1
+        # -- known-fields present on other but not self are added --
+        assert meta.file_directory == "tmp/"
+        # -- known-fields present on both self and other are updated --
+        assert meta.page_number == 2
+        # -- ad-hoc-fields present on self but not other are unchanged --
+        assert meta.coefficient == 0.58
+        # -- ad-hoc-fields present on other but not self are added --
+        assert meta.quotient == 1.4
+        # -- ad-hoc-fields present on both self and other are updated --
+        assert meta.stem_length == 20
+        # -- other is left unchanged --
+        assert other.category_depth is None
+        assert other.file_directory == "tmp/"
+        assert other.page_number == 2
+        assert other.text_as_html is None
+        assert other.url is None
+        assert other.quotient == 1.4
+        assert other.stem_length == 20
+        with pytest.raises(AttributeError, match="etadata' object has no attribute 'coefficient'"):
+            other.coefficient
+
+    def but_it_raises_on_attempt_to_update_from_a_non_ElementMetadata_object(self):
+        meta = ElementMetadata()
+        with pytest.raises(ValueError, match=r"ate\(\)' must be an instance of 'ElementMetadata'"):
+            meta.update({"coefficient": "0.56"})  # pyright: ignore[reportArgumentType]
+
+    # -- It knows when it is equal to another instance -------------------------------------------
+
+    def it_is_equal_to_another_instance_with_the_same_known_field_values(self):
+        meta = ElementMetadata(
+            category_depth=1,
+            coordinates=CoordinatesMetadata(
+                points=((1, 2), (1, 4), (3, 4), (3, 2)),
+                system=RelativeCoordinateSystem(),
+            ),
+            data_source=DataSourceMetadata(
+                url="https://www.nih.gov/about-nih/who-we-are/nih-director",
+                date_created="2023-11-08",
+            ),
+            file_directory="tmp/",
+            languages=["eng"],
+            page_number=2,
+            text_as_html="<table></table>",
+            url="https://google.com",
+        )
+        assert meta == ElementMetadata(
+            category_depth=1,
+            coordinates=CoordinatesMetadata(
+                points=((1, 2), (1, 4), (3, 4), (3, 2)),
+                system=RelativeCoordinateSystem(),
+            ),
+            data_source=DataSourceMetadata(
+                url="https://www.nih.gov/about-nih/who-we-are/nih-director",
+                date_created="2023-11-08",
+            ),
+            file_directory="tmp/",
+            languages=["eng"],
+            page_number=2,
+            text_as_html="<table></table>",
+            url="https://google.com",
+        )
+
+    def but_it_is_never_equal_to_a_non_ElementMetadata_object(self):
+        class NotElementMetadata:
+            pass
+
+        meta = ElementMetadata()
+        other = NotElementMetadata()
+
+        # -- all the "fields" are the same --
+        assert meta.__dict__ == other.__dict__
+        # -- but it is rejected solely because its type is different --
+        assert meta != other
+
+    def it_is_equal_to_another_instance_with_the_same_ad_hoc_field_values(self):
+        meta = ElementMetadata(category_depth=1)
+        meta.coefficient = 0.58
+        other = ElementMetadata(category_depth=1)
+        other.coefficient = 0.58
+
+        assert meta == other
+
+    def but_it_is_not_equal_to_an_instance_with_ad_hoc_fields_that_differ(self):
+        meta = ElementMetadata(category_depth=1)
+        meta.coefficient = 0.58
+        other = ElementMetadata(category_depth=1)
+        other.coefficient = 0.72
+
+        assert meta != other
+
+    def it_is_not_equal_when_a_list_field_contains_different_items(self):
+        meta = ElementMetadata(languages=["eng"])
+        assert meta != ElementMetadata(languages=["eng", "spa"])
+
+    def and_it_is_not_equal_when_the_coordinates_sub_object_field_differs(self):
+        meta = ElementMetadata(
+            coordinates=CoordinatesMetadata(
+                points=((1, 2), (1, 4), (3, 4), (3, 2)),
+                system=RelativeCoordinateSystem(),
+            )
+        )
+        assert meta != ElementMetadata(
+            coordinates=CoordinatesMetadata(
+                points=((2, 2), (2, 4), (3, 4), (4, 2)),
+                system=RelativeCoordinateSystem(),
+            )
+        )
+
+    def and_it_is_not_equal_when_the_data_source_sub_object_field_differs(self):
+        meta = ElementMetadata(
+            data_source=DataSourceMetadata(
+                url="https://www.nih.gov/about-nih/who-we-are/nih-director",
+                date_created="2023-11-08",
+            )
+        )
+        assert meta != ElementMetadata(
+            data_source=DataSourceMetadata(
+                url="https://www.nih.gov/about-nih/who-we-are/nih-director",
+                date_created="2023-11-09",
+            )
+        )
+
+    # -- There is a consolidation-strategy for all known fields ----------------------------------
+
+    def it_can_find_the_consolidation_strategy_for_each_of_its_known_fields(self):
+        metadata = ElementMetadata()
+        metadata_field_names = sorted(metadata._known_field_names)
+        consolidation_strategies = ConsolidationStrategy.field_consolidation_strategies()
+
+        for field_name in metadata_field_names:
+            assert field_name in consolidation_strategies, (
+                f"ElementMetadata field `.{field_name}` does not have a consolidation strategy."
+                f" Add one in `ConsolidationStrategy.field_consolidation_strategies()."
+            )
+
+
+def test_hash_ids_are_unique_for_duplicate_elements():
+    # GIVEN
+    parent = Text(text="Parent", metadata=ElementMetadata(page_number=1))
+    elements: list[Element] = [
+        parent,
+        Text(text="Element", metadata=ElementMetadata(page_number=1, parent_id=parent.id)),
+        Text(text="Element", metadata=ElementMetadata(page_number=1, parent_id=parent.id)),
+    ]
+
+    # WHEN
+    updated_elements = assign_and_map_hash_ids(copy.deepcopy(elements))
+    ids = [element.id for element in updated_elements]
+
+    # THEN
+    assert len(ids) == len(set(ids)), "Recalculated IDs must be unique."
+    assert elements[1].metadata.parent_id == elements[2].metadata.parent_id
+
+    for idx, updated_element in enumerate(updated_elements):
+        assert updated_element.id != elements[idx].id, "IDs haven't changed after recalculation"
+        if updated_element.metadata.parent_id is not None:
+            assert updated_element.metadata.parent_id in ids, "Parent ID not in the list of IDs"
+            assert updated_element.metadata.parent_id != elements[idx].metadata.parent_id, (
+                "Parent ID hasn't changed after recalculation"
+            )
+
+
+def test_hash_ids_can_handle_duplicated_element_instances():
+    # GIVEN
+    parent = Text(text="Parent", metadata=ElementMetadata(page_number=1))
+    element = Text(text="Element", metadata=ElementMetadata(page_number=1, parent_id=parent.id))
+    elements: list[Element] = [parent, element, element]
+
+    # WHEN
+    updated_elements = assign_and_map_hash_ids(copy.deepcopy(elements))
+    ids = [element.id for element in updated_elements]
+
+    # THEN
+    assert len(ids) == len(set(ids)) + 1, "One element is duplicated so uniques should be one less."
+    assert elements[1].metadata.parent_id == elements[2].metadata.parent_id
+
+
+def test_hash_ids_are_deterministic():
+    parent = Text(text="Parent", metadata=ElementMetadata(page_number=1))
+    elements: list[Element] = [
+        parent,
+        Text(text="Element", metadata=ElementMetadata(page_number=1, parent_id=parent.id)),
+        Text(text="Element", metadata=ElementMetadata(page_number=1, parent_id=parent.id)),
+    ]
+
+    updated_elements = assign_and_map_hash_ids(elements)
+    ids = [element.id for element in updated_elements]
+    parent_ids = [element.metadata.parent_id for element in updated_elements]
+
+    assert ids == [
+        "ea9eb7e80383c190f8cafce1ad666624",
+        "4112a8d24886276e18e759d06956021b",
+        "eba84bbe7f03e8b91a1527323040ee3d",
+    ]
+    assert parent_ids == [
+        None,
+        "ea9eb7e80383c190f8cafce1ad666624",
+        "ea9eb7e80383c190f8cafce1ad666624",
+    ]
+
+
+@pytest.mark.parametrize(
+    ("text", "sequence_number", "filename", "page_number", "expected_hash"),
+    [
+        # -- pdf files support page numbers --
+        ("foo", 1, "foo.pdf", 1, "4bb264eb23ceb44cd8fcc5af44f8dc71"),
+        ("foo", 2, "foo.pdf", 1, "75fc1de48cf724ec00aa8d1c5a0d3758"),
+        # -- txt files don't have a page number --
+        ("some text", 0, "some.txt", None, "1a2627b5760c06b1440102f11a1edb0f"),
+        ("some text", 1, "some.txt", None, "e3fd10d867c4a1c0264dde40e3d7e45a"),
+    ],
+)
+def test_id_to_hash_calculates(
+    text: str, sequence_number: int, filename: str, page_number: int | None, expected_hash: str
+):
+    element = Text(
+        text=text,
+        metadata=ElementMetadata(filename=filename, page_number=page_number),
+    )
+    assert element.id_to_hash(sequence_number) == expected_hash, "Returned ID does not match"
+    assert element.id == expected_hash, "ID should be set"
+
+
+def test_formskeysvalues_reads_saves():
+    filename = example_doc_path("test_evaluate_files/unstructured_output/form.json")
+    as_read = partition_json(filename=filename)
+    tmp_file = io.StringIO()
+    json.dump([element.to_dict() for element in as_read], tmp_file)
+    tmp_file.seek(0)
+    as_read_2 = partition_json(file=tmp_file)  # type: ignore[arg-type]
+    assert as_read == as_read_2

test_unstructured/documents/test_html_sanitization.py

@@ -0,0 +1,278 @@
+"""Unit tests for the shared HTML output-sanitization policy (GHSA-v5mq-3xhg-98m9)."""
+
+import pytest
+
+from unstructured.documents.html_sanitization import (
+    ALLOWED_URL_SCHEMES,
+    is_event_handler_attribute,
+    is_safe_tag,
+    is_safe_url,
+    sanitize_attributes,
+    sanitize_html_fragment,
+    sanitize_style_attribute,
+)
+
+
+class DescribeIsSafeUrl:
+    @pytest.mark.parametrize(
+        "url",
+        [
+            "http://example.com",
+            "https://example.com/path?q=1",
+            "mailto:user@example.com",
+            "tel:+15551234",
+            "/relative/path",
+            "relative/path",
+            "#anchor",
+            "?query=only",
+        ],
+    )
+    def it_allows_safe_urls(self, url: str):
+        assert is_safe_url(url) is True
+
+    @pytest.mark.parametrize(
+        "url",
+        [
+            "javascript:alert(1)",
+            "JaVaScRiPt:alert(1)",
+            "  javascript:alert(1)",
+            "java\tscript:alert(1)",
+            "java\nscript:alert(1)",
+            "vbscript:msgbox(1)",
+            "data:text/html,<script>alert(1)</script>",
+            "data:application/javascript,alert(1)",
+            "file:///etc/passwd",
+        ],
+    )
+    def it_rejects_dangerous_urls(self, url: str):
+        assert is_safe_url(url) is False
+
+    def it_only_permits_raster_image_data_uris_on_img_src(self):
+        # -- data: is in the scheme allowlist but narrowed to raster img[src] so
+        # -- base64 images survive while script-executable data URIs do not --
+        assert "data" in ALLOWED_URL_SCHEMES
+        assert (
+            is_safe_url(
+                "data:image/gif;base64,R0lGOD",
+                tag_name="img",
+                attribute_name="src",
+            )
+            is True
+        )
+        assert (
+            is_safe_url(
+                "data:image/svg+xml;base64,PHN2Zz4=",
+                tag_name="img",
+                attribute_name="src",
+            )
+            is False
+        )
+        assert (
+            is_safe_url(
+                "data:image/png;base64,iVBORw0KGgo=",
+                tag_name="a",
+                attribute_name="href",
+            )
+            is False
+        )
+        assert (
+            is_safe_url(
+                "data:text/html;base64,PHNjcmlwdD4=",
+                tag_name="img",
+                attribute_name="src",
+            )
+            is False
+        )
+
+
+class DescribeIsEventHandlerAttribute:
+    @pytest.mark.parametrize("name", ["onerror", "onload", "onmouseover", "ONCLICK", " onfocus"])
+    def it_detects_event_handlers(self, name: str):
+        assert is_event_handler_attribute(name) is True
+
+    @pytest.mark.parametrize("name", ["class", "href", "id", "data-src", "title"])
+    def it_ignores_non_event_handlers(self, name: str):
+        assert is_event_handler_attribute(name) is False
+
+
+class DescribeIsSafeTag:
+    @pytest.mark.parametrize("tag", ["div", "p", "a", "img", "table", "TD", "svg"])
+    def it_allows_known_tags(self, tag: str):
+        assert is_safe_tag(tag) is True
+
+    @pytest.mark.parametrize("tag", ["script", "iframe", "object", "embed", "", None])
+    def it_rejects_unknown_tags(self, tag):
+        assert is_safe_tag(tag) is False
+
+
+class DescribeSanitizeAttributes:
+    def it_drops_event_handler_attributes(self):
+        result = sanitize_attributes(
+            {"onerror": "alert(1)", "onmouseover": "x", "class": "Foo"},
+            tag_name="p",
+        )
+        assert result == {"class": "Foo"}
+
+    def it_drops_url_attributes_with_unsafe_schemes(self):
+        result = sanitize_attributes({"href": "javascript:alert(1)", "id": "x"}, tag_name="a")
+        assert result == {"id": "x"}
+
+    def it_keeps_url_attributes_with_safe_schemes(self):
+        link_result = sanitize_attributes({"href": "https://example.com"}, tag_name="a")
+        image_result = sanitize_attributes({"src": "/img.png"}, tag_name="img")
+        assert link_result == {"href": "https://example.com"}
+        assert image_result == {"src": "/img.png"}
+
+    def it_keeps_raster_data_image_uris_on_img_src(self):
+        result = sanitize_attributes({"src": "data:image/png;base64,AAAA"}, tag_name="img")
+        assert result == {"src": "data:image/png;base64,AAAA"}
+
+    def it_drops_data_image_uris_outside_img_src(self):
+        result = sanitize_attributes({"href": "data:image/png;base64,AAAA"}, tag_name="a")
+        assert result == {}
+
+    def it_drops_svg_data_image_uris(self):
+        result = sanitize_attributes({"src": "data:image/svg+xml;base64,PHN2Zz4="}, tag_name="img")
+        assert result == {}
+
+    def it_drops_malformed_attribute_names(self):
+        # -- a name that isn't a valid HTML attribute name can't be emitted safely --
+        result = sanitize_attributes({'x"><svg onload=alert(1)>': "y", "id": "ok"}, tag_name="p")
+        assert result == {"id": "ok"}
+
+    def it_drops_attributes_not_allowed_on_the_tag(self):
+        result = sanitize_attributes(
+            {
+                "action": "https://example.com/submit",
+                "class": "Form",
+                "formaction": "https://example.com/button",
+                "http-equiv": "refresh",
+                "srcset": "https://example.com/a.png 1x",
+            },
+            tag_name="form",
+        )
+        assert result == {"class": "Form"}
+
+    def it_allows_tag_specific_attributes(self):
+        result = sanitize_attributes(
+            {"colspan": "2", "rowspan": "3", "headers": "h1", "scope": "col"},
+            tag_name="td",
+        )
+        assert result == {"colspan": "2", "rowspan": "3", "headers": "h1", "scope": "col"}
+
+    def it_filters_inline_style_values(self):
+        result = sanitize_attributes(
+            {
+                "style": (
+                    "background-color: lightblue; position: fixed; inset: 0; "
+                    "z-index: 9999; border: 1px solid black"
+                )
+            },
+            tag_name="p",
+        )
+        assert result == {"style": "background-color: lightblue; border: 1px solid black"}
+
+    def it_adds_noopener_noreferrer_for_blank_targets(self):
+        result = sanitize_attributes(
+            {"href": "https://example.com", "target": "_blank"},
+            tag_name="a",
+        )
+        assert result == {
+            "href": "https://example.com",
+            "target": "_blank",
+            "rel": "noopener noreferrer",
+        }
+
+    def it_does_not_html_escape_values(self):
+        # -- escaping happens once, at emit time; values pass through untouched here --
+        result = sanitize_attributes({"title": 'a & b < c "d"'}, tag_name="p")
+        assert result == {"title": 'a & b < c "d"'}
+
+    def it_scheme_filters_list_valued_url_attributes(self):
+        result = sanitize_attributes({"href": ["javascript:alert(1)"]}, tag_name="a")
+        assert result == {}
+
+
+class DescribeSanitizeStyleAttribute:
+    def it_keeps_safe_presentation_declarations(self):
+        style = "background-color: lightblue; text-align: right; border-collapse: collapse"
+        assert sanitize_style_attribute(style) == style
+
+    def it_drops_layout_overlay_declarations(self):
+        style = "position: fixed; inset: 0; z-index: 9999; color: red"
+        assert sanitize_style_attribute(style) == "color: red"
+
+    def it_drops_url_and_expression_values(self):
+        style = "background-color: url(javascript:alert(1)); color: expression(alert(1))"
+        assert sanitize_style_attribute(style) == ""
+
+
+class DescribeSanitizeHtmlFragment:
+    def it_strips_event_handlers(self):
+        assert "onerror" not in sanitize_html_fragment('<img src="x" onerror="alert(1)">')
+        assert "onmouseover" not in sanitize_html_fragment('<p onmouseover="alert(1)">hi</p>')
+
+    def it_strips_javascript_hrefs(self):
+        assert "javascript:" not in sanitize_html_fragment('<a href="javascript:alert(1)">x</a>')
+
+    def it_removes_disallowed_tags(self):
+        cleaned = sanitize_html_fragment("<script>alert(1)</script><p>ok</p>")
+        assert "<script" not in cleaned
+        assert "<p>" in cleaned or "ok" in cleaned
+
+    def it_preserves_base64_image_sources(self):
+        cleaned = sanitize_html_fragment('<img src="data:image/png;base64,iVBORw0KGgo=" alt="ok">')
+        assert "data:image/png" in cleaned
+
+    def it_preserves_avif_base64_image_sources(self):
+        cleaned = sanitize_html_fragment('<img src="data:image/avif;base64,AAAAAA==" alt="ok">')
+        assert "data:image/avif" in cleaned
+
+    def it_drops_svg_data_image_sources(self):
+        cleaned = sanitize_html_fragment(
+            '<img src="data:image/svg+xml;base64,PHN2Zz48L3N2Zz4=" alt="bad">'
+        )
+        assert "data:image/svg" not in cleaned
+
+    def it_drops_data_image_links(self):
+        cleaned = sanitize_html_fragment('<a href="data:image/png;base64,AAAA">x</a>')
+        assert "data:image/png" not in cleaned
+
+    def it_drops_non_image_data_uris(self):
+        cleaned = sanitize_html_fragment('<img src="data:text/html,<script>alert(1)</script>">')
+        assert "data:text/html" not in cleaned
+        assert "<script" not in cleaned
+
+    def it_adds_noopener_noreferrer_to_links(self):
+        cleaned = sanitize_html_fragment('<a href="https://example.com" target="_blank">x</a>')
+        assert "noopener" in cleaned
+        assert "noreferrer" in cleaned
+
+    def it_does_not_add_rel_to_same_tab_links(self):
+        # -- same-tab links must keep their Referer header (no unconditional rel) --
+        cleaned = sanitize_html_fragment('<a href="https://example.com">x</a>')
+        assert "rel=" not in cleaned
+        assert "noreferrer" not in cleaned
+
+    def it_does_not_duplicate_rel_on_blank_target_links(self):
+        cleaned = sanitize_html_fragment(
+            '<a href="https://example.com" target="_blank" rel="noopener noreferrer">x</a>'
+        )
+        assert cleaned.count("rel=") == 1
+
+    def it_filters_inline_styles(self):
+        cleaned = sanitize_html_fragment(
+            '<p style="position:fixed;inset:0;z-index:9999;color:red">x</p>'
+        )
+        assert "position" not in cleaned
+        assert "inset" not in cleaned
+        assert "z-index" not in cleaned
+        assert "color:red" in cleaned
+
+    def it_preserves_legitimate_table_formatting(self):
+        cleaned = sanitize_html_fragment(
+            '<table><tr><td colspan="2" style="border: 1px solid black;">A</td></tr></table>'
+        )
+        assert "<table" in cleaned
+        assert 'colspan="2"' in cleaned
+        assert "border" in cleaned

test_unstructured/documents/test_mappings.py

@@ -0,0 +1,46 @@
+from collections import defaultdict
+from typing import Type
+
+from unstructured.documents import elements, ontology
+from unstructured.documents.mappings import (
+    ALL_ONTOLOGY_ELEMENT_TYPES,
+    HTML_TAG_TO_DEFAULT_ELEMENT_TYPE_MAP,
+    ONTOLOGY_CLASS_TO_UNSTRUCTURED_ELEMENT_TYPE,
+    get_all_subclasses,
+)
+from unstructured.documents.ontology import OntologyElement
+
+
+def test_if_all_html_tags_have_default_ontology_type():
+    html_tag_to_possible_ontology_classes: dict[str, list[Type[ontology.OntologyElement]]] = (
+        defaultdict(list)
+    )
+
+    for ontology_class in ALL_ONTOLOGY_ELEMENT_TYPES:
+        for tag in ontology_class().allowed_tags:
+            html_tag_to_possible_ontology_classes[tag].append(ontology_class)
+
+    for html_tag, possible_ontology_classes in html_tag_to_possible_ontology_classes.items():
+        assert html_tag in HTML_TAG_TO_DEFAULT_ELEMENT_TYPE_MAP
+        assert HTML_TAG_TO_DEFAULT_ELEMENT_TYPE_MAP[html_tag] in possible_ontology_classes + [
+            ontology.UncategorizedText
+        ]  # In some cases it is better to use unknown type than assign incorrect type
+
+
+def test_all_expected_ontology_types_are_subclasses_of_OntologyElement():
+    for element_type in HTML_TAG_TO_DEFAULT_ELEMENT_TYPE_MAP.values():
+        assert issubclass(element_type, OntologyElement)
+
+
+def test_ontology_to_unstructured_mapping_has_valid_types():
+    for (
+        ontology_element,
+        unstructured_element,
+    ) in ONTOLOGY_CLASS_TO_UNSTRUCTURED_ELEMENT_TYPE.items():
+        assert issubclass(unstructured_element, elements.Element)
+        assert issubclass(ontology_element, ontology.OntologyElement)
+
+
+def test_all_ontology_elements_are_defined_in_mapping_to_unstructured():
+    for ontology_element in get_all_subclasses(ontology.OntologyElement):
+        assert ontology_element in ONTOLOGY_CLASS_TO_UNSTRUCTURED_ELEMENT_TYPE

test_unstructured/documents/test_ontology_to_unstructured_parsing.py

@@ -0,0 +1,513 @@
+from pathlib import Path
+
+import pytest
+from bs4 import BeautifulSoup
+
+from unstructured.chunking.basic import chunk_elements
+from unstructured.chunking.title import chunk_by_title
+from unstructured.documents.elements import ElementMetadata, Text
+from unstructured.documents.ontology import (
+    Column,
+    Document,
+    Heading,
+    Hyperlink,
+    Image,
+    Page,
+    Paragraph,
+    Section,
+    Subtitle,
+    Table,
+    Title,
+    remove_ids_and_class_from_table,
+)
+from unstructured.embed.openai import OpenAIEmbeddingConfig, OpenAIEmbeddingEncoder
+from unstructured.partition.html import partition_html
+from unstructured.partition.html.transformations import (
+    can_unstructured_elements_be_merged,
+    ontology_to_unstructured_elements,
+    parse_html_to_ontology,
+)
+from unstructured.partition.json import partition_json
+from unstructured.staging.base import elements_from_json
+
+
+def test_remove_ids_and_class_from_table():
+    html_text = """
+    <table>
+        <tr class="TableRow">
+            <td><img class="Signature" alt="cell 1"/></td>
+            <td>cell 2</td>
+        </tr>
+        <tr>
+            <td><IMG class="Signature" alt="cell 3"/></td>
+            <td>cell 4</td>
+        </tr>
+        <tr>
+            <td><input class="Checkbox" type="checkbox"/></td>
+            <td>Option 1</td>
+        </tr>
+    </table>
+    """
+    soup = BeautifulSoup(html_text, "html.parser")
+    assert (
+        str(remove_ids_and_class_from_table(soup))
+        == """
+<table>
+<tr>
+<td><img alt="cell 1" class="Signature"/></td>
+<td>cell 2</td>
+</tr>
+<tr>
+<td><img alt="cell 3" class="Signature"/></td>
+<td>cell 4</td>
+</tr>
+<tr>
+<td><input class="Checkbox" type="checkbox"/></td>
+<td>Option 1</td>
+</tr>
+</table>
+"""
+    )
+
+
+def test_page_number_is_passed_correctly():
+    ontology = Document(
+        children=[
+            Page(
+                children=[Paragraph(text="Paragraph1")],
+                additional_attributes={"data-page-number": "1"},
+            ),
+            Page(
+                children=[Paragraph(text="Paragraph2")],
+                additional_attributes={"data-page-number": "2"},
+            ),
+        ]
+    )
+    unstructured_elements = ontology_to_unstructured_elements(ontology)
+    page1, p1, page2, p2 = unstructured_elements
+    assert p1.metadata.page_number == 1
+    assert p2.metadata.page_number == 2
+
+
+def test_invalid_page_number_is_not_passed():
+    ontology = Document(
+        children=[
+            Page(
+                children=[Paragraph(text="Paragraph1")],
+                additional_attributes={"data-page-number": "invalid"},
+            )
+        ]
+    )
+    unstructured_elements = ontology_to_unstructured_elements(ontology)
+    page1, p1 = unstructured_elements
+    assert not p1.metadata.page_number
+
+
+def test_category_depth_is_not_derived_from_layout_nesting():
+    """ML-1328: `category_depth` reflects heading level, not DOM/layout nesting.
+
+    Layout containers (Page/Column) and non-heading content carry no `category_depth`, so a
+    paragraph reads the same whether it sits in a single-column page or inside a column of a
+    two-column page (i.e. depth does not change solely due to multi-column layout).
+    """
+    ontology = Document(
+        children=[
+            Page(children=[Paragraph(text="Paragraph1")]),
+            Page(
+                children=[
+                    Column(children=[Paragraph(text="Paragraph2")]),
+                    Column(children=[Paragraph(text="Paragraph3")]),
+                ]
+            ),
+        ]
+    )
+
+    unstructured_elements = ontology_to_unstructured_elements(ontology)
+    page1, p1, page2, c1, p2, c2, p3 = unstructured_elements
+
+    # -- layout containers are not headings -> no depth --
+    assert page1.metadata.category_depth is None
+    assert page2.metadata.category_depth is None
+    assert c1.metadata.category_depth is None
+    assert c2.metadata.category_depth is None
+
+    # -- plain paragraphs are not headings -> no depth, regardless of single- vs multi-column --
+    assert p1.metadata.category_depth is None
+    assert p2.metadata.category_depth is None
+    assert p3.metadata.category_depth is None
+
+
+def test_category_depth_is_derived_from_heading_level():
+    """ML-1328: heading elements get `category_depth` from their HTML heading level."""
+    ontology = Document(
+        children=[
+            Page(
+                children=[
+                    Title(text="Section"),  # <h1> -> 0
+                    Subtitle(text="Subsection"),  # <h2> -> 1
+                    Heading(text="Sub-subsection", html_tag_name="h3"),  # <h3> -> 2
+                    Paragraph(text="Body text"),  # not a heading -> None
+                ]
+            ),
+        ]
+    )
+
+    unstructured_elements = ontology_to_unstructured_elements(ontology)
+    _page, title, subtitle, heading, body = unstructured_elements
+
+    assert title.metadata.category_depth == 0
+    assert subtitle.metadata.category_depth == 1
+    assert heading.metadata.category_depth == 2
+    assert body.metadata.category_depth is None
+
+
+def test_partition_html_v2_assigns_heading_based_parent_id():
+    """ML-1328 (AC #3): partition_html(v2) yields section->subsection parent_id.
+
+    Hierarchy is assigned by the `@apply_metadata` decorator from the heading-level
+    `category_depth`, exactly as for every other partitioner -- the v2 converter only
+    sets `category_depth`, it does not run `set_element_hierarchy` itself. Both production
+    callers (unstructured and the VLM partitioner) go through `partition_html`, so the
+    decorator always runs; this exercises that real path end to end.
+    """
+    html = (
+        '<div class="Page">'
+        '<h1 class="Title">Section A</h1>'
+        '<p class="NarrativeText">intro body under A</p>'
+        '<h2 class="Subtitle">Sub A1</h2>'
+        '<p class="NarrativeText">body under A1</p>'
+        '<h3 class="Heading">Sub A1a</h3>'
+        '<p class="NarrativeText">body under A1a</p>'
+        '<h2 class="Subtitle">Sub A2</h2>'
+        '<p class="NarrativeText">body under A2</p>'
+        "</div>"
+    )
+
+    elements = partition_html(text=html, html_parser_version="v2")
+    by_id = {e.id: e for e in elements}
+    by_text = {e.text: e for e in elements if e.text}
+
+    def parent_of(text):
+        pid = by_text[text].metadata.parent_id
+        return by_id.get(pid) if pid else None
+
+    # top-level heading has no heading ancestor
+    assert parent_of("Section A") is None
+    # content + subsections are parented to their enclosing heading
+    assert parent_of("intro body under A") is by_text["Section A"]
+    assert parent_of("Sub A1") is by_text["Section A"]
+    assert parent_of("body under A1") is by_text["Sub A1"]
+    assert parent_of("Sub A1a") is by_text["Sub A1"]
+    assert parent_of("body under A1a") is by_text["Sub A1a"]
+    # a sibling subsection resets back to its section, not the deeper preceding heading
+    assert parent_of("Sub A2") is by_text["Section A"]
+    assert parent_of("body under A2") is by_text["Sub A2"]
+
+
+def test_converter_leaves_content_parent_id_for_the_metadata_layer():
+    """ML-1328 contract (abstraction boundary): called directly, the converter sets
+    `category_depth` but leaves content `parent_id=None` -- hierarchy is the
+    `@apply_metadata` layer's job, as for every other partitioner. This documents the
+    intended boundary (a direct caller does NOT get heading-based parent_id) and guards
+    against silently reintroducing a self-sufficient hierarchy pass in the converter.
+    """
+    ontology = Document(
+        children=[
+            Page(
+                children=[
+                    Title(text="Section A"),  # h1 -> category_depth 0
+                    Paragraph(text="intro body"),
+                    Subtitle(text="Sub A1"),  # h2 -> category_depth 1
+                ]
+            ),
+        ]
+    )
+
+    page, section_a, intro, sub_a1 = ontology_to_unstructured_elements(ontology)
+
+    # layout container keeps its tree parent (physical structure preserved)
+    assert page.metadata.parent_id is not None
+    # content carries heading-level category_depth ...
+    assert section_a.metadata.category_depth == 0
+    assert sub_a1.metadata.category_depth == 1
+    # ... but no parent_id yet -- the decorator assigns the heading-based parent downstream
+    assert section_a.metadata.parent_id is None
+    assert intro.metadata.parent_id is None
+    assert sub_a1.metadata.parent_id is None
+
+
+def test_chunking_is_applied_on_elements():
+    ontology = Document(
+        children=[
+            Page(children=[Paragraph(text="Paragraph1")]),
+            Page(
+                children=[
+                    Column(children=[Paragraph(text="Paragraph2")]),
+                    Column(children=[Paragraph(text="Paragraph3")]),
+                ]
+            ),
+        ]
+    )
+
+    unstructured_elements = ontology_to_unstructured_elements(ontology)
+
+    chunked_basic = chunk_elements(unstructured_elements)
+    assert str(chunked_basic[0]) == "Paragraph1\n\nParagraph2\n\nParagraph3"
+    chunked_by_title = chunk_by_title(unstructured_elements)
+    assert str(chunked_by_title[0]) == "Paragraph1\n\nParagraph2\n\nParagraph3"
+
+
+def test_embeddings_are_applied_on_elements(mocker):
+    ontology = Document(
+        children=[
+            Page(children=[Paragraph(text="Paragraph1")]),
+            Page(
+                children=[
+                    Column(children=[Paragraph(text="Paragraph2")]),
+                    Column(children=[Paragraph(text="Paragraph3")]),
+                ]
+            ),
+        ]
+    )
+
+    unstructured_elements = ontology_to_unstructured_elements(ontology)
+    # Mocked client with the desired behavior for embed_documents
+    mock_client = mocker.MagicMock()
+    mock_client.embed_documents.return_value = [1, 2, 3, 4, 5, 6, 7]
+
+    # Mock get_client to return our mock_client
+    mocker.patch.object(OpenAIEmbeddingConfig, "get_client", return_value=mock_client)
+
+    encoder = OpenAIEmbeddingEncoder(config=OpenAIEmbeddingConfig(api_key="api_key"))
+    elements = encoder.embed_documents(
+        elements=unstructured_elements,
+    )
+
+    assert len(elements) == 7
+
+    page1, p1, page2, c1, p2, c2, p3 = elements
+
+    assert p1.embeddings == 2
+    assert p2.embeddings == 5
+    assert p3.embeddings == 7
+
+
+@pytest.mark.parametrize(
+    ("html_file_path", "json_file_path"),
+    [
+        ("html_files/example.html", "unstructured_json_output/example.json"),
+    ],
+)
+def test_ingest(html_file_path, json_file_path):
+    html_file_path = Path(__file__).parent / html_file_path
+    json_file_path = Path(__file__).parent / json_file_path
+
+    html_code = html_file_path.read_text()
+    expected_json_elements = elements_from_json(str(json_file_path))
+
+    ontology = parse_html_to_ontology(html_code)
+    unstructured_elements = ontology_to_unstructured_elements(ontology)
+    assert unstructured_elements == expected_json_elements
+
+
+@pytest.mark.parametrize("json_file_path", ["unstructured_json_output/example.json"])
+def test_parsed_ontology_can_be_serialized_from_json(json_file_path):
+    json_file_path = Path(__file__).parent / json_file_path
+
+    expected_json_elements = elements_from_json(str(json_file_path))
+
+    json_elements_text = json_file_path.read_text()
+    elements = partition_json(text=json_elements_text)
+
+    assert len(elements) == len(expected_json_elements)
+    for i in range(len(elements)):
+        assert elements[i] == expected_json_elements[i]
+        # The partitioning output comes from PDF file, so only stem is compared
+        # as the suffix is different .pdf != .json
+        assert Path(elements[i].metadata.filename).stem == json_file_path.stem
+
+
+@pytest.mark.parametrize(
+    ("html_file_path", "json_file_path"),
+    [
+        ("html_files/example.html", "unstructured_json_output/example.json"),
+        ("html_files/example_full_doc.html", "unstructured_json_output/example_full_doc.json"),
+        (
+            "html_files/example_with_alternative_text.html",
+            "unstructured_json_output/example_with_alternative_text.json",
+        ),
+        ("html_files/three_tables.html", "unstructured_json_output/three_tables.json"),
+        (
+            "html_files/example_with_inline_fields.html",
+            "unstructured_json_output/example_with_inline_fields.json",
+        ),
+    ],
+)
+def test_parsed_ontology_can_be_serialized_from_html(html_file_path, json_file_path):
+    html_file_path = Path(__file__).parent / html_file_path
+    json_file_path = Path(__file__).parent / json_file_path
+    expected_json_elements = elements_from_json(str(json_file_path))
+    html_code = html_file_path.read_text()
+
+    predicted_elements = partition_html(
+        text=html_code, html_parser_version="v2", unique_element_ids=True
+    )
+
+    assert len(expected_json_elements) == len(predicted_elements)
+
+    for i in range(len(expected_json_elements)):
+        assert expected_json_elements[i] == predicted_elements[i]
+        assert (
+            expected_json_elements[i].metadata.text_as_html
+            == predicted_elements[i].metadata.text_as_html
+        )
+
+
+def test_inline_elements_are_squeezed():
+    ontology = Document(
+        children=[
+            Page(
+                children=[
+                    Hyperlink(text="Hyperlink1"),
+                    Hyperlink(text="Hyperlink2"),
+                    Hyperlink(text="Hyperlink3"),
+                ],
+            )
+        ]
+    )
+    unstructured_elements = ontology_to_unstructured_elements(ontology)
+    assert len(unstructured_elements) == 2
+
+    page, text1 = unstructured_elements
+    assert text1.text == "Hyperlink1 Hyperlink2 Hyperlink3"
+
+
+def test_text_elements_are_squeezed():
+    ontology = Document(
+        children=[
+            Page(
+                children=[
+                    Paragraph(text="Paragraph1"),
+                    Paragraph(text="Paragraph2"),
+                ],
+            )
+        ]
+    )
+    unstructured_elements = ontology_to_unstructured_elements(ontology)
+    assert len(unstructured_elements) == 2
+
+    page, text1 = unstructured_elements
+    assert text1.text == "Paragraph1 Paragraph2"
+
+
+def test_inline_elements_are_squeezed_when_image():
+    ontology = Document(
+        children=[
+            Page(
+                children=[
+                    Paragraph(text="Paragraph1"),
+                    Hyperlink(text="Hyperlink1"),
+                    Image(text="Image1"),
+                    Hyperlink(text="Hyperlink2"),
+                    Hyperlink(text="Hyperlink3"),
+                    Paragraph(text="Paragraph2"),
+                    Paragraph(text="Paragraph3"),
+                ],
+            )
+        ]
+    )
+    unstructured_elements = ontology_to_unstructured_elements(ontology)
+    assert len(unstructured_elements) == 4
+
+    page, text1, image, text2 = unstructured_elements
+    assert text1.text == "Paragraph1 Hyperlink1"
+    assert text2.text == "Hyperlink2 Hyperlink3 Paragraph2 Paragraph3"
+
+    assert '<a class="Hyperlink"' in text1.metadata.text_as_html
+    assert '<p class="Paragraph"' in text1.metadata.text_as_html
+
+    assert '<a class="Hyperlink"' in text2.metadata.text_as_html
+    assert '<p class="Paragraph"' in text2.metadata.text_as_html
+
+
+def test_inline_elements_are_squeezed_when_table():
+    ontology = Document(
+        children=[
+            Page(
+                children=[
+                    Hyperlink(text="Hyperlink1"),
+                    Paragraph(text="Paragraph1"),
+                    Paragraph(text="Paragraph2"),
+                    Table(text="Table1"),
+                    Paragraph(text="Paragraph2"),
+                    Hyperlink(text="Hyperlink2"),
+                    Hyperlink(text="Hyperlink3"),
+                ],
+            )
+        ]
+    )
+    unstructured_elements = ontology_to_unstructured_elements(ontology)
+    assert len(unstructured_elements) == 4
+
+    page, text1, table1, text3 = unstructured_elements
+    assert text1.text == "Hyperlink1 Paragraph1 Paragraph2"
+    assert table1.text == "Table1"
+    assert text3.text == "Paragraph2 Hyperlink2 Hyperlink3"
+
+
+def test_inline_elements_are_on_many_depths():
+    ontology = Document(
+        children=[
+            Page(
+                children=[
+                    Hyperlink(text="Hyperlink1"),
+                    Paragraph(text="Paragraph1"),
+                    Section(
+                        children=[
+                            Section(
+                                children=[
+                                    Hyperlink(text="Hyperlink2"),
+                                    Hyperlink(text="Hyperlink3"),
+                                ]
+                            ),
+                            Paragraph(text="Paragraph2"),
+                            Hyperlink(text="Hyperlink4"),
+                        ]
+                    ),
+                ],
+            )
+        ]
+    )
+    unstructured_elements = ontology_to_unstructured_elements(ontology)
+
+    assert len(unstructured_elements) == 6
+
+    page, text1, section1, section2, text2, text3 = unstructured_elements
+
+    assert text1.text == "Hyperlink1 Paragraph1"
+    assert text2.text == "Hyperlink2 Hyperlink3"
+    assert text3.text == "Paragraph2 Hyperlink4"
+
+
+def _inline_element(text: str) -> Text:
+    """A childless inline (Hyperlink) element -- mergeable on the content rules alone."""
+    html = f'<a class="Hyperlink">{text}</a>'
+    return Text(text=text, metadata=ElementMetadata(text_as_html=html))
+
+
+def test_inline_elements_at_the_same_nesting_depth_can_be_merged():
+    # ML-1328: "same level in the HTML tree" is the DOM-nesting depth. Two childless inline
+    # elements sitting at the same depth are eligible to be merged into one element.
+    first, second = _inline_element("a"), _inline_element("b")
+
+    assert can_unstructured_elements_be_merged(first, second, current_depth=2, next_depth=2) is True
+
+
+def test_inline_elements_at_different_nesting_depths_are_not_merged():
+    # ML-1328: elements on different levels of the HTML tree must never merge, even when their
+    # content (childless inline) would otherwise allow it.
+    first, second = _inline_element("a"), _inline_element("b")
+
+    assert (
+        can_unstructured_elements_be_merged(first, second, current_depth=1, next_depth=2) is False
+    )

test_unstructured/documents/unstructured_json_output/example.json

@@ -0,0 +1,146 @@
+[
+    {
+        "element_id": "eda37931eb954fcc8dec8804c7e8fa4c",
+        "metadata": {
+            "category_depth": 0,
+            "file_directory": "test_unstructured/documents/html_files",
+            "filename": "example.html",
+            "filetype": "text/html",
+            "languages": [
+                "eng"
+            ],
+            "last_modified": "2025-06-12T11:12:20",
+            "page_number": 1,
+            "parent_id": "037b418b76eb4ac1bd40326ff67e67b0",
+            "text_as_html": "<div class=\"Page\" data-page-number=\"1\" />"
+        },
+        "text": "",
+        "type": "UncategorizedText"
+    },
+    {
+        "element_id": "97eb491421584ad892074d039779fbfa",
+        "metadata": {
+            "category_depth": 1,
+            "file_directory": "test_unstructured/documents/html_files",
+            "filename": "example.html",
+            "filetype": "text/html",
+            "languages": [
+                "eng"
+            ],
+            "last_modified": "2025-06-12T11:12:20",
+            "page_number": 1,
+            "parent_id": "eda37931eb954fcc8dec8804c7e8fa4c",
+            "text_as_html": "<header class=\"Header\"><h1 class=\"Title\">Header</h1><time class=\"CalendarDate\">Date: October 30, 2023</time></header>"
+        },
+        "text": "Header Date: October 30, 2023",
+        "type": "Header"
+    },
+    {
+        "element_id": "4afb6e4a90e14835b958dadb77cd8331",
+        "metadata": {
+            "category_depth": 1,
+            "file_directory": "test_unstructured/documents/html_files",
+            "filename": "example.html",
+            "filetype": "text/html",
+            "languages": [
+                "eng"
+            ],
+            "last_modified": "2025-06-12T11:12:20",
+            "page_number": 1,
+            "parent_id": "eda37931eb954fcc8dec8804c7e8fa4c",
+            "text_as_html": "<form class=\"Form\"><label class=\"FormField\" for=\"company-name\">From field name</label><input class=\"FormFieldValue\" value=\"Example value\" /></form>"
+        },
+        "text": "From field name Example value",
+        "type": "UncategorizedText"
+    },
+    {
+        "element_id": "d8f996f2bc9a49f4979aac58a2a9ee93",
+        "metadata": {
+            "category_depth": 1,
+            "file_directory": "test_unstructured/documents/html_files",
+            "filename": "example.html",
+            "filetype": "text/html",
+            "languages": [
+                "eng"
+            ],
+            "last_modified": "2025-06-12T11:12:20",
+            "page_number": 1,
+            "parent_id": "eda37931eb954fcc8dec8804c7e8fa4c",
+            "text_as_html": "<section class=\"Section\" />"
+        },
+        "text": "",
+        "type": "UncategorizedText"
+    },
+    {
+        "element_id": "d2c12f995ab248808900f66aec479e9d",
+        "metadata": {
+            "category_depth": 2,
+            "file_directory": "test_unstructured/documents/html_files",
+            "filename": "example.html",
+            "filetype": "text/html",
+            "languages": [
+                "eng"
+            ],
+            "last_modified": "2025-06-12T11:12:20",
+            "page_number": 1,
+            "parent_id": "d8f996f2bc9a49f4979aac58a2a9ee93",
+            "text_as_html": "<table class=\"Table\"><thead><tr><th>Description</th><th>Row header</th></tr></thead><tbody><tr><td>Value description</td><td><span>50 $</span><span>(1.32 %)</span></td></tr></tbody></table>"
+        },
+        "text": "Description Row header Value description 50 $ (1.32 %)",
+        "type": "Table"
+    },
+    {
+        "element_id": "8e3f0d85329343008593f43afcad3327",
+        "metadata": {
+            "category_depth": 1,
+            "file_directory": "test_unstructured/documents/html_files",
+            "filename": "example.html",
+            "filetype": "text/html",
+            "languages": [
+                "eng"
+            ],
+            "last_modified": "2025-06-12T11:12:20",
+            "page_number": 1,
+            "parent_id": "eda37931eb954fcc8dec8804c7e8fa4c",
+            "text_as_html": "<section class=\"Section\" />"
+        },
+        "text": "",
+        "type": "UncategorizedText"
+    },
+    {
+        "element_id": "5deaad75854741ccb69767881ef399db",
+        "metadata": {
+            "category_depth": 2,
+            "file_directory": "test_unstructured/documents/html_files",
+            "filename": "example.html",
+            "filetype": "text/html",
+            "languages": [
+                "eng"
+            ],
+            "last_modified": "2025-06-12T11:12:20",
+            "page_number": 1,
+            "parent_id": "8e3f0d85329343008593f43afcad3327",
+            "text_as_html": "<h2 class=\"Subtitle\">2. Subtitle</h2>"
+        },
+        "text": "2. Subtitle",
+        "type": "Title"
+    },
+    {
+        "element_id": "9e61f29755bc4b6dbb41ea575d41edb6",
+        "metadata": {
+            "category_depth": 2,
+            "file_directory": "test_unstructured/documents/html_files",
+            "filename": "example.html",
+            "filetype": "text/html",
+            "languages": [
+                "eng"
+            ],
+            "last_modified": "2025-06-12T11:12:20",
+            "page_number": 1,
+            "parent_id": "8e3f0d85329343008593f43afcad3327",
+            "text_as_html": "<p class=\"NarrativeText\">Paragraph text</p>"
+        },
+        "text": "Paragraph text",
+        "type": "NarrativeText"
+    }
+]

test_unstructured/documents/unstructured_json_output/example_with_alternative_text.json

@@ -0,0 +1,38 @@
+[
+    {
+        "element_id": "8157292897eb45e68229b5816da6e46c",
+        "metadata": {
+            "category_depth": 0,
+            "file_directory": "test_unstructured/documents/html_files",
+            "filename": "example_with_alternative_text.html",
+            "filetype": "text/html",
+            "languages": [
+                "eng"
+            ],
+            "last_modified": "2025-06-12T11:12:20",
+            "page_number": 1,
+            "parent_id": "6f5f66a6a23642a7958aeff927e343cd",
+            "text_as_html": "<div class=\"Page\" data-page-number=\"1\" />"
+        },
+        "text": "",
+        "type": "UncategorizedText"
+    },
+    {
+        "element_id": "70c89a734fe1497293a2e01b2f35b887",
+        "metadata": {
+            "category_depth": 1,
+            "file_directory": "test_unstructured/documents/html_files",
+            "filename": "example_with_alternative_text.html",
+            "filetype": "text/html",
+            "languages": [
+                "eng"
+            ],
+            "last_modified": "2025-06-12T11:12:20",
+            "page_number": 1,
+            "parent_id": "8157292897eb45e68229b5816da6e46c",
+            "text_as_html": "<header class=\"Header\"><img class=\"Logo\" alt=\"New York logo\" /><img class=\"Image\" alt=\"A line graph showing the comparison of 5 year cumulative total return for stocks\" /></header>"
+        },
+        "text": "New York logo A line graph showing the comparison of 5 year cumulative total return for stocks",
+        "type": "Header"
+    }
+]

test_unstructured/documents/unstructured_json_output/example_with_inline_fields.json

@@ -0,0 +1,38 @@
+[
+    {
+        "element_id": "7aeac07bad7f44359c3b2403d2ca3d2a",
+        "metadata": {
+            "category_depth": 0,
+            "file_directory": "test_unstructured/documents/html_files",
+            "filename": "example_with_inline_fields.html",
+            "filetype": "text/html",
+            "languages": [
+                "eng"
+            ],
+            "last_modified": "2025-06-12T11:12:20",
+            "page_number": 1,
+            "parent_id": "8bf6d9316275493499a373771a6e46d0",
+            "text_as_html": "<div class=\"Page\" data-page-number=\"1\" />"
+        },
+        "text": "",
+        "type": "UncategorizedText"
+    },
+    {
+        "element_id": "f8f71f7ce10748f08bee0f9922a04406",
+        "metadata": {
+            "category_depth": 1,
+            "file_directory": "test_unstructured/documents/html_files",
+            "filename": "example_with_inline_fields.html",
+            "filetype": "text/html",
+            "languages": [
+                "eng"
+            ],
+            "last_modified": "2025-06-12T11:12:20",
+            "page_number": 1,
+            "parent_id": "7aeac07bad7f44359c3b2403d2ca3d2a",
+            "text_as_html": "<header class=\"Header\"><p class=\"NarrativeText\">Table of Contents</p><address class=\"Address\">68 Prince Street Palmdale, CA 93550</address><a class=\"Hyperlink\">www.google.com</a><span class=\"UncategorizedText\">More text</span></header>"
+        },
+        "text": "Table of Contents 68 Prince Street Palmdale, CA 93550 www.google.com More text",
+        "type": "Header"
+    }
+]

test_unstructured/documents/unstructured_json_output/three_tables.json

@@ -0,0 +1,44 @@
+[
+    {
+        "element_id": "2428404551304d4db5925f6afee11ed5",
+        "metadata": {
+            "category_depth": 0,
+            "filetype": "text/html",
+            "languages": [
+                "eng"
+            ],
+            "parent_id": "517f8559ba594270bdd67e1b02bf19a2",
+            "text_as_html": "<table class=\"Table\"><tr><th>Header 1</th><th>Header 2</th></tr><tr><td>Row 1, Cell 1</td><td>Row 1, Cell 2</td></tr><tr><td>Row 2, Cell 1</td><td>Row 2, Cell 2</td></tr></table>"
+        },
+        "text": "Header 1 Header 2 Row 1, Cell 1 Row 1, Cell 2 Row 2, Cell 1 Row 2, Cell 2",
+        "type": "Table"
+    },
+    {
+        "element_id": "9f91cae321c74b31bb1c83ac86cd7afb",
+        "metadata": {
+            "category_depth": 0,
+            "filetype": "text/html",
+            "languages": [
+                "eng"
+            ],
+            "parent_id": "517f8559ba594270bdd67e1b02bf19a2",
+            "text_as_html": "<table class=\"Table\"><tr><th colspan=\"3\">Big Table Header</th></tr><tr><td rowspan=\"2\">Merged Cell 1</td><td>Cell 2</td><td>Cell 3</td></tr><tr><td colspan=\"2\">Merged Cell 4 and 5</td></tr><tr><td>Cell 6</td><td>Cell 7</td><td>Cell 8</td></tr><tr><td>Cell 9</td><td colspan=\"2\">A cell with a lot of text. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.</td></tr><tr><td>Cell 10</td><td>Cell 11</td><td>Cell 12</td></tr></table>"
+        },
+        "text": "Big Table Header Merged Cell 1 Cell 2 Cell 3 Merged Cell 4 and 5 Cell 6 Cell 7 Cell 8 Cell 9 A cell with a lot of text. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Cell 10 Cell 11 Cell 12",
+        "type": "Table"
+    },
+    {
+        "element_id": "da6c34391e544b3480e45d68f40870fa",
+        "metadata": {
+            "category_depth": 0,
+            "filetype": "text/html",
+            "languages": [
+                "eng"
+            ],
+            "parent_id": "517f8559ba594270bdd67e1b02bf19a2",
+            "text_as_html": "<table class=\"TableOfContents\"><tr><th>Chapter</th><th>Title</th><th>Page</th></tr><tr><td>1</td><td>Introduction</td><td>1</td></tr><tr><td>2</td><td>Getting Started</td><td>5</td></tr><tr><td>3</td><td>Basic Concepts</td><td>12</td></tr><tr><td>4</td><td>Advanced Topics</td><td>25</td></tr><tr><td>5</td><td>Conclusion</td><td>40</td></tr></table>"
+        },
+        "text": "Chapter Title Page 1 Introduction 1 2 Getting Started 5 3 Basic Concepts 12 4 Advanced Topics 25 5 Conclusion 40",
+        "type": "Table"
+    }
+]

test_unstructured/embed/test_mixedbreadai.py

@@ -0,0 +1,41 @@
+from unstructured.documents.elements import Text
+from unstructured.embed.mixedbreadai import (
+    MixedbreadAIEmbeddingConfig,
+    MixedbreadAIEmbeddingEncoder,
+)
+
+
+def test_embed_documents_does_not_break_element_to_dict(mocker):
+    mock_client = mocker.MagicMock()
+
+    def mock_embeddings(
+        model,
+        normalized,
+        encoding_format,
+        truncation_strategy,
+        request_options,
+        input,
+    ):
+        mock_response = mocker.MagicMock()
+        mock_response.data = [mocker.MagicMock(embedding=[i, i + 1]) for i in range(len(input))]
+        return mock_response
+
+    mock_client.embeddings.side_effect = mock_embeddings
+
+    # Mock get_client to return our mock_client
+    mocker.patch.object(MixedbreadAIEmbeddingConfig, "get_client", return_value=mock_client)
+
+    encoder = MixedbreadAIEmbeddingEncoder(
+        config=MixedbreadAIEmbeddingConfig(
+            api_key="api_key", model_name="mixedbread-ai/mxbai-embed-large-v1"
+        )
+    )
+
+    elements = encoder.embed_documents(
+        elements=[Text("This is sentence 1"), Text("This is sentence 2")],
+    )
+    assert len(elements) == 2
+    assert elements[0].to_dict()["text"] == "This is sentence 1"
+    assert elements[1].to_dict()["text"] == "This is sentence 2"
+    assert elements[0].embeddings is not None
+    assert elements[1].embeddings is not None

test_unstructured/embed/test_octoai.py

@@ -0,0 +1,19 @@
+from unstructured.documents.elements import Text
+from unstructured.embed.octoai import OctoAiEmbeddingConfig, OctoAIEmbeddingEncoder
+
+
+def test_embed_documents_does_not_break_element_to_dict(mocker):
+    # Mocked client with the desired behavior for embed_documents
+    mock_client = mocker.MagicMock()
+    mock_client.embed_documents.return_value = [1, 2]
+
+    # Mock get_client to return our mock_client
+    mocker.patch.object(OctoAiEmbeddingConfig, "get_client", return_value=mock_client)
+
+    encoder = OctoAIEmbeddingEncoder(config=OctoAiEmbeddingConfig(api_key="api_key"))
+    elements = encoder.embed_documents(
+        elements=[Text("This is sentence 1"), Text("This is sentence 2")],
+    )
+    assert len(elements) == 2
+    assert elements[0].to_dict()["text"] == "This is sentence 1"
+    assert elements[1].to_dict()["text"] == "This is sentence 2"

test_unstructured/embed/test_openai.py

@@ -0,0 +1,19 @@
+from unstructured.documents.elements import Text
+from unstructured.embed.openai import OpenAIEmbeddingConfig, OpenAIEmbeddingEncoder
+
+
+def test_embed_documents_does_not_break_element_to_dict(mocker):
+    # Mocked client with the desired behavior for embed_documents
+    mock_client = mocker.MagicMock()
+    mock_client.embed_documents.return_value = [1, 2]
+
+    # Mock get_client to return our mock_client
+    mocker.patch.object(OpenAIEmbeddingConfig, "get_client", return_value=mock_client)
+
+    encoder = OpenAIEmbeddingEncoder(config=OpenAIEmbeddingConfig(api_key="api_key"))
+    elements = encoder.embed_documents(
+        elements=[Text("This is sentence 1"), Text("This is sentence 2")],
+    )
+    assert len(elements) == 2
+    assert elements[0].to_dict()["text"] == "This is sentence 1"
+    assert elements[1].to_dict()["text"] == "This is sentence 2"

test_unstructured/embed/test_vertexai.py

@@ -0,0 +1,19 @@
+from unstructured.documents.elements import Text
+from unstructured.embed.vertexai import VertexAIEmbeddingConfig, VertexAIEmbeddingEncoder
+
+
+def test_embed_documents_does_not_break_element_to_dict(mocker):
+    # Mocked client with the desired behavior for embed_documents
+    mock_client = mocker.MagicMock()
+    mock_client.embed_documents.return_value = [1, 2]
+
+    # Mock create_client to return our mock_client
+    mocker.patch.object(VertexAIEmbeddingConfig, "get_client", return_value=mock_client)
+
+    encoder = VertexAIEmbeddingEncoder(config=VertexAIEmbeddingConfig(api_key="api_key"))
+    elements = encoder.embed_documents(
+        elements=[Text("This is sentence 1"), Text("This is sentence 2")],
+    )
+    assert len(elements) == 2
+    assert elements[0].to_dict()["text"] == "This is sentence 1"
+    assert elements[1].to_dict()["text"] == "This is sentence 2"

test_unstructured/embed/test_voyageai.py

@@ -0,0 +1,242 @@
+from unittest.mock import Mock
+
+from unstructured.documents.elements import Text
+from unstructured.embed.voyageai import VoyageAIEmbeddingConfig, VoyageAIEmbeddingEncoder
+
+
+def test_embed_documents_does_not_break_element_to_dict(mocker):
+    # Mocked client with the desired behavior for embed_documents
+    embed_response = Mock()
+    embed_response.embeddings = [[1], [2]]
+    mock_client = mocker.MagicMock()
+    mock_client.embed.return_value = embed_response
+    mock_client.tokenize.return_value = [[1], [1]]  # Mock token counts
+
+    # Mock get_client to return our mock_client
+    mocker.patch.object(VoyageAIEmbeddingConfig, "get_client", return_value=mock_client)
+
+    encoder = VoyageAIEmbeddingEncoder(
+        config=VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-3-large")
+    )
+    elements = encoder.embed_documents(
+        elements=[Text("This is sentence 1"), Text("This is sentence 2")],
+    )
+    assert len(elements) == 2
+    assert elements[0].to_dict()["text"] == "This is sentence 1"
+    assert elements[1].to_dict()["text"] == "This is sentence 2"
+
+
+def test_embed_documents_voyage_3_5(mocker):
+    """Test embedding with voyage-3.5 model."""
+    embed_response = Mock()
+    embed_response.embeddings = [[1.0] * 1024, [2.0] * 1024]
+    mock_client = mocker.MagicMock()
+    mock_client.embed.return_value = embed_response
+    mock_client.tokenize.return_value = [[1, 2, 3], [1, 2]]  # Mock token counts
+
+    mocker.patch.object(VoyageAIEmbeddingConfig, "get_client", return_value=mock_client)
+
+    encoder = VoyageAIEmbeddingEncoder(
+        config=VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-3.5")
+    )
+    elements = encoder.embed_documents(
+        elements=[Text("Test document 1"), Text("Test document 2")],
+    )
+    assert len(elements) == 2
+    assert len(elements[0].embeddings) == 1024
+    assert len(elements[1].embeddings) == 1024
+
+
+def test_embed_documents_voyage_3_5_lite(mocker):
+    """Test embedding with voyage-3.5-lite model."""
+    embed_response = Mock()
+    embed_response.embeddings = [[1.0] * 512, [2.0] * 512, [3.0] * 512]
+    mock_client = mocker.MagicMock()
+    mock_client.embed.return_value = embed_response
+    mock_client.tokenize.return_value = [[1], [1], [1]]  # Mock token counts
+
+    mocker.patch.object(VoyageAIEmbeddingConfig, "get_client", return_value=mock_client)
+
+    encoder = VoyageAIEmbeddingEncoder(
+        config=VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-3.5-lite")
+    )
+    elements = encoder.embed_documents(
+        elements=[Text("Test 1"), Text("Test 2"), Text("Test 3")],
+    )
+    assert len(elements) == 3
+    assert all(len(e.embeddings) == 512 for e in elements)
+
+
+def test_embed_documents_contextual_model(mocker):
+    """Test embedding with voyage-context-3 model."""
+    # Mock contextualized_embed response
+    contextualized_response = Mock()
+    result_item = Mock()
+    result_item.embeddings = [[1.0] * 1024, [2.0] * 1024]
+    contextualized_response.results = [result_item]
+
+    mock_client = mocker.MagicMock()
+    mock_client.contextualized_embed.return_value = contextualized_response
+    mock_client.tokenize.return_value = [[1, 2], [1, 2, 3]]  # Mock token counts
+
+    mocker.patch.object(VoyageAIEmbeddingConfig, "get_client", return_value=mock_client)
+
+    encoder = VoyageAIEmbeddingEncoder(
+        config=VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-context-3")
+    )
+    elements = encoder.embed_documents(
+        elements=[Text("Context document 1"), Text("Context document 2")],
+    )
+    assert len(elements) == 2
+    assert len(elements[0].embeddings) == 1024
+    assert len(elements[1].embeddings) == 1024
+    # Verify contextualized_embed was called
+    mock_client.contextualized_embed.assert_called_once()
+
+
+def test_count_tokens(mocker):
+    """Test token counting functionality."""
+    mock_client = mocker.MagicMock()
+    mock_client.tokenize.return_value = [[1, 2], [1, 2, 3, 4, 5]]  # Different token counts
+
+    mocker.patch.object(VoyageAIEmbeddingConfig, "get_client", return_value=mock_client)
+
+    encoder = VoyageAIEmbeddingEncoder(
+        config=VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-3.5")
+    )
+    texts = ["short text", "this is a longer text with more tokens"]
+    token_counts = encoder.count_tokens(texts)
+
+    assert len(token_counts) == 2
+    assert token_counts[0] == 2
+    assert token_counts[1] == 5
+
+
+def test_count_tokens_empty_list(mocker):
+    """Test token counting with empty list."""
+    mocker.patch.object(VoyageAIEmbeddingConfig, "get_client", return_value=mocker.MagicMock())
+
+    encoder = VoyageAIEmbeddingEncoder(
+        config=VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-3.5")
+    )
+    token_counts = encoder.count_tokens([])
+    assert token_counts == []
+
+
+def test_get_token_limit(mocker):
+    """Test getting token limit for different models."""
+    mocker.patch.object(VoyageAIEmbeddingConfig, "get_client", return_value=mocker.MagicMock())
+
+    # Test voyage-3.5 model
+    config = VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-3.5")
+    assert config.get_token_limit() == 320_000
+
+    # Test voyage-3.5-lite model
+    config_lite = VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-3.5-lite")
+    assert config_lite.get_token_limit() == 1_000_000
+
+    # Test context model
+    config_context = VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-context-3")
+    assert config_context.get_token_limit() == 32_000
+
+    # Test voyage-2 model
+    config_v2 = VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-2")
+    assert config_v2.get_token_limit() == 320_000
+
+    # Test unknown model (should use default)
+    config_unknown = VoyageAIEmbeddingConfig(api_key="api_key", model_name="unknown-model")
+    assert config_unknown.get_token_limit() == 120_000
+
+
+def test_is_context_model(mocker):
+    """Test the _is_context_model helper method."""
+    mocker.patch.object(VoyageAIEmbeddingConfig, "get_client", return_value=mocker.MagicMock())
+
+    # Test with context model
+    encoder_context = VoyageAIEmbeddingEncoder(
+        config=VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-context-3")
+    )
+    assert encoder_context._is_context_model() is True
+
+    # Test with regular model
+    encoder_regular = VoyageAIEmbeddingEncoder(
+        config=VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-3.5")
+    )
+    assert encoder_regular._is_context_model() is False
+
+
+def test_build_batches_with_token_limits(mocker):
+    """Test that batching respects token limits."""
+    mock_client = mocker.MagicMock()
+    # Simulate different token counts for each text
+    mock_client.tokenize.return_value = [[1] * 10, [1] * 20, [1] * 15, [1] * 25]
+
+    mocker.patch.object(VoyageAIEmbeddingConfig, "get_client", return_value=mock_client)
+
+    encoder = VoyageAIEmbeddingEncoder(
+        config=VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-2")
+    )
+    texts = ["text1", "text2", "text3", "text4"]
+    batches = list(encoder._build_batches(texts, mock_client))
+
+    # Should create at least one batch
+    assert len(batches) >= 1
+    # Total texts should be preserved
+    total_texts = sum(len(batch) for batch in batches)
+    assert total_texts == len(texts)
+
+
+def test_embed_query(mocker):
+    """Test embedding a single query."""
+    embed_response = Mock()
+    embed_response.embeddings = [[1.0] * 1024]
+    mock_client = mocker.MagicMock()
+    mock_client.embed.return_value = embed_response
+
+    mocker.patch.object(VoyageAIEmbeddingConfig, "get_client", return_value=mock_client)
+
+    encoder = VoyageAIEmbeddingEncoder(
+        config=VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-3.5")
+    )
+    embedding = encoder.embed_query("test query")
+
+    assert len(embedding) == 1024
+    # Verify embed was called with input_type="query"
+    mock_client.embed.assert_called_once()
+    call_kwargs = mock_client.embed.call_args[1]
+    assert call_kwargs["input_type"] == "query"
+
+
+def test_embed_documents_with_output_dimension(mocker):
+    """Test embedding with custom output dimension."""
+    embed_response = Mock()
+    embed_response.embeddings = [[1.0] * 512, [2.0] * 512]
+    mock_client = mocker.MagicMock()
+    mock_client.embed.return_value = embed_response
+    mock_client.tokenize.return_value = [[1], [1]]
+
+    mocker.patch.object(VoyageAIEmbeddingConfig, "get_client", return_value=mock_client)
+
+    encoder = VoyageAIEmbeddingEncoder(
+        config=VoyageAIEmbeddingConfig(
+            api_key="api_key", model_name="voyage-3.5", output_dimension=512
+        )
+    )
+    elements = encoder.embed_documents(
+        elements=[Text("Test 1"), Text("Test 2")],
+    )
+    assert len(elements) == 2
+    # Verify output_dimension was passed
+    call_kwargs = mock_client.embed.call_args[1]
+    assert call_kwargs["output_dimension"] == 512
+
+
+def test_embed_documents_empty_list(mocker):
+    """Test embedding empty list of documents."""
+    mocker.patch.object(VoyageAIEmbeddingConfig, "get_client", return_value=mocker.MagicMock())
+
+    encoder = VoyageAIEmbeddingEncoder(
+        config=VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-3.5")
+    )
+    elements = encoder.embed_documents(elements=[])
+    assert elements == []

test_unstructured/file_utils/test_encoding.py

@@ -0,0 +1,71 @@
+"""Test encoding detection error handling (PR #4071)."""
+
+import os
+import pickle
+import sys
+import tempfile
+from unittest.mock import patch
+
+import pytest
+
+from unstructured.errors import UnprocessableEntityError
+from unstructured.file_utils.encoding import detect_file_encoding
+
+
+def test_charset_detection_failure():
+    """Test encoding detection failure with memory safety checks."""
+    large_data = b"\x80\x81\x82\x83" * 250_000  # 1MB of invalid UTF-8
+
+    with tempfile.NamedTemporaryFile(mode="wb", delete=False) as f:
+        f.write(large_data)
+        temp_file_path = f.name
+
+    try:
+        detect_result = {"encoding": None, "confidence": None}
+        with patch("unstructured.file_utils.encoding.detect", return_value=detect_result):
+            with patch("unstructured.file_utils.encoding.COMMON_ENCODINGS", ["utf_8"]):  # Will fail
+                with pytest.raises(UnprocessableEntityError) as exc_info:
+                    detect_file_encoding(filename=temp_file_path)
+
+                exception = exc_info.value
+
+                assert "Unable to determine file encoding" in str(exception)
+
+                # Ensure no .object attribute that would store file content (prevents memory bloat)
+                # See: https://docs.python.org/3/library/exceptions.html#UnicodeError.object
+                assert not hasattr(exception, "object")
+
+                # Exception should be lightweight regardless of file size
+                exception_memory = sys.getsizeof(exception)
+                serialized_size = len(pickle.dumps(exception))
+
+                assert exception_memory < 10_000  # Small in-memory footprint
+                assert serialized_size < 10_000  # Small serialization footprint
+    finally:
+        os.unlink(temp_file_path)
+
+
+def test_decode_failure():
+    """Test decode failure with memory safety checks."""
+    # Invalid UTF-16: BOM followed by odd number of bytes
+    invalid_utf16 = b"\xff\xfe" + b"A\x00B\x00" + b"\x00"
+
+    detect_result = {"encoding": "utf-16", "confidence": 0.95}
+    with patch("unstructured.file_utils.encoding.detect", return_value=detect_result):
+        with pytest.raises(UnprocessableEntityError) as exc_info:
+            detect_file_encoding(file=invalid_utf16)
+
+        exception = exc_info.value
+
+        assert "detected 'utf-16' but decode failed" in str(exception)
+
+        # Ensure no .object attribute that would store file content (prevents memory bloat)
+        # See: https://docs.python.org/3/library/exceptions.html#UnicodeError.object
+        assert not hasattr(exception, "object")
+
+        # Exception should be lightweight
+        exception_memory = sys.getsizeof(exception)
+        serialized_size = len(pickle.dumps(exception))
+
+        assert exception_memory < 10_000  # Small in-memory footprint
+        assert serialized_size < 10_000  # Small serialization footprint

test_unstructured/file_utils/test_file_conversion.py

@@ -0,0 +1,54 @@
+import os
+import pathlib
+import tempfile
+from unittest.mock import patch
+
+import pypandoc
+import pytest
+
+from test_unstructured.unit_utils import FixtureRequest, example_doc_path, stdlib_fn_mock
+from unstructured.file_utils.file_conversion import (
+    convert_file_to_html_text_using_pandoc,
+    convert_file_to_text,
+)
+
+DIRECTORY = pathlib.Path(__file__).parent.resolve()
+
+
+def test_convert_file_to_text():
+    filename = os.path.join(DIRECTORY, "..", "..", "example-docs", "winter-sports.epub")
+    html_text = convert_file_to_text(filename, source_format="epub", target_format="html")
+    assert html_text.startswith("<p>")
+
+
+def test_convert_to_file_raises_if_pandoc_not_available():
+    filename = os.path.join(DIRECTORY, "..", "..", "example-docs", "winter-sports.epub")
+    with patch.object(pypandoc, "convert_file", side_effect=FileNotFoundError):
+        with pytest.raises(FileNotFoundError):
+            convert_file_to_text(filename, source_format="epub", target_format="html")
+
+
+@pytest.mark.parametrize(
+    ("source_format", "filename"),
+    [
+        ("epub", "winter-sports.epub"),
+        ("org", "README.org"),
+        ("rst", "README.rst"),
+        ("rtf", "fake-doc.rtf"),
+    ],
+)
+def test_convert_file_to_html_text_using_pandoc(
+    request: FixtureRequest, tmp_path: pathlib.Path, source_format: str, filename: str
+):
+    # -- Get a real tempdir: `tmp_path`
+    # -- Mock tempfile.TemporaryDirectory() using `stdlib_fn_mock`
+    # -- Set the return value of mock.__enter__ to the real tempdir
+    tempdir_ = stdlib_fn_mock(request, tempfile, "TemporaryDirectory")
+    tempdir_.return_value.__enter__.return_value = tmp_path
+
+    with open(example_doc_path(filename), "rb") as f:
+        html_text = convert_file_to_html_text_using_pandoc(file=f, source_format=source_format)
+
+    assert isinstance(html_text, str)
+    assert len(list(tmp_path.iterdir())) == 1
+    tempdir_.return_value.__exit__.assert_called_once()

test_unstructured/file_utils/test_model.py

@@ -0,0 +1,244 @@
+"""Test suite for `unstructured.file_utils.filetype`."""
+
+from __future__ import annotations
+
+import pytest
+
+from unstructured.file_utils.model import FileType, create_file_type, register_partitioner
+
+
+class DescribeFileType:
+    """Unit-test suite for `unstructured.file_utils.model.Filetype`."""
+
+    # -- .__lt__() ----------------------------------------------
+
+    def it_is_a_collection_ordered_by_name_and_can_be_sorted(self):
+        """FileType is a total order on name, e.g. FileType.A < FileType.B."""
+        assert FileType.EML < FileType.HTML < FileType.XML
+
+    # -- .from_extension() --------------------------------------
+
+    @pytest.mark.parametrize(
+        ("ext", "file_type"),
+        [
+            (".bmp", FileType.BMP),
+            (".html", FileType.HTML),
+            (".eml", FileType.EML),
+            (".p7s", FileType.EML),
+            (".java", FileType.TXT),
+        ],
+    )
+    def it_can_recognize_a_file_type_from_an_extension(self, ext: str, file_type: FileType | None):
+        assert FileType.from_extension(ext) is file_type
+
+    @pytest.mark.parametrize("ext", [".foobar", ".xyz", ".mdx", "", ".", None])
+    def but_not_when_that_extension_is_empty_or_None_or_not_registered(self, ext: str | None):
+        assert FileType.from_extension(ext) is None
+
+    # -- .from_mime_type() --------------------------------------
+
+    @pytest.mark.parametrize(
+        ("mime_type", "file_type"),
+        [
+            ("image/bmp", FileType.BMP),
+            ("text/x-csv", FileType.CSV),
+            ("application/msword", FileType.DOC),
+            ("message/rfc822", FileType.EML),
+            ("text/plain", FileType.TXT),
+            ("text/yaml", FileType.TXT),
+            ("application/xml", FileType.XML),
+            ("text/xml", FileType.XML),
+            ("inode/x-empty", FileType.EMPTY),
+        ],
+    )
+    def it_can_recognize_a_file_type_from_a_mime_type(
+        self, mime_type: str, file_type: FileType | None
+    ):
+        assert FileType.from_mime_type(mime_type) is file_type
+
+    @pytest.mark.parametrize("mime_type", ["text/css", "image/gif", "foo/bar", None])
+    def but_not_when_that_mime_type_is_not_registered_by_a_file_type_or_None(
+        self, mime_type: str | None
+    ):
+        assert FileType.from_mime_type(mime_type) is None
+
+    # -- .extra_name --------------------------------------------
+
+    @pytest.mark.parametrize(
+        ("file_type", "expected_value"),
+        [
+            (FileType.BMP, "image"),
+            (FileType.DOC, "doc"),
+            (FileType.DOCX, "docx"),
+            (FileType.EML, None),
+            (FileType.EMPTY, None),
+            (FileType.MSG, "msg"),
+            (FileType.PDF, "pdf"),
+            (FileType.XLS, "xlsx"),
+            (FileType.UNK, None),
+            (FileType.WAV, "audio"),
+            (FileType.ZIP, None),
+        ],
+    )
+    def and_it_knows_which_pip_extra_needs_to_be_installed_to_get_those_dependencies(
+        self, file_type: FileType, expected_value: str | None
+    ):
+        assert file_type.extra_name == expected_value
+
+    # -- .importable_package_dependencies -----------------------
+
+    @pytest.mark.parametrize(
+        ("file_type", "expected_value"),
+        [
+            (FileType.BMP, ("unstructured_inference",)),
+            (FileType.CSV, ("pandas",)),
+            (FileType.DOC, ("docx",)),
+            (FileType.EMPTY, ()),
+            (FileType.HTML, ()),
+            (FileType.ODT, ("docx", "pypandoc")),
+            (FileType.PDF, ("pdf2image", "pdfminer", "PIL")),
+            (FileType.UNK, ()),
+            (FileType.WAV, ()),  # STT agent deps validated at runtime
+            (FileType.ZIP, ()),
+        ],
+    )
+    def it_knows_which_importable_packages_its_partitioner_depends_on(
+        self, file_type: FileType, expected_value: tuple[str, ...]
+    ):
+        assert file_type.importable_package_dependencies == expected_value
+
+    # -- .is_partitionable --------------------------------------
+
+    @pytest.mark.parametrize(
+        ("file_type", "expected_value"),
+        [
+            (FileType.BMP, True),
+            (FileType.CSV, True),
+            (FileType.DOC, True),
+            (FileType.EML, True),
+            (FileType.JPG, True),
+            (FileType.PDF, True),
+            (FileType.PPTX, True),
+            (FileType.WAV, True),
+            (FileType.ZIP, False),
+            (FileType.EMPTY, False),
+            (FileType.UNK, False),
+        ],
+    )
+    def it_knows_whether_files_of_its_type_are_directly_partitionable(
+        self, file_type: FileType, expected_value: str
+    ):
+        assert file_type.is_partitionable is expected_value
+
+    # -- .mime_type ---------------------------------------------
+
+    @pytest.mark.parametrize(
+        ("file_type", "mime_type"),
+        [
+            (FileType.BMP, "image/bmp"),
+            (FileType.CSV, "text/csv"),
+            (FileType.DOC, "application/msword"),
+            (FileType.EML, "message/rfc822"),
+            (FileType.HTML, "text/html"),
+            (FileType.JPG, "image/jpeg"),
+            (FileType.PDF, "application/pdf"),
+            (FileType.TXT, "text/plain"),
+            (FileType.XML, "application/xml"),
+            (FileType.EMPTY, "inode/x-empty"),
+            (FileType.UNK, "application/octet-stream"),
+        ],
+    )
+    def it_knows_its_canonical_MIME_type(self, file_type: FileType, mime_type: str):
+        assert file_type.mime_type == mime_type
+
+    # -- .partitioner_function_name -----------------------------
+
+    @pytest.mark.parametrize(
+        ("file_type", "expected_value"),
+        [
+            (FileType.BMP, "partition_image"),
+            (FileType.CSV, "partition_csv"),
+            (FileType.DOC, "partition_doc"),
+            (FileType.DOCX, "partition_docx"),
+            (FileType.JPG, "partition_image"),
+            (FileType.PNG, "partition_image"),
+            (FileType.TIFF, "partition_image"),
+            (FileType.WAV, "partition_audio"),
+        ],
+    )
+    def it_knows_its_partitioner_function_name(self, file_type: FileType, expected_value: str):
+        assert file_type.partitioner_function_name == expected_value
+
+    @pytest.mark.parametrize("file_type", [FileType.ZIP, FileType.EMPTY, FileType.UNK])
+    def but_it_raises_on_partitioner_function_name_access_when_the_file_type_is_not_partitionable(
+        self, file_type: FileType
+    ):
+        with pytest.raises(ValueError, match="`.partitioner_function_name` is undefined because "):
+            file_type.partitioner_function_name
+
+    # -- .partitioner_module_qname ------------------------------
+
+    @pytest.mark.parametrize(
+        ("file_type", "expected_value"),
+        [
+            (FileType.BMP, "unstructured.partition.image"),
+            (FileType.CSV, "unstructured.partition.csv"),
+            (FileType.DOC, "unstructured.partition.doc"),
+            (FileType.DOCX, "unstructured.partition.docx"),
+            (FileType.JPG, "unstructured.partition.image"),
+            (FileType.PNG, "unstructured.partition.image"),
+            (FileType.TIFF, "unstructured.partition.image"),
+            (FileType.WAV, "unstructured.partition.audio"),
+        ],
+    )
+    def it_knows_the_fully_qualified_name_of_its_partitioner_module(
+        self, file_type: FileType, expected_value: str
+    ):
+        assert file_type.partitioner_module_qname == expected_value
+
+    @pytest.mark.parametrize("file_type", [FileType.ZIP, FileType.EMPTY, FileType.UNK])
+    def but_it_raises_on_partitioner_module_qname_access_when_the_file_type_is_not_partitionable(
+        self, file_type: FileType
+    ):
+        with pytest.raises(ValueError, match="`.partitioner_module_qname` is undefined because "):
+            file_type.partitioner_module_qname
+
+    # -- .partitioner_shortname ---------------------------------
+
+    @pytest.mark.parametrize(
+        ("file_type", "expected_value"),
+        [
+            (FileType.BMP, "image"),
+            (FileType.CSV, "csv"),
+            (FileType.DOC, "doc"),
+            (FileType.DOCX, "docx"),
+            (FileType.JPG, "image"),
+            (FileType.PNG, "image"),
+            (FileType.TIFF, "image"),
+            (FileType.WAV, "audio"),
+            (FileType.XLS, "xlsx"),
+            (FileType.XLSX, "xlsx"),
+        ],
+    )
+    def it_provides_access_to_the_partitioner_shortname(
+        self, file_type: FileType, expected_value: str
+    ):
+        assert file_type.partitioner_shortname == expected_value
+
+
+def test_create_file_type():
+    file_type = create_file_type("FOO", canonical_mime_type="application/foo", extensions=[".foo"])
+
+    assert FileType.from_extension(".foo") is file_type
+    assert FileType.from_mime_type("application/foo") is file_type
+
+
+def test_register_partitioner():
+    file_type = create_file_type("FOO", canonical_mime_type="application/foo", extensions=[".foo"])
+
+    @register_partitioner(file_type)
+    def partition_foo():
+        pass
+
+    assert file_type.partitioner_function_name == "partition_foo"
+    assert file_type.partitioner_module_qname == "test_unstructured.file_utils.test_model"

test_unstructured/metrics/test_element_type.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import pytest
+
+from test_unstructured.unit_utils import example_doc_path
+from unstructured.metrics.element_type import (
+    FrequencyDict,
+    calculate_element_type_percent_match,
+    get_element_type_frequency,
+)
+from unstructured.partition.auto import partition
+from unstructured.staging.base import elements_to_json
+
+
+@pytest.mark.parametrize(
+    ("filename", "frequency"),
+    [
+        (
+            "fake-email.txt",
+            {
+                ("NarrativeText", None): 1,
+                ("UncategorizedText", None): 1,
+                ("ListItem", 1): 2,
+            },
+        ),
+        (
+            "sample-presentation.pptx",
+            {
+                ("Title", 0): 4,
+                ("Title", 1): 2,
+                ("NarrativeText", 0): 2,
+                ("PageBreak", None): 3,
+                ("ListItem", 0): 6,
+                ("ListItem", 1): 6,
+                ("ListItem", 2): 3,
+                ("Table", None): 1,
+            },
+        ),
+    ],
+)
+def test_get_element_type_frequency(filename: str, frequency: dict[tuple[str, int | None], int]):
+    elements = partition(example_doc_path(filename))
+    elements_freq = get_element_type_frequency(elements_to_json(elements))
+    assert elements_freq == frequency
+
+
+@pytest.mark.parametrize(
+    ("filename", "expected_frequency", "percent_matched"),
+    [
+        (
+            "fake-email.txt",
+            {
+                ("UncategorizedText", None): 1,
+                ("ListItem", 1): 2,
+                ("NarrativeText", None): 2,
+            },
+            (0.8, 0.8, 0.80),
+        ),
+        (
+            "sample-presentation.pptx",
+            {
+                ("Title", 0): 3,
+                ("Title", 1): 1,
+                ("NarrativeText", None): 1,
+                ("NarrativeText", 0): 3,
+                ("ListItem", 0): 6,
+                ("ListItem", 1): 6,
+                ("ListItem", 2): 3,
+                ("Table", None): 1,
+            },
+            (0.92, 0.92, 0.92),
+        ),
+        (
+            "handbook-1p.docx",
+            {
+                ("Header", None): 1,
+                ("UncategorizedText", 0): 6,
+                ("ListItem", 3): 3,
+                ("NarrativeText", 0): 7,
+                ("Footer", None): 1,
+            },
+            (0.78, 0.72, 0.81),
+        ),
+        (
+            "handbook-1p.docx",
+            {
+                ("Header", None): 1,
+                ("UncategorizedText", 0): 6,
+                ("NarrativeText", 0): 7,
+                ("PageBreak", None): 1,
+                ("Footer", None): 1,
+            },
+            (0.94, 0.88, 0.98),
+        ),
+    ],
+)
+def test_calculate_element_type_percent_match(
+    filename: str, expected_frequency: FrequencyDict, percent_matched: tuple[float, float, float]
+):
+    elements = partition(example_doc_path(filename))
+    elements_frequency = get_element_type_frequency(elements_to_json(elements))
+    assert (
+        round(calculate_element_type_percent_match(elements_frequency, expected_frequency), 2)
+        == percent_matched[0]
+    )
+    assert (
+        round(calculate_element_type_percent_match(elements_frequency, expected_frequency, 0.0), 2)
+        == percent_matched[1]
+    )
+    assert (
+        round(calculate_element_type_percent_match(elements_frequency, expected_frequency, 0.8), 2)
+        == percent_matched[2]
+    )

test_unstructured/metrics/test_evaluate.py

@@ -0,0 +1,597 @@
+import os
+import pathlib
+import shutil
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from unstructured.metrics.evaluate import (
+    ElementTypeMetricsCalculator,
+    TableStructureMetricsCalculator,
+    TextExtractionMetricsCalculator,
+    filter_metrics,
+    get_mean_grouping,
+)
+
+is_in_docker = os.path.exists("/.dockerenv")
+
+EXAMPLE_DOCS_DIRECTORY = os.path.join(
+    pathlib.Path(__file__).parent.resolve(), "..", "..", "example-docs"
+)
+TESTING_FILE_DIR = os.path.join(EXAMPLE_DOCS_DIRECTORY, "test_evaluate_files")
+
+UNSTRUCTURED_OUTPUT_DIRNAME = "unstructured_output"
+GOLD_CCT_DIRNAME = "gold_standard_cct"
+GOLD_ELEMENT_TYPE_DIRNAME = "gold_standard_element_type"
+GOLD_TABLE_STRUCTURE_DIRNAME = "gold_standard_table_structure"
+UNSTRUCTURED_CCT_DIRNAME = "unstructured_output_cct"
+UNSTRUCTURED_TABLE_STRUCTURE_DIRNAME = "unstructured_output_table_structure"
+
+DUMMY_DF_CCT = pd.DataFrame(
+    {
+        "filename": [
+            "Bank Good Credit Loan.pptx",
+            "Performance-Audit-Discussion.pdf",
+            "currency.csv",
+        ],
+        "doctype": ["pptx", "pdf", "csv"],
+        "connector": ["connector1", "connector1", "connector2"],
+        "cct-accuracy": [0.812, 0.994, 0.887],
+        "cct-%missing": [0.001, 0.002, 0.041],
+    }
+)
+
+DUMMY_DF_ELEMENT_TYPE = pd.DataFrame(
+    {
+        "filename": [
+            "Bank Good Credit Loan.pptx",
+            "Performance-Audit-Discussion.pdf",
+            "currency.csv",
+        ],
+        "doctype": ["pptx", "pdf", "csv"],
+        "connector": ["connector1", "connector1", "connector2"],
+        "element-type-accuracy": [0.812, 0.994, 0.887],
+    }
+)
+
+
+@pytest.fixture
+def mock_dependencies():
+    with (
+        patch("unstructured.metrics.evaluate.calculate_accuracy") as mock_calculate_accuracy,
+        patch(
+            "unstructured.metrics.evaluate.calculate_percent_missing_text"
+        ) as mock_calculate_percent_missing_text,
+        patch.object(TextExtractionMetricsCalculator, "_get_ccts") as mock_get_ccts,
+        patch(
+            "unstructured.metrics.evaluate.get_element_type_frequency"
+        ) as mock_get_element_type_frequency,
+        patch(
+            "unstructured.metrics.evaluate.calculate_element_type_percent_match"
+        ) as mock_calculate_element_type_percent_match,
+        patch("unstructured.metrics.evaluate._read_text_file") as mock_read_text_file,
+        patch.object(Path, "exists") as mock_path_exists,
+        patch(
+            "unstructured.metrics.evaluate.TableEvalProcessor.from_json_files"
+        ) as mock_table_eval_processor_from_json_files,
+        patch.object(
+            TableStructureMetricsCalculator, "supported_metric_names"
+        ) as mock_supported_metric_names,
+    ):
+        mocks = {
+            "mock_calculate_accuracy": mock_calculate_accuracy,
+            "mock_calculate_percent_missing_text": mock_calculate_percent_missing_text,
+            "mock_get_ccts": mock_get_ccts,
+            "mock_get_element_type_frequency": mock_get_element_type_frequency,
+            "mock_read_text_file": mock_read_text_file,
+            "mock_calculate_element_type_percent_match": mock_calculate_element_type_percent_match,
+            "mock_table_eval_processor_from_json_files": mock_table_eval_processor_from_json_files,
+            "mock_supported_metric_names": mock_supported_metric_names,
+            "mock_path_exists": mock_path_exists,
+        }
+
+        # setup mocks
+        mocks["mock_calculate_accuracy"].return_value = 0.5
+        mocks["mock_calculate_percent_missing_text"].return_value = 0.5
+        mocks["mock_get_ccts"].return_value = ["output_cct", "source_cct"]
+        mocks["mock_get_element_type_frequency"].side_effect = [{"ele1": 1}, {"ele2": 3}]
+        mocks["mock_calculate_element_type_percent_match"].return_value = 0.5
+        mocks["mock_supported_metric_names"].return_value = ["table_level_acc"]
+        mocks["mock_path_exists"].return_value = True
+        mocks["mock_read_text_file"].side_effect = ["output_text", "source_text"]
+
+        yield mocks
+
+
+@pytest.fixture()
+def _cleanup_after_test():
+    """Fixture for removing side-effects of running tests in this file."""
+
+    def remove_generated_directories():
+        """Remove directories created from running tests."""
+
+        # Directories to be removed:
+        target_dir_names = [
+            "test_evaluate_results_cct",
+            "test_evaluate_results_cct_txt",
+            "test_evaluate_results_element_type",
+            "test_evaluate_result_table_structure",
+        ]
+        subdirs = (d for d in os.scandir(TESTING_FILE_DIR) if d.is_dir())
+        for d in subdirs:
+            if d.name in target_dir_names:
+                shutil.rmtree(d.path)
+
+    # Run test as normal
+    yield
+    remove_generated_directories()
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_text_extraction_evaluation():
+    output_dir = os.path.join(TESTING_FILE_DIR, UNSTRUCTURED_OUTPUT_DIRNAME)
+    source_dir = os.path.join(TESTING_FILE_DIR, GOLD_CCT_DIRNAME)
+    export_dir = os.path.join(TESTING_FILE_DIR, "test_evaluate_results_cct")
+
+    TextExtractionMetricsCalculator(
+        documents_dir=output_dir, ground_truths_dir=source_dir
+    ).calculate(export_dir=export_dir, visualize_progress=False, display_agg_df=False)
+
+    assert os.path.isfile(os.path.join(export_dir, "all-docs-cct.tsv"))
+    df = pd.read_csv(os.path.join(export_dir, "all-docs-cct.tsv"), sep="\t")
+    assert len(df) == 3
+    assert len(df.columns) == 5
+    assert df.iloc[0].filename == "Bank Good Credit Loan.pptx"
+
+
+@pytest.mark.parametrize(
+    ("calculator_class", "output_dirname", "source_dirname", "path", "expected_length", "kwargs"),
+    [
+        (
+            TextExtractionMetricsCalculator,
+            UNSTRUCTURED_CCT_DIRNAME,
+            GOLD_CCT_DIRNAME,
+            Path("Bank Good Credit Loan.pptx.txt"),
+            5,
+            {"document_type": "txt"},
+        ),
+        (
+            TableStructureMetricsCalculator,
+            UNSTRUCTURED_TABLE_STRUCTURE_DIRNAME,
+            GOLD_TABLE_STRUCTURE_DIRNAME,
+            Path("IRS-2023-Form-1095-A.pdf.json"),
+            14,
+            {},
+        ),
+        (
+            ElementTypeMetricsCalculator,
+            UNSTRUCTURED_OUTPUT_DIRNAME,
+            GOLD_ELEMENT_TYPE_DIRNAME,
+            Path("IRS-form-1987.pdf.json"),
+            4,
+            {},
+        ),
+    ],
+)
+def test_process_document_returns_the_correct_amount_of_values(
+    calculator_class, output_dirname, source_dirname, path, expected_length, kwargs
+):
+    output_dir = Path(TESTING_FILE_DIR) / output_dirname
+    source_dir = Path(TESTING_FILE_DIR) / source_dirname
+
+    calculator = calculator_class(documents_dir=output_dir, ground_truths_dir=source_dir, **kwargs)
+    output_list = calculator._process_document(path)
+    assert len(output_list) == expected_length
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test", "mock_dependencies")
+@pytest.mark.parametrize(
+    ("calculator_class", "output_dirname", "source_dirname", "path", "kwargs"),
+    [
+        (
+            TextExtractionMetricsCalculator,
+            UNSTRUCTURED_CCT_DIRNAME,
+            GOLD_CCT_DIRNAME,
+            Path("2310.03502text_to_image_synthesis1-7.pdf.txt"),
+            {"document_type": "txt"},
+        ),
+    ],
+)
+def test_TextExtractionMetricsCalculator_process_document_returns_the_correct_doctype(
+    mock_dependencies, calculator_class, output_dirname, source_dirname, path, kwargs
+):
+    output_dir = Path(TESTING_FILE_DIR) / output_dirname
+    source_dir = Path(TESTING_FILE_DIR) / source_dirname
+    mock_calculate_accuracy = mock_dependencies["mock_calculate_accuracy"]
+    mock_calculate_percent_missing_text = mock_dependencies["mock_calculate_percent_missing_text"]
+    mock_get_ccts = mock_dependencies["mock_get_ccts"]
+    calculator = calculator_class(documents_dir=output_dir, ground_truths_dir=source_dir, **kwargs)
+    output_list = calculator._process_document(path)
+    assert output_list[1] == ".pdf"
+    assert mock_calculate_accuracy.call_count == 1
+    assert mock_calculate_percent_missing_text.call_count == 1
+    assert mock_get_ccts.call_count == 1
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test", "mock_dependencies")
+@pytest.mark.parametrize(
+    ("calculator_class", "output_dirname", "source_dirname", "path", "kwargs"),
+    [
+        (
+            TableStructureMetricsCalculator,
+            UNSTRUCTURED_TABLE_STRUCTURE_DIRNAME,
+            GOLD_TABLE_STRUCTURE_DIRNAME,
+            Path("tablib-627mTABLES-2310.07875-p7.pdf.json"),
+            {},
+        ),
+        # (
+        #     ElementTypeMetricsCalculator,
+        #     UNSTRUCTURED_OUTPUT_DIRNAME,
+        #     GOLD_ELEMENT_TYPE_DIRNAME,
+        #     Path("IRS-form.1987.pdf.json"),
+        #     {},
+        # ),
+    ],
+)
+def test_TableStructureMetricsCalculator_process_document_returns_the_correct_doctype(
+    mock_dependencies, calculator_class, output_dirname, source_dirname, path, kwargs
+):
+    output_dir = Path(TESTING_FILE_DIR) / output_dirname
+    source_dir = Path(TESTING_FILE_DIR) / source_dirname
+    calculator = calculator_class(documents_dir=output_dir, ground_truths_dir=source_dir, **kwargs)
+    calculator._ground_truths_dir = source_dir
+    calculator._documents_dir = output_dir
+    calculator._ground_truth_paths = [source_dir / path]
+    mock_report = MagicMock()
+    mock_report.total_predicted_tables = 3
+    mock_report.table_evel_acc = 0.83
+    mock_table_eval_processor_from_json_files = mock_dependencies[
+        "mock_table_eval_processor_from_json_files"
+    ]
+    mock_table_eval_processor_from_json_files.return_value.process_file.return_value = mock_report
+
+    output_list = calculator._process_document(path)
+    assert output_list[1] == ".pdf"
+    assert mock_table_eval_processor_from_json_files.call_count == 1
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test", "mock_dependencies")
+@pytest.mark.parametrize(
+    ("calculator_class", "output_dirname", "source_dirname", "path", "kwargs"),
+    [
+        (
+            ElementTypeMetricsCalculator,
+            UNSTRUCTURED_OUTPUT_DIRNAME,
+            GOLD_ELEMENT_TYPE_DIRNAME,
+            Path("IRS-form.1987.pdf.json"),
+            {},
+        ),
+    ],
+)
+def test_ElementTypeMetricsCalculator_process_document_returns_the_correct_doctype(
+    mock_dependencies, calculator_class, output_dirname, source_dirname, path, kwargs
+):
+    output_dir = Path(TESTING_FILE_DIR) / output_dirname
+    source_dir = Path(TESTING_FILE_DIR) / source_dirname
+    calculator = calculator_class(documents_dir=output_dir, ground_truths_dir=source_dir, **kwargs)
+    mock_element_type_frequency = mock_dependencies["mock_get_element_type_frequency"]
+    mock_read_text_file = mock_dependencies["mock_read_text_file"]
+    mock_calculate_element_type_percent_match = mock_dependencies[
+        "mock_calculate_element_type_percent_match"
+    ]
+    output_list = calculator._process_document(path)
+    assert output_list[1] == ".pdf"
+    assert mock_read_text_file.call_count == 2
+    assert mock_element_type_frequency.call_count == 2
+    assert mock_calculate_element_type_percent_match.call_count == 1
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_text_extraction_evaluation_type_txt():
+    output_dir = os.path.join(TESTING_FILE_DIR, UNSTRUCTURED_CCT_DIRNAME)
+    source_dir = os.path.join(TESTING_FILE_DIR, GOLD_CCT_DIRNAME)
+    export_dir = os.path.join(TESTING_FILE_DIR, "test_evaluate_results_cct")
+
+    TextExtractionMetricsCalculator(
+        documents_dir=output_dir, ground_truths_dir=source_dir, document_type="txt"
+    ).calculate(export_dir=export_dir)
+
+    df = pd.read_csv(os.path.join(export_dir, "all-docs-cct.tsv"), sep="\t")
+    assert len(df) == 3
+    assert len(df.columns) == 5
+    assert df.iloc[0].filename == "Bank Good Credit Loan.pptx"
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_element_type_evaluation():
+    output_dir = os.path.join(TESTING_FILE_DIR, UNSTRUCTURED_OUTPUT_DIRNAME)
+    source_dir = os.path.join(TESTING_FILE_DIR, GOLD_ELEMENT_TYPE_DIRNAME)
+    export_dir = os.path.join(TESTING_FILE_DIR, "test_evaluate_results_cct")
+
+    ElementTypeMetricsCalculator(
+        documents_dir=output_dir,
+        ground_truths_dir=source_dir,
+    ).calculate(export_dir=export_dir, visualize_progress=False)
+
+    assert os.path.isfile(os.path.join(export_dir, "all-docs-element-type-frequency.tsv"))
+    df = pd.read_csv(os.path.join(export_dir, "all-docs-element-type-frequency.tsv"), sep="\t")
+    assert len(df) == 1
+    assert len(df.columns) == 4
+    assert df.iloc[0].filename == "IRS-form-1987.pdf"
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_table_structure_evaluation():
+    output_dir = os.path.join(TESTING_FILE_DIR, UNSTRUCTURED_TABLE_STRUCTURE_DIRNAME)
+    source_dir = os.path.join(TESTING_FILE_DIR, GOLD_TABLE_STRUCTURE_DIRNAME)
+    export_dir = os.path.join(TESTING_FILE_DIR, "test_evaluate_result_table_structure")
+
+    TableStructureMetricsCalculator(
+        documents_dir=output_dir,
+        ground_truths_dir=source_dir,
+    ).calculate(export_dir=export_dir, visualize_progress=False)
+
+    assert os.path.isfile(os.path.join(export_dir, "all-docs-table-structure-accuracy.tsv"))
+    assert os.path.isfile(os.path.join(export_dir, "aggregate-table-structure-accuracy.tsv"))
+    df = pd.read_csv(os.path.join(export_dir, "all-docs-table-structure-accuracy.tsv"), sep="\t")
+    agg_df = pd.read_csv(
+        os.path.join(export_dir, "aggregate-table-structure-accuracy.tsv"), sep="\t"
+    ).set_index("metric")
+    assert len(df) == 2
+    assert len(df.columns) == 15
+    assert df.iloc[1].filename == "IRS-2023-Form-1095-A.pdf"
+    assert (
+        np.round(np.average(df["table_level_acc"], weights=df["total_tables"]), 3)
+        == agg_df.loc["table_level_acc", "average"]
+    )
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_text_extraction_takes_list():
+    output_dir = os.path.join(TESTING_FILE_DIR, UNSTRUCTURED_OUTPUT_DIRNAME)
+    output_list = ["currency.csv.json"]
+    source_dir = os.path.join(TESTING_FILE_DIR, GOLD_CCT_DIRNAME)
+    export_dir = os.path.join(TESTING_FILE_DIR, "test_evaluate_results_cct")
+
+    TextExtractionMetricsCalculator(
+        documents_dir=output_dir,
+        ground_truths_dir=source_dir,
+    ).on_files(document_paths=output_list).calculate(export_dir=export_dir)
+
+    # check that only the listed files are included
+    assert os.path.isfile(os.path.join(export_dir, "all-docs-cct.tsv"))
+    df = pd.read_csv(os.path.join(export_dir, "all-docs-cct.tsv"), sep="\t")
+    assert len(df) == len(output_list)
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_text_extraction_with_grouping():
+    output_dir = os.path.join(TESTING_FILE_DIR, UNSTRUCTURED_OUTPUT_DIRNAME)
+    source_dir = os.path.join(TESTING_FILE_DIR, GOLD_CCT_DIRNAME)
+    export_dir = os.path.join(TESTING_FILE_DIR, "test_evaluate_results_cct")
+
+    TextExtractionMetricsCalculator(
+        documents_dir=output_dir,
+        ground_truths_dir=source_dir,
+        group_by="doctype",
+    ).calculate(export_dir=export_dir)
+
+    df = pd.read_csv(os.path.join(export_dir, "all-doctype-agg-cct.tsv"), sep="\t")
+    assert len(df) == 4  # metrics row and doctype rows
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_text_extraction_wrong_type():
+    output_dir = os.path.join(TESTING_FILE_DIR, UNSTRUCTURED_OUTPUT_DIRNAME)
+    source_dir = os.path.join(TESTING_FILE_DIR, GOLD_CCT_DIRNAME)
+    export_dir = os.path.join(TESTING_FILE_DIR, "test_evaluate_results_cct")
+    with pytest.raises(ValueError):
+        TextExtractionMetricsCalculator(
+            documents_dir=output_dir, ground_truths_dir=source_dir, document_type="invalid type"
+        ).calculate(export_dir=export_dir)
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+@pytest.mark.parametrize(("grouping", "count_row"), [("doctype", 3), ("connector", 2)])
+def test_get_mean_grouping_df_input(grouping: str, count_row: int):
+    export_dir = os.path.join(TESTING_FILE_DIR, "test_evaluate_results_cct")
+    get_mean_grouping(
+        group_by=grouping,
+        data_input=DUMMY_DF_CCT,
+        export_dir=export_dir,
+        eval_name="text_extraction",
+    )
+    grouped_df = pd.read_csv(os.path.join(export_dir, f"all-{grouping}-agg-cct.tsv"), sep="\t")
+    assert grouped_df[grouping].dropna().nunique() == count_row
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_get_mean_grouping_tsv_input():
+    output_dir = os.path.join(TESTING_FILE_DIR, UNSTRUCTURED_OUTPUT_DIRNAME)
+    source_dir = os.path.join(TESTING_FILE_DIR, GOLD_CCT_DIRNAME)
+    export_dir = os.path.join(TESTING_FILE_DIR, "test_evaluate_results_cct")
+
+    TextExtractionMetricsCalculator(
+        documents_dir=output_dir,
+        ground_truths_dir=source_dir,
+    ).calculate(export_dir=export_dir)
+
+    filename = os.path.join(export_dir, "all-docs-cct.tsv")
+    get_mean_grouping(
+        group_by="doctype",
+        data_input=filename,
+        export_dir=export_dir,
+        eval_name="text_extraction",
+    )
+    grouped_df = pd.read_csv(os.path.join(export_dir, "all-doctype-agg-cct.tsv"), sep="\t")
+    assert grouped_df["doctype"].dropna().nunique() == 3
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_get_mean_grouping_invalid_group():
+    output_dir = os.path.join(TESTING_FILE_DIR, UNSTRUCTURED_OUTPUT_DIRNAME)
+    source_dir = os.path.join(TESTING_FILE_DIR, GOLD_CCT_DIRNAME)
+    export_dir = os.path.join(TESTING_FILE_DIR, "test_evaluate_results_cct")
+
+    TextExtractionMetricsCalculator(
+        documents_dir=output_dir,
+        ground_truths_dir=source_dir,
+    ).calculate(export_dir=export_dir)
+
+    df = pd.read_csv(os.path.join(export_dir, "all-docs-cct.tsv"), sep="\t")
+    with pytest.raises(ValueError):
+        get_mean_grouping(
+            group_by="invalid",
+            data_input=df,
+            export_dir=export_dir,
+            eval_name="text_extraction",
+        )
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_text_extraction_grouping_empty_df():
+    empty_df = pd.DataFrame()
+    with pytest.raises(SystemExit):
+        get_mean_grouping("doctype", empty_df, "some_dir", eval_name="text_extraction")
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_get_mean_grouping_missing_grouping_column():
+    df_with_no_grouping = pd.DataFrame({"some_column": [1, 2, 3]})
+    with pytest.raises(SystemExit):
+        get_mean_grouping("doctype", df_with_no_grouping, "some_dir", "text_extraction")
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_get_mean_grouping_all_null_grouping_column():
+    df_with_null_grouping = pd.DataFrame({"doctype": [None, None, None]})
+    with pytest.raises(SystemExit):
+        get_mean_grouping("doctype", df_with_null_grouping, "some_dir", eval_name="text_extraction")
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_get_mean_grouping_invalid_eval_name():
+    with pytest.raises(ValueError):
+        get_mean_grouping("doctype", DUMMY_DF_ELEMENT_TYPE, "some_dir", eval_name="invalid")
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+@pytest.mark.parametrize(("group_by", "count_row"), [("doctype", 3), ("connector", 2)])
+def test_get_mean_grouping_element_type(group_by: str, count_row: int):
+    export_dir = os.path.join(TESTING_FILE_DIR, "test_evaluate_results_element_type")
+    get_mean_grouping(
+        group_by=group_by,
+        data_input=DUMMY_DF_ELEMENT_TYPE,
+        export_dir=export_dir,
+        eval_name="element_type",
+    )
+    grouped_df = pd.read_csv(
+        os.path.join(export_dir, f"all-{group_by}-agg-element-type.tsv"), sep="\t"
+    )
+    assert grouped_df[group_by].dropna().nunique() == count_row
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_filter_metrics():
+    with open(os.path.join(TESTING_FILE_DIR, "filter_list.txt"), "w") as file:
+        file.write("Bank Good Credit Loan.pptx\n")
+        file.write("Performance-Audit-Discussion.pdf\n")
+    export_dir = os.path.join(TESTING_FILE_DIR, "test_evaluate_results_cct")
+
+    filter_metrics(
+        data_input=DUMMY_DF_CCT,
+        filter_list=os.path.join(TESTING_FILE_DIR, "filter_list.txt"),
+        filter_by="filename",
+        export_filename="filtered_metrics.tsv",
+        export_dir=export_dir,
+        return_type="file",
+    )
+    filtered_df = pd.read_csv(os.path.join(export_dir, "filtered_metrics.tsv"), sep="\t")
+    assert len(filtered_df) == 2
+    assert filtered_df["filename"].iloc[0] == "Bank Good Credit Loan.pptx"
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_get_mean_grouping_all_file():
+    with open(os.path.join(TESTING_FILE_DIR, "filter_list.txt"), "w") as file:
+        file.write("Bank Good Credit Loan.pptx\n")
+        file.write("Performance-Audit-Discussion.pdf\n")
+    export_dir = os.path.join(TESTING_FILE_DIR, "test_evaluate_results_cct")
+
+    filter_metrics(
+        data_input=DUMMY_DF_CCT,
+        filter_list=["Bank Good Credit Loan.pptx", "Performance-Audit-Discussion.pdf"],
+        filter_by="filename",
+        export_filename="filtered_metrics.tsv",
+        export_dir=export_dir,
+        return_type="file",
+    )
+    filtered_df = pd.read_csv(os.path.join(export_dir, "filtered_metrics.tsv"), sep="\t")
+
+    get_mean_grouping(
+        group_by="all",
+        data_input=filtered_df,
+        export_dir=export_dir,
+        eval_name="text_extraction",
+        export_filename="two-filename-agg-cct.tsv",
+    )
+    grouped_df = pd.read_csv(os.path.join(export_dir, "two-filename-agg-cct.tsv"), sep="\t")
+
+    assert np.isclose(float(grouped_df.iloc[1, 0]), 0.903)
+    assert np.isclose(float(grouped_df.iloc[1, 1]), 0.129)
+    assert np.isclose(float(grouped_df.iloc[1, 2]), 0.091)
+
+
+@pytest.mark.skipif(is_in_docker, reason="Skipping this test in Docker container")
+@pytest.mark.usefixtures("_cleanup_after_test")
+def test_get_mean_grouping_all_file_txt():
+    with open(os.path.join(TESTING_FILE_DIR, "filter_list.txt"), "w") as file:
+        file.write("Bank Good Credit Loan.pptx\n")
+        file.write("Performance-Audit-Discussion.pdf\n")
+    export_dir = os.path.join(TESTING_FILE_DIR, "test_evaluate_results_cct")
+
+    filter_metrics(
+        data_input=DUMMY_DF_CCT,
+        filter_list=os.path.join(TESTING_FILE_DIR, "filter_list.txt"),
+        filter_by="filename",
+        export_filename="filtered_metrics.tsv",
+        export_dir=export_dir,
+        return_type="file",
+    )
+    filtered_df = pd.read_csv(os.path.join(export_dir, "filtered_metrics.tsv"), sep="\t")
+
+    get_mean_grouping(
+        group_by="all",
+        data_input=filtered_df,
+        export_dir=export_dir,
+        eval_name="text_extraction",
+        export_filename="two-filename-agg-cct.tsv",
+    )
+    grouped_df = pd.read_csv(os.path.join(export_dir, "two-filename-agg-cct.tsv"), sep="\t")
+
+    assert np.isclose(float(grouped_df.iloc[1, 0]), 0.903)
+    assert np.isclose(float(grouped_df.iloc[1, 1]), 0.129)
+    assert np.isclose(float(grouped_df.iloc[1, 2]), 0.091)

test_unstructured/metrics/test_table_alignment.py

@@ -0,0 +1,14 @@
+from unstructured.metrics.table.table_alignment import TableAlignment
+
+
+def test_get_element_level_alignment_when_no_match():
+    example_table = [{"row_index": 0, "col_index": 0, "content": "a"}]
+    metrics = TableAlignment.get_element_level_alignment(
+        predicted_table_data=[example_table],
+        ground_truth_table_data=[example_table],
+        matched_indices=[-1],
+    )
+    assert metrics["col_index_acc"] == 0
+    assert metrics["row_index_acc"] == 0
+    assert metrics["row_content_acc"] == 0
+    assert metrics["col_content_acc"] == 0

test_unstructured/metrics/test_table_detection_metrics.py

@@ -0,0 +1,33 @@
+import pytest
+
+from unstructured.metrics.table.table_eval import calculate_table_detection_metrics
+
+
+@pytest.mark.parametrize(
+    ("matched_indices", "ground_truth_tables_number", "expected_metrics"),
+    [
+        ([0, 1, 2], 3, (1, 1, 1)),  # everything was predicted correctly
+        ([2, 1, 0], 3, (1, 1, 1)),  # everything was predicted correctly
+        (
+            [-1, 2, -1, 1, 0, -1],
+            3,
+            (1, 0.5, 0.66),
+        ),  # some false positives, all tables matched, too many predictions
+        ([2, 2, 1, 1], 8, (0.25, 0.5, 0.33)),
+        # Some false negatives, all predictions matched with gt, not enough predictions
+        # The precision here is not 1 as only one from tables matched with '1' index can be correct
+        ([1, -1], 2, (0.5, 0.5, 0.5)),  # typical case with false positive and false negative
+        ([-1, -1, -1], 2, (0, 0, 0)),  # nothing was matched
+        ([-1, -1, -1], 0, (0, 0, 0)),  # there was no table in ground truth
+        ([], 0, (0, 0, 0)),  # just zeros to account for errors
+    ],
+)
+def test_calculate_table_metrics(matched_indices, ground_truth_tables_number, expected_metrics):
+    expected_recall, expected_precision, expected_f1 = expected_metrics
+    pred_recall, pred_precision, pred_f1 = calculate_table_detection_metrics(
+        matched_indices=matched_indices, ground_truth_tables_number=ground_truth_tables_number
+    )
+
+    assert pred_recall == expected_recall
+    assert pred_precision == expected_precision
+    assert pred_f1 == pytest.approx(expected_f1, abs=0.01)

test_unstructured/metrics/test_table_formats.py

@@ -0,0 +1,33 @@
+import pytest
+
+from unstructured.metrics.table.table_formats import SimpleTableCell
+
+
+@pytest.mark.parametrize(
+    ("row_nums", "column_nums", "x", "y", "w", "h"),
+    [
+        ([3, 2, 1], [6, 7], 6, 1, 2, 3),
+        ([2], [6, 7], 6, 2, 2, 1),
+        ([1, 2, 3], [20], 20, 1, 1, 3),
+        ([5], [5], 5, 5, 1, 1),
+    ],
+)
+def test_simple_table_cell_parsing_from_table_transformer_when_expected_input(
+    row_nums, column_nums, x, y, w, h
+):
+    table_transformer_cell = {"row_nums": row_nums, "column_nums": column_nums, "cell text": "text"}
+    transformed_cell = SimpleTableCell.from_table_transformer_cell(table_transformer_cell)
+    expected_cell = SimpleTableCell(x=x, y=y, w=w, h=h, content="text")
+    assert expected_cell == transformed_cell
+
+
+def test_simple_table_cell_parsing_from_table_transformer_when_missing_row_nums():
+    cell = {"row_nums": [], "column_nums": [1], "cell text": "text"}
+    with pytest.raises(ValueError, match='has missing values under "row_nums" key'):
+        SimpleTableCell.from_table_transformer_cell(cell)
+
+
+def test_simple_table_cell_parsing_from_table_transformer_when_missing_column_nums():
+    cell = {"row_nums": [1], "column_nums": [], "cell text": "text"}
+    with pytest.raises(ValueError, match='has missing values under "column_nums" key'):
+        SimpleTableCell.from_table_transformer_cell(cell)

test_unstructured/metrics/test_table_structure.py

@@ -0,0 +1,700 @@
+from unittest import mock
+
+import numpy as np
+import pytest
+
+from test_unstructured.unit_utils import example_doc_path
+from unstructured.metrics.table.table_alignment import TableAlignment
+from unstructured.metrics.table.table_eval import TableEvalProcessor
+from unstructured.metrics.table_structure import (
+    eval_table_transformer_for_file,
+    image_or_pdf_to_dataframe,
+)
+
+
+@pytest.mark.parametrize(
+    "filename",
+    [
+        example_doc_path("img/table-multi-row-column-cells.png"),
+        example_doc_path("pdf/table-multi-row-column-cells.pdf"),
+    ],
+)
+def test_image_or_pdf_to_dataframe(filename):
+    df = image_or_pdf_to_dataframe(filename)
+    assert ["Blind", "5", "1", "4", "34.5%, n=1", "1199 sec, n=1"] in df.values
+
+
+def test_eval_table_transformer_for_file():
+    score = eval_table_transformer_for_file(
+        example_doc_path("img/table-multi-row-column-cells.png"),
+        example_doc_path("table-multi-row-column-cells-actual.csv"),
+    )
+    # avoid severe degradation of performance
+    assert 0.8 < score < 1
+
+
+def test_table_eval_processor_simple():
+    prediction = [
+        {
+            "type": "Table",
+            "metadata": {
+                "text_as_html": """<table><thead><tr><th>r1c1</th><th>r1c2</th></tr></thead>
+                    <tbody><tr><td>r2c1</td><td>r2c2</td></tr></tbody></table>"""
+            },
+        }
+    ]
+
+    ground_truth = [
+        {
+            "type": "Table",
+            "text": [
+                {
+                    "id": "ee862c7a-d27e-4484-92de-4faa42a63f3b",
+                    "x": 0,
+                    "y": 0,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r1c1",
+                },
+                {
+                    "id": "6237ac7b-bfc8-40d2-92f2-d138277205e2",
+                    "x": 0,
+                    "y": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r2c1",
+                },
+                {
+                    "id": "9d0933a9-5984-4cad-80d9-6752bf9bc4df",
+                    "x": 1,
+                    "y": 0,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r1c2",
+                },
+                {
+                    "id": "1152d043-5ead-4ab8-8b88-888d48831ac2",
+                    "x": 1,
+                    "y": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r2c2",
+                },
+            ],
+        }
+    ]
+
+    te_processor = TableEvalProcessor(prediction, ground_truth)
+    result = te_processor.process_file()
+    assert result.total_tables == 1
+    assert result.table_level_acc == 1.0
+    assert result.element_row_level_index_acc == 1.0
+    assert result.element_col_level_index_acc == 1.0
+    assert result.element_row_level_content_acc == 1.0
+    assert result.element_col_level_content_acc == 1.0
+
+
+def test_table_eval_processor_simple_when_input_as_cells():
+    prediction = [
+        {
+            "type": "Table",
+            "metadata": {
+                "table_as_cells": [
+                    {
+                        "x": 1,
+                        "y": 1,
+                        "w": 1,
+                        "h": 1,
+                        "content": "r2c2",
+                    },
+                    {
+                        "x": 0,
+                        "y": 0,
+                        "w": 1,
+                        "h": 1,
+                        "content": "r1c1",
+                    },
+                    {
+                        "x": 0,
+                        "y": 1,
+                        "w": 1,
+                        "h": 1,
+                        "content": "r2c1",
+                    },
+                    {
+                        "x": 1,
+                        "y": 0,
+                        "w": 1,
+                        "h": 1,
+                        "content": "r1c2",
+                    },
+                ]
+            },
+        }
+    ]
+
+    ground_truth = [
+        {
+            "type": "Table",
+            "text": [
+                {
+                    "id": "ee862c7a-d27e-4484-92de-4faa42a63f3b",
+                    "x": 0,
+                    "y": 0,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r1c1",
+                },
+                {
+                    "id": "6237ac7b-bfc8-40d2-92f2-d138277205e2",
+                    "x": 0,
+                    "y": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r2c1",
+                },
+                {
+                    "id": "9d0933a9-5984-4cad-80d9-6752bf9bc4df",
+                    "x": 1,
+                    "y": 0,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r1c2",
+                },
+                {
+                    "id": "1152d043-5ead-4ab8-8b88-888d48831ac2",
+                    "x": 1,
+                    "y": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r2c2",
+                },
+            ],
+        }
+    ]
+
+    te_processor = TableEvalProcessor(prediction, ground_truth, source_type="cells")
+    result = te_processor.process_file()
+    assert result.total_tables == 1
+    assert result.table_level_acc == 1.0
+    assert result.element_row_level_index_acc == 1.0
+    assert result.element_col_level_index_acc == 1.0
+    assert result.element_row_level_content_acc == 1.0
+    assert result.element_col_level_content_acc == 1.0
+
+
+def test_table_eval_processor_when_wrong_source_type():
+    prediction = [
+        {
+            "type": "Table",
+            "metadata": {"table_as_cells": []},
+        }
+    ]
+
+    ground_truth = [
+        {
+            "type": "Table",
+            "text": [],
+        }
+    ]
+
+    te_processor = TableEvalProcessor(prediction, ground_truth, source_type="wrong_type")
+    with pytest.raises(ValueError):
+        te_processor.process_file()
+
+
+@pytest.mark.parametrize(
+    "text_as_html",
+    [
+        """
+<table>
+    <thead>
+        <tr>
+            <th>r1c1</th>
+            <th>r1c2</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>r2c1</td>
+            <td>r2c2</td>
+        </tr>
+        <tr>
+            <td>r3c1</td>
+            <td>r3c2</td>
+        </tr>
+    </tbody>
+</table>
+""",
+        """
+<table>
+    <tr>
+        <th>r1c1</th>
+        <th>r1c2</th>
+    </tr>
+    <tbody>
+        <tr>
+            <td>r2c1</td>
+            <td>r2c2</td>
+        </tr>
+        <tr>
+            <td>r3c1</td>
+            <td>r3c2</td>
+        </tr>
+    </tbody>
+</table>
+""",
+        """
+<table>
+    </tbody>
+        <tr>
+            <td>r1c1</td>
+            <td>r1c2</td>
+        </tr>
+        <tr>
+            <td>r2c1</td>
+            <td>r2c2</td>
+        </tr>
+        <tr>
+            <td>r3c1</td>
+            <td>r3c2</td>
+        </tr>
+    </tbody>
+</table>
+""",
+    ],
+)
+def test_table_eval_processor_various_table_html_structures(text_as_html):
+    prediction = [{"type": "Table", "metadata": {"text_as_html": text_as_html}}]
+
+    ground_truth = [
+        {
+            "type": "Table",
+            "text": [
+                {
+                    "id": "ee862c7a-d27e-4484-92de-4faa42a63f3b",
+                    "x": 0,
+                    "y": 0,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r1c1",
+                },
+                {
+                    "id": "6237ac7b-bfc8-40d2-92f2-d138277205e2",
+                    "x": 0,
+                    "y": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r2c1",
+                },
+                {
+                    "id": "9d0933a9-5984-4cad-80d9-6752bf9bc4df",
+                    "x": 1,
+                    "y": 0,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r1c2",
+                },
+                {
+                    "id": "1152d043-5ead-4ab8-8b88-888d48831ac2",
+                    "x": 1,
+                    "y": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r2c2",
+                },
+                {
+                    "id": "364f4a17-2979-4506-ae77-e8adf8e3f554",
+                    "x": 0,
+                    "y": 2,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r3c1",
+                },
+                {
+                    "id": "30f87503-ac1f-4db1-b924-b316af585702",
+                    "x": 1,
+                    "y": 2,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r3c2",
+                },
+            ],
+        }
+    ]
+
+    te_processor = TableEvalProcessor(prediction, ground_truth)
+    result = te_processor.process_file()
+    assert result.total_tables == 1
+    assert result.table_level_acc == 1.0
+    assert result.element_row_level_index_acc == 1.0
+    assert result.element_col_level_index_acc == 1.0
+    assert result.element_row_level_content_acc == 1.0
+    assert result.element_col_level_content_acc == 1.0
+
+
+def test_table_eval_processor_non_str_values_in_table():
+    prediction = [
+        {
+            "type": "Table",
+            "metadata": {
+                "text_as_html": """
+<table>
+    <thead>
+        <tr>
+            <th>11</th>
+            <th>12</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>21</td>
+            <td>22</td>
+        </tr>
+    </tbody>
+</table>"""
+            },
+        }
+    ]
+
+    ground_truth = [
+        {
+            "type": "Table",
+            "text": [
+                {
+                    "id": "ee862c7a-d27e-4484-92de-4faa42a63f3b",
+                    "x": 0,
+                    "y": 0,
+                    "w": 1,
+                    "h": 1,
+                    "content": "11",
+                },
+                {
+                    "id": "6237ac7b-bfc8-40d2-92f2-d138277205e2",
+                    "x": 0,
+                    "y": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "21",
+                },
+                {
+                    "id": "9d0933a9-5984-4cad-80d9-6752bf9bc4df",
+                    "x": 1,
+                    "y": 0,
+                    "w": 1,
+                    "h": 1,
+                    "content": "12",
+                },
+                {
+                    "id": "1152d043-5ead-4ab8-8b88-888d48831ac2",
+                    "x": 1,
+                    "y": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "22",
+                },
+            ],
+        }
+    ]
+
+    te_processor = TableEvalProcessor(prediction, ground_truth)
+    result = te_processor.process_file()
+    assert result.total_tables == 1
+    assert result.table_level_acc == 1.0
+    assert result.element_row_level_index_acc == 1.0
+    assert result.element_col_level_index_acc == 1.0
+    assert result.element_row_level_content_acc == 1.0
+    assert result.element_col_level_content_acc == 1.0
+
+
+def test_table_eval_processor_merged_cells():
+    prediction = [
+        {
+            "type": "Table",
+            "metadata": {
+                "text_as_html": """
+<table>
+    <thead>
+        <tr>
+            <th rowspan="2">r1c1</th>
+            <th>r1c2</th>
+            <th colspan="2">r1c3</th>
+        </tr>
+        <tr>
+            <th>r2c2</th>
+            <th>r2c3</th>
+            <th>r2c4</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>r3c1</td>
+            <td>r3c2</td>
+            <td colspan="2" rowspan="2">r3c3</td>
+        </tr>
+        <tr>
+            <td>r4c1</td>
+            <td>r4c2</td>
+        </tr>
+    </tbody>
+</table>
+"""
+            },
+        }
+    ]
+
+    ground_truth = [
+        {
+            "type": "Table",
+            "text": [
+                {
+                    "id": "f399ef57-5b88-4509-8971-9cb63246866e",
+                    "x": 0,
+                    "y": 0,
+                    "w": 1,
+                    "h": 2,
+                    "content": "r1c1",
+                },
+                {
+                    "id": "2dfdec2f-e8f3-4be7-a6ac-8ff21c4e8556",
+                    "x": 0,
+                    "y": 2,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r3c1",
+                },
+                {
+                    "id": "9c771c58-88c7-49d8-9c12-85d0e44b920e",
+                    "x": 0,
+                    "y": 3,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r4c1",
+                },
+                {
+                    "id": "5bd6f3f0-34c5-495b-8a28-c4ac96989ef8",
+                    "x": 1,
+                    "y": 0,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r1c2",
+                },
+                {
+                    "id": "7b8e6bc2-a310-4dd6-997c-313f951e7f96",
+                    "x": 1,
+                    "y": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r2c2",
+                },
+                {
+                    "id": "1c152ad4-12fa-4a7b-90de-a992aa6410a4",
+                    "x": 1,
+                    "y": 2,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r3c2",
+                },
+                {
+                    "id": "55063f64-0003-4217-b6ca-aff5914793ff",
+                    "x": 1,
+                    "y": 3,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r4c2",
+                },
+                {
+                    "id": "22852e86-0e22-4d32-b63a-9ba7dd4118a2",
+                    "x": 2,
+                    "y": 0,
+                    "w": 2,
+                    "h": 1,
+                    "content": "r1c3",
+                },
+                {
+                    "id": "eae013c5-5597-4a8b-9771-82e28c5c5cba",
+                    "x": 2,
+                    "y": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r2c3",
+                },
+                {
+                    "id": "0dea3a42-8523-4d6e-9e70-d65cc2314678",
+                    "x": 2,
+                    "y": 2,
+                    "w": 2,
+                    "h": 2,
+                    "content": "r3c3",
+                },
+                {
+                    "id": "60093e2c-d3e2-4146-92b5-97a2fc16c061",
+                    "x": 3,
+                    "y": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r2c4",
+                },
+            ],
+        }
+    ]
+
+    te_processor = TableEvalProcessor(prediction, ground_truth)
+    result = te_processor.process_file()
+    assert result.total_tables == 1
+    assert result.table_level_acc == 1.0
+    assert result.element_row_level_index_acc == 1.0
+    assert result.element_col_level_index_acc == 1.0
+    assert result.element_row_level_content_acc == 1.0
+    assert result.element_col_level_content_acc == 1.0
+
+
+def test_table_eval_processor_when_no_match_with_pred():
+    prediction = [
+        {
+            "type": "Table",
+            "metadata": {"text_as_html": """<table><tr><td>Some cell</td></tr></table>"""},
+        }
+    ]
+
+    ground_truth = [
+        {
+            "type": "Table",
+            "text": [
+                {
+                    "id": "ee862c7a-d27e-4484-92de-4faa42a63f3b",
+                    "x": 0,
+                    "y": 0,
+                    "w": 1,
+                    "h": 1,
+                    "content": "11",
+                },
+                {
+                    "id": "6237ac7b-bfc8-40d2-92f2-d138277205e2",
+                    "x": 0,
+                    "y": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "21",
+                },
+                {
+                    "id": "9d0933a9-5984-4cad-80d9-6752bf9bc4df",
+                    "x": 1,
+                    "y": 0,
+                    "w": 1,
+                    "h": 1,
+                    "content": "12",
+                },
+                {
+                    "id": "1152d043-5ead-4ab8-8b88-888d48831ac2",
+                    "x": 1,
+                    "y": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "22",
+                },
+            ],
+        }
+    ]
+
+    with mock.patch.object(TableAlignment, "get_table_level_alignment") as align_fn:
+        align_fn.return_value = [-1]
+        te_processor = TableEvalProcessor(prediction, ground_truth)
+        result = te_processor.process_file()
+
+    assert result.total_tables == 1
+    assert result.table_level_acc == 0
+    assert result.element_row_level_index_acc == 0
+    assert result.element_col_level_index_acc == 0
+    assert result.element_row_level_content_acc == 0
+    assert result.element_col_level_content_acc == 0
+
+
+def test_table_eval_processor_when_no_tables():
+    prediction = [{}]
+
+    ground_truth = [{}]
+
+    te_processor = TableEvalProcessor(prediction, ground_truth)
+    result = te_processor.process_file()
+    assert result.total_tables == 0
+    assert result.table_level_acc == 1
+    assert np.isnan(result.element_row_level_index_acc)
+    assert np.isnan(result.element_col_level_index_acc)
+    assert np.isnan(result.element_row_level_content_acc)
+    assert np.isnan(result.element_col_level_content_acc)
+
+
+def test_table_eval_processor_when_only_gt():
+    prediction = []
+
+    ground_truth = [
+        {
+            "type": "Table",
+            "text": [
+                {
+                    "id": "ee862c7a-d27e-4484-92de-4faa42a63f3b",
+                    "x": 0,
+                    "y": 0,
+                    "w": 1,
+                    "h": 1,
+                    "content": "11",
+                },
+                {
+                    "id": "6237ac7b-bfc8-40d2-92f2-d138277205e2",
+                    "x": 0,
+                    "y": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "21",
+                },
+                {
+                    "id": "9d0933a9-5984-4cad-80d9-6752bf9bc4df",
+                    "x": 1,
+                    "y": 0,
+                    "w": 1,
+                    "h": 1,
+                    "content": "12",
+                },
+                {
+                    "id": "1152d043-5ead-4ab8-8b88-888d48831ac2",
+                    "x": 1,
+                    "y": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "22",
+                },
+            ],
+        }
+    ]
+
+    te_processor = TableEvalProcessor(prediction, ground_truth)
+    result = te_processor.process_file()
+
+    assert result.total_tables == 1
+    assert result.table_level_acc == 0
+    assert result.element_row_level_index_acc == 0
+    assert result.element_col_level_index_acc == 0
+    assert result.element_row_level_content_acc == 0
+    assert result.element_col_level_content_acc == 0
+
+
+def test_table_eval_processor_when_only_pred():
+    prediction = [
+        {
+            "type": "Table",
+            "metadata": {"text_as_html": """<table><tr><td>Some cell</td></tr></table>"""},
+        }
+    ]
+
+    ground_truth = [{}]
+
+    te_processor = TableEvalProcessor(prediction, ground_truth)
+    result = te_processor.process_file()
+
+    assert result.total_tables == 0
+    assert result.table_level_acc == 0
+    assert result.element_row_level_index_acc == 0
+    assert result.element_col_level_index_acc == 0
+    assert result.element_row_level_content_acc == 0
+    assert result.element_col_level_content_acc == 0

test_unstructured/metrics/test_text_extraction.py

@@ -0,0 +1,891 @@
+import re
+
+import pytest
+
+from unstructured.metrics import text_extraction
+from unstructured.metrics.table.table_extraction import (
+    deckerd_table_to_html,
+    extract_cells_from_table_as_cells,
+    extract_cells_from_text_as_html,
+    html_table_to_deckerd,
+)
+from unstructured.partition.auto import partition
+
+
+def test_calculate_edit_distance():
+    source_cct = "I like pizza. I like bagels."
+    source_cct_word_space = "I like p i z z a . I like bagles."
+    source_cct_spaces = re.sub(r"\s+", " ", " ".join(source_cct))
+    source_cct_no_space = source_cct.replace(" ", "")
+    source_cct_one_sentence = "I like pizza."
+    source_cct_missing_word = "I like pizza. I like ."
+    source_cct_addn_char = "I like pizza. I like beagles."
+    source_cct_dup_word = "I like pizza pizza. I like bagels."
+
+    assert (
+        round(text_extraction.calculate_edit_distance(source_cct, source_cct, return_as="score"), 2)
+        == 1.0
+    )
+    assert (
+        round(
+            text_extraction.calculate_edit_distance(
+                source_cct_word_space,
+                source_cct,
+                return_as="score",
+            ),
+            2,
+        )
+        == 0.75
+    )
+    assert (
+        round(
+            text_extraction.calculate_edit_distance(
+                source_cct_spaces,
+                source_cct,
+                return_as="score",
+            ),
+            2,
+        )
+        == 0.39
+    )
+    assert (
+        round(
+            text_extraction.calculate_edit_distance(
+                source_cct_no_space,
+                source_cct,
+                return_as="score",
+            ),
+            2,
+        )
+        == 0.64
+    )
+    assert (
+        round(
+            text_extraction.calculate_edit_distance(
+                source_cct_one_sentence,
+                source_cct,
+                return_as="score",
+            ),
+            2,
+        )
+        == 0.0
+    )
+    assert (
+        round(
+            text_extraction.calculate_edit_distance(
+                source_cct_missing_word,
+                source_cct,
+                return_as="score",
+            ),
+            2,
+        )
+        == 0.57
+    )
+    assert (
+        round(
+            text_extraction.calculate_edit_distance(
+                source_cct_addn_char,
+                source_cct,
+                return_as="score",
+            ),
+            2,
+        )
+        == 0.89
+    )
+    assert (
+        round(
+            text_extraction.calculate_edit_distance(
+                source_cct_dup_word,
+                source_cct,
+                return_as="score",
+            ),
+            2,
+        )
+        == 0.79
+    )
+
+
+@pytest.mark.parametrize(
+    ("filename", "standardize_whitespaces", "expected_score", "expected_distance"),
+    [
+        ("fake-text.txt", False, 0.78, 38),
+        ("fake-text.txt", True, 0.92, 12),
+    ],
+)
+def test_calculate_edit_distance_with_filename(
+    filename, standardize_whitespaces, expected_score, expected_distance
+):
+    with open("example-docs/fake-text.txt") as f:
+        source_cct = f.read()
+
+    elements = partition(filename=f"example-docs/{filename}")
+    output_cct = "\n".join([str(el) for el in elements])
+
+    score = text_extraction.calculate_edit_distance(
+        output_cct, source_cct, return_as="score", standardize_whitespaces=standardize_whitespaces
+    )
+    distance = text_extraction.calculate_edit_distance(
+        output_cct,
+        source_cct,
+        return_as="distance",
+        standardize_whitespaces=standardize_whitespaces,
+    )
+
+    assert score >= 0
+    assert score <= 1.0
+    assert distance >= 0
+    assert round(score, 2) == expected_score
+    assert distance == expected_distance
+
+
+@pytest.mark.parametrize(
+    ("text1", "text2"),
+    [
+        (
+            "The  dog\rloved the cat, but\t\n    the cat\tloved the\n cow",
+            "The dog loved the cat, but the cat loved the cow",
+        ),
+        (
+            "Hello    my\tname\tis H a r p e r, \nwhat's your\vname?",
+            "Hello my name is H a r p e r, what's your name?",
+        ),
+        (
+            "I have a\t\n\tdog and a\tcat,\fI love my\n\n\n\ndog.",
+            "I have a dog and a cat, I love my dog.",
+        ),
+        (
+            """
+            Name    Age City           Occupation
+            Alice   30  New York       Engineer
+            Bob     25  Los Angeles    Designer
+            Charlie 35  Chicago        Teacher
+            David   40  San Francisco  Developer
+            """,
+            """
+            Name\tAge\tCity\tOccupation
+            Alice\t30\tNew York\tEngineer
+            Bob\t25\tLos Angeles\tDesigner
+            Charlie\t35\tChicago\tTeacher
+            David\t40\tSan Francisco\tDeveloper
+            """,
+        ),
+        (
+            """
+            Name\tAge\tCity\tOccupation
+            Alice\t30\tNew York\tEngineer
+            Bob\t25\tLos Angeles\tDesigner
+            Charlie\t35\tChicago\tTeacher
+            David\t40\tSan Francisco\tDeveloper
+            """,
+            "Name\tAge\tCity\tOccupation\n\n \nAlice\t30\tNew York\tEngineer\nBob\t25\tLos Angeles\tDesigner\nCharlie\t35\tChicago\tTeacher\nDavid\t40\tSan Francisco\tDeveloper",  # noqa: E501
+        ),
+    ],
+)
+def test_calculate_edit_distance_with_various_whitespace_1(text1, text2):
+    assert (
+        text_extraction.calculate_edit_distance(
+            text1, text2, return_as="score", standardize_whitespaces=True
+        )
+        == 1.0
+    )
+    assert (
+        text_extraction.calculate_edit_distance(
+            text1, text2, return_as="distance", standardize_whitespaces=True
+        )
+        == 0
+    )
+    assert (
+        text_extraction.calculate_edit_distance(
+            text1, text2, return_as="score", standardize_whitespaces=False
+        )
+        < 1.0
+    )
+    assert (
+        text_extraction.calculate_edit_distance(
+            text1, text2, return_as="distance", standardize_whitespaces=False
+        )
+        > 0
+    )
+
+
+def test_calculate_edit_distance_with_various_whitespace_2():
+    source_cct_tabs = """
+            Name\tAge\tCity\tOccupation
+            Alice\t30\tNew York\tEngineer
+            Bob\t25\tLos Angeles\tDesigner
+            Charlie\t35\tChicago\tTeacher
+            David\t40\tSan Francisco\tDeveloper
+            """
+    source_cct_with_borders = """
+
+            | Name    | Age | City         | Occupation     |
+            |---------|-----|--------------|----------------|
+            | Alice   | 30  | New York     | Engineer       |
+            | Bob     | 25  | Los Angeles  | Designer       |
+            | Charlie | 35  | Chicago      | Teacher        |
+            | David   | 40  | San Francisco| Developer      |
+
+            """
+    assert text_extraction.calculate_edit_distance(
+        source_cct_tabs, source_cct_with_borders, return_as="score", standardize_whitespaces=True
+    ) > text_extraction.calculate_edit_distance(
+        source_cct_tabs, source_cct_with_borders, return_as="score", standardize_whitespaces=False
+    )
+    assert text_extraction.calculate_edit_distance(
+        source_cct_tabs, source_cct_with_borders, return_as="distance", standardize_whitespaces=True
+    ) < text_extraction.calculate_edit_distance(
+        source_cct_tabs,
+        source_cct_with_borders,
+        return_as="distance",
+        standardize_whitespaces=False,
+    )
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        (
+            "The dog loved the cat, but the cat loved the cow",
+            {"the": 4, "cat": 2, "loved": 2, "dog": 1, "but": 1, "cow": 1},
+        ),
+        (
+            "Hello my name is H a r p e r, what's your name?",
+            {"hello": 1, "my": 1, "name": 2, "is": 1, "what's": 1, "your": 1},
+        ),
+        (
+            "I have a dog and a cat, I love my dog.",
+            {"i": 2, "have": 1, "a": 2, "dog": 2, "and": 1, "cat": 1, "love": 1, "my": 1},
+        ),
+        (
+            "My dog's hair is red, but the dogs' houses are blue.",
+            {
+                "my": 1,
+                "dog's": 1,
+                "hair": 1,
+                "is": 1,
+                "red": 1,
+                "but": 1,
+                "the": 1,
+                "dogs'": 1,
+                "houses": 1,
+                "are": 1,
+                "blue": 1,
+            },
+        ),
+        (
+            """Sometimes sentences have a dash - like this one!
+                    A hyphen connects 2 words with no gap: easy-peasy.""",
+            {
+                "sometimes": 1,
+                "sentences": 1,
+                "have": 1,
+                "a": 2,
+                "dash": 1,
+                "like": 1,
+                "this": 1,
+                "one": 1,
+                "hyphen": 1,
+                "connects": 1,
+                "2": 1,
+                "words": 1,
+                "with": 1,
+                "no": 1,
+                "gap": 1,
+                "easy-peasy": 1,
+            },
+        ),
+    ],
+)
+def test_bag_of_words(text, expected):
+    assert text_extraction.bag_of_words(text) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        (
+            "The  dog\rloved the cat, but\t\n    the cat\tloved the\n cow\n\n",
+            "The dog loved the cat, but the cat loved the cow",
+        ),
+        (
+            "\n\nHello    my\tname\tis H a r p e r, \nwhat's your\vname?",
+            "Hello my name is H a r p e r, what's your name?",
+        ),
+        (
+            "I have a\t\n\tdog and a\tcat,\fI love my\n\n\n\ndog.",
+            "I have a dog and a cat, I love my dog.",
+        ),
+        (
+            """L     is for the way you look at me
+            O    is for the only one I see
+            V    is very, very extraordinary
+            E    is even more than anyone that you adore can""",
+            "L is for the way you look at me O is for the only one I see V is very, very extraordinary E is even more than anyone that you adore can",  # noqa: E501
+        ),
+        (
+            """
+            | Name    | Age | City         | Occupation     |
+            |---------|-----|--------------|----------------|
+            | Alice   | 30  | New York     | Engineer       |
+            | Bob     | 25  | Los Angeles  | Designer       |
+            | Charlie | 35  | Chicago      | Teacher        |
+            | David   | 40  | San Francisco| Developer      |
+            """,
+            "| Name | Age | City | Occupation | |---------|-----|--------------|----------------| | Alice | 30 | New York | Engineer | | Bob | 25 | Los Angeles | Designer | | Charlie | 35 | Chicago | Teacher | | David | 40 | San Francisco| Developer |",  # noqa: E501
+        ),
+    ],
+)
+def test_prepare_string(text, expected):
+    assert text_extraction.prepare_str(text, standardize_whitespaces=True) == expected
+    assert text_extraction.prepare_str(text) == text
+
+
+@pytest.mark.parametrize(
+    ("input_text", "expected_output"),
+    [
+        # Mixed quotes in longer sentences
+        (
+            "She said \"Hello\" and then whispered 'Goodbye' before leaving.",
+            "She said \"Hello\" and then whispered 'Goodbye' before leaving.",
+        ),
+        # Double low-9 quotes with complex content
+        (
+            "„To be, or not to be, that is the question\" - Shakespeare's famous quote.",
+            '"To be, or not to be, that is the question" - Shakespeare\'s famous quote.',
+        ),
+        # Angle quotes with nested quotes
+        (
+            '«When he said "life is beautiful," I believed him» wrote Maria.',
+            '"When he said "life is beautiful," I believed him" wrote Maria.',
+        ),
+        # Heavy ornament quotes in dialogue
+        (
+            "❝Do you remember when we first met?❞ she asked with a smile.",
+            '"Do you remember when we first met?" she asked with a smile.',
+        ),
+        # Double prime quotes with punctuation
+        (
+            "〝The meeting starts at 10:00, don't be late!〟 announced the manager.",
+            '"The meeting starts at 10:00, don\'t be late!" announced the manager.',
+        ),
+        # Corner brackets with nested quotes
+        (
+            '「He told me "This is important" yesterday」, she explained.',
+            "'He told me \"This is important\" yesterday', she explained.",
+        ),
+        # White corner brackets with multiple sentences
+        (
+            "『The sun was setting. The birds were singing. It was peaceful.』",
+            "'The sun was setting. The birds were singing. It was peaceful.'",
+        ),
+        # Vertical corner brackets with numbers and special characters
+        ("﹂Meeting #123 @ 15:00 - Don't forget!﹁", "'Meeting #123 @ 15:00 - Don't forget!'"),
+        # Complex mixed quote types
+        (
+            '「Hello」, ❝World❞, "Test", \'Example\', „Quote", «Final»',
+            '\'Hello\', "World", "Test", \'Example\', "Quote", "Final"',
+        ),
+        # Quotes with multiple apostrophes
+        ("It's John's book, isn't it?", "It's John's book, isn't it?"),
+        # Single angle quotes with nested content
+        (
+            '‹Testing the system\'s capability for "quoted" text›',
+            "'Testing the system's capability for \"quoted\" text'",
+        ),
+        # Heavy single ornament quotes with multiple sentences
+        (
+            "❛First sentence. Second sentence. Third sentence.❜",
+            "'First sentence. Second sentence. Third sentence.'",
+        ),
+        # Mix of various quote types in complex text
+        (
+            '「Chapter 1」: ❝The Beginning❞ - „A new story" begins «today».',
+            '\'Chapter 1\': "The Beginning" - "A new story" begins "today".',
+        ),
+        # --- Regression: U+201C / U+2018 were silently dropped by duplicate dict keys ---
+        # U+201C left double quotation mark (isolated)
+        ("\u201c", '"'),
+        # U+2018 left single quotation mark (isolated)
+        ("\u2018", "'"),
+        # Left + right double smart quotes wrapping a word
+        ("\u201cHello\u201d", '"Hello"'),
+        # Left + right single smart quotes wrapping a word
+        ("\u2018world\u2019", "'world'"),
+        # Mixed left/right smart quotes in a sentence
+        (
+            "She said \u201cHello\u201d and then whispered \u2018Goodbye\u2019",
+            "She said \"Hello\" and then whispered 'Goodbye'",
+        ),
+        # Possessive with left single smart quote
+        ("\u2018tis the season", "'tis the season"),
+    ],
+)
+def test_standardize_quotes(input_text, expected_output):
+    assert text_extraction.standardize_quotes(input_text) == expected_output
+
+
+@pytest.mark.parametrize(
+    ("codepoint", "expected"),
+    [
+        pytest.param(cp, '"', id=f"U+{ord(cp):04X}->double")
+        for cp in text_extraction._DOUBLE_QUOTE_CODEPOINTS
+    ]
+    + [
+        pytest.param(cp, "'", id=f"U+{ord(cp):04X}->single")
+        for cp in text_extraction._SINGLE_QUOTE_CODEPOINTS
+    ],
+)
+def test_standardize_quotes_every_codepoint(codepoint, expected):
+    """Every codepoint in the translation table must map to its ASCII equivalent."""
+    assert text_extraction.standardize_quotes(codepoint) == expected
+
+
+@pytest.mark.parametrize(
+    ("output_text", "source_text", "expected_percentage"),
+    [
+        (
+            "extra",
+            "",
+            0,
+        ),
+        (
+            "",
+            "Source text has a sentence.",
+            1,
+        ),
+        (
+            "The original s e n t e n c e is normal.",
+            "The original sentence is normal...",
+            0.2,
+        ),
+        (
+            "We saw 23% improvement in this quarter.",
+            "We saw 23% improvement in sales this quarter.",
+            0.125,
+        ),
+        (
+            "no",
+            "Is it possible to have more than everything missing?",
+            1,
+        ),
+    ],
+)
+def test_calculate_percent_missing_text(output_text, source_text, expected_percentage):
+    assert (
+        text_extraction.calculate_percent_missing_text(output_text, source_text)
+        == expected_percentage
+    )
+
+
+@pytest.mark.parametrize(
+    ("table_as_cells", "expected_extraction"),
+    [
+        pytest.param(
+            [
+                {"x": 0, "y": 0, "w": 1, "h": 1, "content": "Month A."},
+                {"x": 0, "y": 1, "w": 1, "h": 1, "content": "22"},
+            ],
+            [
+                {"row_index": 0, "col_index": 0, "content": "Month A."},
+                {"row_index": 1, "col_index": 0, "content": "22"},
+            ],
+            id="Simple table, 1 head cell, 1 body cell, no spans",
+        ),
+        pytest.param(
+            [
+                {"x": 0, "y": 0, "w": 1, "h": 1, "content": "Month A."},
+                {"x": 1, "y": 0, "w": 1, "h": 1, "content": "Month B."},
+                {"x": 2, "y": 0, "w": 1, "h": 1, "content": "Month C."},
+                {"x": 0, "y": 1, "w": 1, "h": 1, "content": "11"},
+                {"x": 1, "y": 1, "w": 1, "h": 1, "content": "12"},
+                {"x": 2, "y": 1, "w": 1, "h": 1, "content": "13"},
+                {"x": 0, "y": 2, "w": 1, "h": 1, "content": "21"},
+                {"x": 1, "y": 2, "w": 1, "h": 1, "content": "22"},
+                {"x": 2, "y": 2, "w": 1, "h": 1, "content": "23"},
+            ],
+            [
+                {"row_index": 0, "col_index": 0, "content": "Month A."},
+                {"row_index": 0, "col_index": 1, "content": "Month B."},
+                {"row_index": 0, "col_index": 2, "content": "Month C."},
+                {"row_index": 1, "col_index": 0, "content": "11"},
+                {"row_index": 1, "col_index": 1, "content": "12"},
+                {"row_index": 1, "col_index": 2, "content": "13"},
+                {"row_index": 2, "col_index": 0, "content": "21"},
+                {"row_index": 2, "col_index": 1, "content": "22"},
+                {"row_index": 2, "col_index": 2, "content": "23"},
+            ],
+            id="Simple table, 3 head cell, 5 body cell, no spans",
+        ),
+        # +----------+---------------------+----------+
+        # |          |       h1col23       |  h1col4  |
+        # | h12col1  |----------+----------+----------|
+        # |          |  h2col2  |       h2col34       |
+        # |----------|----------+----------+----------+
+        # |  r3col1  |  r3col2  |                     |
+        # |----------+----------|      r34col34       |
+        # |       r4col12       |                     |
+        # +----------+----------+----------+----------+
+        pytest.param(
+            [
+                {
+                    "y": 0,
+                    "x": 0,
+                    "w": 2,
+                    "h": 1,
+                    "content": "h12col1",
+                },
+                {
+                    "y": 0,
+                    "x": 1,
+                    "w": 1,
+                    "h": 2,
+                    "content": "h1col23",
+                },
+                {
+                    "y": 0,
+                    "x": 3,
+                    "w": 1,
+                    "h": 1,
+                    "content": "h1col4",
+                },
+                {
+                    "y": 1,
+                    "x": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "h2col2",
+                },
+                {
+                    "y": 1,
+                    "x": 2,
+                    "w": 1,
+                    "h": 2,
+                    "content": "h2col34",
+                },
+                {
+                    "y": 2,
+                    "x": 0,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r3col1",
+                },
+                {
+                    "y": 2,
+                    "x": 1,
+                    "w": 1,
+                    "h": 1,
+                    "content": "r3col2",
+                },
+                {
+                    "y": 2,
+                    "x": 2,
+                    "w": 2,
+                    "h": 2,
+                    "content": "r34col34",
+                },
+                {
+                    "y": 3,
+                    "x": 0,
+                    "w": 1,
+                    "h": 2,
+                    "content": "r4col12",
+                },
+            ],
+            [
+                {
+                    "row_index": 0,
+                    "col_index": 0,
+                    "content": "h12col1",
+                },
+                {
+                    "row_index": 0,
+                    "col_index": 1,
+                    "content": "h1col23",
+                },
+                {
+                    "row_index": 0,
+                    "col_index": 3,
+                    "content": "h1col4",
+                },
+                {
+                    "row_index": 1,
+                    "col_index": 1,
+                    "content": "h2col2",
+                },
+                {
+                    "row_index": 1,
+                    "col_index": 2,
+                    "content": "h2col34",
+                },
+                {
+                    "row_index": 2,
+                    "col_index": 0,
+                    "content": "r3col1",
+                },
+                {
+                    "row_index": 2,
+                    "col_index": 1,
+                    "content": "r3col2",
+                },
+                {
+                    "row_index": 2,
+                    "col_index": 2,
+                    "content": "r34col34",
+                },
+                {
+                    "row_index": 3,
+                    "col_index": 0,
+                    "content": "r4col12",
+                },
+            ],
+            id="various spans, with 2 row header",
+        ),
+    ],
+)
+def test_cells_table_extraction_from_prediction(table_as_cells, expected_extraction):
+    example_element = {
+        "type": "Table",
+        "metadata": {"table_as_cells": table_as_cells},
+    }
+    assert extract_cells_from_table_as_cells(example_element) == expected_extraction
+
+
+@pytest.mark.parametrize(
+    ("text_as_html", "expected_extraction"),
+    [
+        pytest.param(
+            """
+<table>
+    <thead>
+        <tr>
+            <th>Month A.</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>22</td>
+        </tr>
+    </tbody>
+</table>"
+            """,
+            [
+                {"row_index": 0, "col_index": 0, "content": "Month A."},
+                {"row_index": 1, "col_index": 0, "content": "22"},
+            ],
+            id="Simple table, 1 head cell, 1 body cell, no spans",
+        ),
+        pytest.param(
+            """
+<table>
+    <thead>
+        <tr>
+            <th>Month A.</th>
+            <th>Month B.</th>
+            <th>Month C.</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>11</td>
+            <td>12</td>
+            <td>13</td>
+        </tr>
+        <tr>
+            <td>21</td>
+            <td>22</td>
+            <td>23</td>
+        </tr>
+    </tbody>
+</table>"
+""",
+            [
+                {"row_index": 0, "col_index": 0, "content": "Month A."},
+                {"row_index": 0, "col_index": 1, "content": "Month B."},
+                {"row_index": 0, "col_index": 2, "content": "Month C."},
+                {"row_index": 1, "col_index": 0, "content": "11"},
+                {"row_index": 1, "col_index": 1, "content": "12"},
+                {"row_index": 1, "col_index": 2, "content": "13"},
+                {"row_index": 2, "col_index": 0, "content": "21"},
+                {"row_index": 2, "col_index": 1, "content": "22"},
+                {"row_index": 2, "col_index": 2, "content": "23"},
+            ],
+            id="Simple table, 3 head cell, 5 body cell, no spans",
+        ),
+        # +----------+---------------------+----------+
+        # |          |       h1col23       |  h1col4  |
+        # | h12col1  |----------+----------+----------|
+        # |          |  h2col2  |       h2col34       |
+        # |----------|----------+----------+----------+
+        # |  r3col1  |  r3col2  |                     |
+        # |----------+----------|      r34col34       |
+        # |       r4col12       |                     |
+        # +----------+----------+----------+----------+
+        pytest.param(
+            """
+<table>
+    <thead>
+        <tr>
+            <th rowspan="2">h12col1</th>
+            <th colspan="2">h1col23</th>
+            <th>h1col4</th>
+        </tr>
+        <tr>
+            <th>h2col2</th>
+            <th colspan="2">h2col34</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>r3col1</td>
+            <td>r3col2</td>
+            <td colspan="2" rowspan="2">r34col34</td>
+        </tr>
+        <tr>
+            <td colspan="2">r4col12</td>
+        </tr>
+    </tbody>
+</table>
+""",
+            [
+                {
+                    "row_index": 0,
+                    "col_index": 0,
+                    "content": "h12col1",
+                },
+                {
+                    "row_index": 0,
+                    "col_index": 1,
+                    "content": "h1col23",
+                },
+                {
+                    "row_index": 0,
+                    "col_index": 3,
+                    "content": "h1col4",
+                },
+                {
+                    "row_index": 1,
+                    "col_index": 1,
+                    "content": "h2col2",
+                },
+                {
+                    "row_index": 1,
+                    "col_index": 2,
+                    "content": "h2col34",
+                },
+                {
+                    "row_index": 2,
+                    "col_index": 0,
+                    "content": "r3col1",
+                },
+                {
+                    "row_index": 2,
+                    "col_index": 1,
+                    "content": "r3col2",
+                },
+                {
+                    "row_index": 2,
+                    "col_index": 2,
+                    "content": "r34col34",
+                },
+                {
+                    "row_index": 3,
+                    "col_index": 0,
+                    "content": "r4col12",
+                },
+            ],
+            id="various spans, with 2 row header",
+        ),
+    ],
+)
+def test_html_table_extraction_from_prediction(text_as_html, expected_extraction):
+    example_element = {
+        "type": "Table",
+        "metadata": {
+            "text_as_html": text_as_html,
+        },
+    }
+    assert extract_cells_from_text_as_html(example_element) == expected_extraction
+
+
+def test_cells_extraction_from_prediction_when_missing_prediction():
+    example_element = {"type": "Table", "metadata": {"text_as_html": "", "table_as_cells": []}}
+    assert extract_cells_from_text_as_html(example_element) is None
+    assert extract_cells_from_table_as_cells(example_element) is None
+
+
+def _trim_html(html: str) -> str:
+    html_lines = [line.strip() for line in html.split("\n") if line]
+    return "".join(html_lines)
+
+
+@pytest.mark.parametrize(
+    "html_to_test",
+    [
+        """
+<table>
+    <thead>
+        <tr>
+            <th>Month A.</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>22</td>
+        </tr>
+    </tbody>
+</table>
+""",
+        """
+<table>
+    <thead>
+        <tr>
+            <th>Month A.</th>
+            <th>Month B.</th>
+            <th>Month C.</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>11</td>
+            <td>12</td>
+            <td>13</td>
+        </tr>
+        <tr>
+            <td>21</td>
+            <td>22</td>
+            <td>23</td>
+        </tr>
+    </tbody>
+</table>
+""",
+        """
+<table>
+    <thead>
+        <tr>
+            <th rowspan="2">h12col1</th>
+            <th colspan="2">h1col23</th>
+            <th>h1col4</th>
+        </tr>
+        <tr>
+            <th>h2col2</th>
+            <th colspan="2">h2col34</th>
+        </tr>
+    </thead>
+    <tbody>
+        <tr>
+            <td>r3col1</td>
+            <td>r3col2</td>
+            <td colspan="2" rowspan="2">r34col34</td>
+        </tr>
+        <tr>
+            <td colspan="2">r4col12</td>
+        </tr>
+    </tbody>
+</table>
+""",
+    ],
+)
+def test_deckerd_html_converter(html_to_test):
+    deckerd_table = html_table_to_deckerd(html_to_test)
+    html_table = deckerd_table_to_html(deckerd_table)
+    assert _trim_html(html_to_test) == html_table

test_unstructured/metrics/test_utils.py

@@ -0,0 +1,35 @@
+import pytest
+
+from unstructured.metrics.utils import (
+    _mean,
+    _pstdev,
+    _stdev,
+    _uniquity_file,
+)
+
+
+@pytest.mark.parametrize(
+    ("numbers", "expected_mean", "expected_stdev", "expected_pstdev"),
+    [
+        ([2, 5, 6, 7], 5, 2.16, 1.871),
+        ([1, 100], 50.5, 70.004, 49.5),
+        ([1], 1, None, None),
+        ([], None, None, None),
+    ],
+)
+def test_stats(numbers, expected_mean, expected_stdev, expected_pstdev):
+    mean = _mean(numbers)
+    stdev = _stdev(numbers)
+    pstdev = _pstdev(numbers)
+    assert mean == expected_mean
+    assert stdev == expected_stdev
+    assert pstdev == expected_pstdev
+
+
+@pytest.mark.parametrize(
+    ("filenames"),
+    [("filename.ext", "filename (1).ext", "randomfile.ext", "filename.txt", "filename (5).txt")],
+)
+def test_uniquity_file(filenames):
+    final_filename = _uniquity_file(filenames, "filename.ext")
+    assert final_filename == "filename (2).ext"

test_unstructured/nlp/mock_nlp.py

@@ -0,0 +1,21 @@
+from typing import List, Tuple
+
+
+def mock_sent_tokenize(text: str) -> List[str]:
+    sentences = text.split(".")
+    return sentences[:-1] if text.endswith(".") else sentences
+
+
+def mock_word_tokenize(text: str) -> List[str]:
+    return text.split(" ")
+
+
+def mock_pos_tag(text: str) -> List[Tuple[str, str]]:
+    tokens = mock_word_tokenize(text)
+    pos_tags: List[Tuple[str, str]] = []
+    for token in tokens:
+        if token.lower() == "ask":
+            pos_tags.append((token, "VB"))
+        else:
+            pos_tags.append((token, ""))
+    return pos_tags

test_unstructured/nlp/test_partition.py

@@ -0,0 +1 @@
+# flake8: noqa

test_unstructured/nlp/test_tokenize.py

@@ -0,0 +1,66 @@
+from unstructured.nlp import tokenize
+
+
+def test_pos_tag():
+    parts_of_speech = tokenize.pos_tag("ITEM 2A. PROPERTIES")
+    tags = dict(parts_of_speech)
+    assert "ITEM" in tags
+    assert "PROPERTIES" in tags
+    assert all(isinstance(t, tuple) and len(t) == 2 for t in parts_of_speech)
+
+
+def test_word_tokenize_caches():
+    tokenize.word_tokenize.cache_clear()
+    assert tokenize.word_tokenize.cache_info().currsize == 0
+    tokenize.word_tokenize("Greetings! I am from outer space.")
+    assert tokenize.word_tokenize.cache_info().currsize == 1
+    tokenize.word_tokenize("Greetings! I am from outer space.")
+    assert tokenize.word_tokenize.cache_info().hits == 1
+
+
+def test_sent_tokenize_caches():
+    tokenize._tokenize_for_cache.cache_clear()
+    assert tokenize._tokenize_for_cache.cache_info().currsize == 0
+    tokenize._tokenize_for_cache("Greetings! I am from outer space.")
+    assert tokenize._tokenize_for_cache.cache_info().currsize == 1
+    tokenize._tokenize_for_cache("Greetings! I am from outer space.")
+    assert tokenize._tokenize_for_cache.cache_info().hits == 1
+
+
+def test_pos_tag_caches():
+    tokenize.pos_tag.cache_clear()
+    assert tokenize.pos_tag.cache_info().currsize == 0
+    tokenize.pos_tag("Greetings! I am from outer space.")
+    assert tokenize.pos_tag.cache_info().currsize == 1
+    tokenize.pos_tag("Greetings! I am from outer space.")
+    assert tokenize.pos_tag.cache_info().hits == 1
+
+
+def test_tokenizers_functions_run():
+    sentence = "I am a big brown bear. What are you?"
+    tokenize.sent_tokenize(sentence)
+    tokenize.word_tokenize(sentence)
+    tokenize.pos_tag(sentence)
+
+
+def test_process_truncates_text_exceeding_spacy_max_length(caplog):
+    # Build text well above spaCy's default 1,000,000-char limit, like the prod trace.
+    nlp = tokenize._get_nlp()
+    long_text = "This is a sentence. " * ((nlp.max_length // 20) + 10_000)
+    assert len(long_text) > nlp.max_length
+
+    with caplog.at_level("WARNING", logger=tokenize.logger.name):
+        # Must not raise spacy ValueError E088.
+        sents = tokenize.sent_tokenize(long_text)
+
+    assert len(sents) > 0
+    assert any("exceeds spaCy max_length" in rec.message for rec in caplog.records)
+
+
+def test_process_does_not_truncate_text_within_limit():
+    nlp = tokenize._get_nlp()
+    text = "Greetings! I am from outer space."
+    assert len(text) <= nlp.max_length
+    doc = tokenize._process(text)
+    # When no truncation occurs the full text round-trips through spaCy.
+    assert doc.text == text

test_unstructured/partition/common/test_common.py

@@ -0,0 +1,468 @@
+import pathlib
+from multiprocessing import Pool
+
+import numpy as np
+import pytest
+from PIL import Image
+from unstructured_inference.constants import IsExtracted
+from unstructured_inference.inference import layout
+from unstructured_inference.inference.elements import TextRegion
+from unstructured_inference.inference.layoutelement import LayoutElement
+
+from test_unstructured.unit_utils import example_doc_path
+from unstructured.documents.coordinates import PixelSpace
+from unstructured.documents.elements import (
+    TYPE_TO_TEXT_ELEMENT_MAP,
+    CheckBox,
+    CoordinatesMetadata,
+    ElementType,
+    FigureCaption,
+    Header,
+    ListItem,
+    NarrativeText,
+    Text,
+    Title,
+)
+from unstructured.documents.elements import (
+    Image as ImageElement,
+)
+from unstructured.partition.common import common
+
+
+class MockPageLayout(layout.PageLayout):
+    def __init__(self, number: int, image: Image.Image):
+        self.number = number
+        self.image = image
+
+    @property
+    def elements(self):
+        return [
+            LayoutElement(
+                type="Headline",
+                text="Charlie Brown and the Great Pumpkin",
+                bbox=None,
+            ),
+            LayoutElement(
+                type="Subheadline",
+                text="The Beginning",
+                bbox=None,
+            ),
+            LayoutElement(
+                type="Text",
+                text="This time Charlie Brown had it really tricky...",
+                bbox=None,
+            ),
+            LayoutElement(
+                type="Title",
+                text="Another book title in the same page",
+                bbox=None,
+            ),
+        ]
+
+
+class MockDocumentLayout(layout.DocumentLayout):
+    @property
+    def pages(self):
+        return [
+            MockPageLayout(number=1, image=Image.new("1", (1, 1))),
+        ]
+
+
+def test_normalize_layout_element_dict():
+    layout_element = {
+        "type": "Title",
+        "coordinates": [[1, 2], [3, 4], [5, 6], [7, 8]],
+        "coordinate_system": None,
+        "text": "Some lovely text",
+    }
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert element == Title(
+        text="Some lovely text",
+        coordinates=[[1, 2], [3, 4], [5, 6], [7, 8]],
+        coordinate_system=coordinate_system,
+    )
+
+
+def test_normalize_layout_element_dict_caption():
+    layout_element = {
+        "type": "Figure",
+        "coordinates": ((1, 2), (3, 4), (5, 6), (7, 8)),
+        "text": "Some lovely text",
+    }
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert element == ImageElement(
+        text="Some lovely text",
+        coordinates=((1, 2), (3, 4), (5, 6), (7, 8)),
+        coordinate_system=coordinate_system,
+    )
+
+
+@pytest.mark.parametrize(
+    ("element_type", "expected_type", "expected_depth"),
+    [
+        ("Title", Title, None),
+        ("Headline", Title, 1),
+        ("Subheadline", Title, 2),
+        ("Header", Header, None),
+    ],
+)
+def test_normalize_layout_element_headline(element_type, expected_type, expected_depth):
+    layout_element = {
+        "type": element_type,
+        "coordinates": [[1, 2], [3, 4], [5, 6], [7, 8]],
+        "text": "Some lovely text",
+    }
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(layout_element, coordinate_system=coordinate_system)
+    assert element.metadata.category_depth == expected_depth
+    assert isinstance(element, expected_type)
+
+
+def test_normalize_layout_element_dict_figure_caption():
+    layout_element = {
+        "type": "FigureCaption",
+        "coordinates": [[1, 2], [3, 4], [5, 6], [7, 8]],
+        "text": "Some lovely text",
+    }
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert element == FigureCaption(
+        text="Some lovely text",
+        coordinates=[[1, 2], [3, 4], [5, 6], [7, 8]],
+        coordinate_system=coordinate_system,
+    )
+
+
+def test_normalize_layout_element_dict_misc():
+    layout_element = {
+        "type": "Misc",
+        "coordinates": [[1, 2], [3, 4], [5, 6], [7, 8]],
+        "text": "Some lovely text",
+    }
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert element == Text(
+        text="Some lovely text",
+        coordinates=[[1, 2], [3, 4], [5, 6], [7, 8]],
+        coordinate_system=coordinate_system,
+    )
+
+
+def test_normalize_layout_element_layout_element():
+    layout_element = LayoutElement.from_coords(
+        type="Text",
+        x1=1,
+        y1=2,
+        x2=3,
+        y2=4,
+        text="Some lovely text",
+    )
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert element == NarrativeText(
+        text="Some lovely text",
+        coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+        coordinate_system=coordinate_system,
+    )
+
+
+def test_normalize_layout_element_layout_element_narrative_text():
+    layout_element = LayoutElement.from_coords(
+        type="NarrativeText",
+        x1=1,
+        y1=2,
+        x2=3,
+        y2=4,
+        text="Some lovely text",
+    )
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert element == NarrativeText(
+        text="Some lovely text",
+        coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+        coordinate_system=coordinate_system,
+    )
+
+
+@pytest.mark.parametrize(
+    ("element_type", "expected_element_class"),
+    TYPE_TO_TEXT_ELEMENT_MAP.items(),
+)
+def test_normalize_layout_element_layout_element_maps_to_appropriate_text_element(
+    element_type: str,
+    expected_element_class: type[Text],
+):
+    layout_element = LayoutElement.from_coords(
+        type=element_type,
+        x1=1,
+        y1=2,
+        x2=3,
+        y2=4,
+        text="Some lovely text",
+    )
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert element == expected_element_class(
+        text="Some lovely text",
+        coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+        coordinate_system=coordinate_system,
+    )
+
+
+@pytest.mark.parametrize(
+    ("element_type", "expected_checked"),
+    [
+        (ElementType.CHECK_BOX_UNCHECKED, False),
+        (ElementType.CHECK_BOX_CHECKED, True),
+        (ElementType.RADIO_BUTTON_UNCHECKED, False),
+        (ElementType.RADIO_BUTTON_CHECKED, True),
+        (ElementType.CHECKED, True),
+        (ElementType.UNCHECKED, False),
+    ],
+)
+def test_normalize_layout_element_checkable(element_type: str, expected_checked: bool):
+    layout_element = LayoutElement.from_coords(
+        type=element_type,
+        x1=1,
+        y1=2,
+        x2=3,
+        y2=4,
+        text="",
+    )
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert isinstance(element, CheckBox)
+    assert element == CheckBox(
+        checked=expected_checked,
+        coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+        coordinate_system=coordinate_system,
+    )
+
+
+def test_normalize_layout_element_enumerated_list():
+    layout_element = LayoutElement.from_coords(
+        type="List",
+        x1=1,
+        y1=2,
+        x2=3,
+        y2=4,
+        text="1. I'm so cool! 2. You're cool too. 3. We're all cool!",
+    )
+    coordinate_system = PixelSpace(width=10, height=20)
+    elements = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert elements == [
+        ListItem(
+            text="I'm so cool!",
+            coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+            coordinate_system=coordinate_system,
+        ),
+        ListItem(
+            text="You're cool too.",
+            coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+            coordinate_system=coordinate_system,
+        ),
+        ListItem(
+            text="We're all cool!",
+            coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+            coordinate_system=coordinate_system,
+        ),
+    ]
+
+
+def test_normalize_layout_element_bulleted_list():
+    layout_element = LayoutElement.from_coords(
+        type="List",
+        x1=1,
+        y1=2,
+        x2=3,
+        y2=4,
+        text="* I'm so cool! * You're cool too. * We're all cool!",
+    )
+    coordinate_system = PixelSpace(width=10, height=20)
+    elements = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert elements == [
+        ListItem(
+            text="I'm so cool!",
+            coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+            coordinate_system=coordinate_system,
+        ),
+        ListItem(
+            text="You're cool too.",
+            coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+            coordinate_system=coordinate_system,
+        ),
+        ListItem(
+            text="We're all cool!",
+            coordinates=((1, 2), (1, 4), (3, 4), (3, 2)),
+            coordinate_system=coordinate_system,
+        ),
+    ]
+
+
+class MockRunOutput:
+    def __init__(self, returncode, stdout, stderr):
+        self.returncode = returncode
+        self.stdout = stdout
+        self.stderr = stderr
+
+
+def test_convert_office_doc_captures_errors(monkeypatch, caplog):
+    from unstructured.partition.common.common import subprocess
+
+    def mock_run(*args, **kwargs):
+        return MockRunOutput(1, "an error occurred".encode(), "error details".encode())
+
+    monkeypatch.setattr(subprocess, "run", mock_run)
+    common.convert_office_doc("no-real.docx", "fake-directory", target_format="docx")
+    assert "soffice failed to convert to format docx with code 1" in caplog.text
+
+
+def test_convert_office_docs_avoids_concurrent_call_to_soffice():
+    paths_to_save = [pathlib.Path(path) for path in ("/tmp/proc1", "/tmp/proc2", "/tmp/proc3")]
+    for path in paths_to_save:
+        path.mkdir(exist_ok=True)
+        (path / "simple.docx").unlink(missing_ok=True)
+    file_to_convert = example_doc_path("simple.doc")
+
+    with Pool(3) as pool:
+        pool.starmap(common.convert_office_doc, [(file_to_convert, path) for path in paths_to_save])
+
+    assert np.sum([(path / "simple.docx").is_file() for path in paths_to_save]) == 3
+
+
+def test_convert_office_docs_respects_wait_timeout():
+    paths_to_save = [
+        pathlib.Path(path) for path in ("/tmp/wait/proc1", "/tmp/wait/proc2", "/tmp/wait/proc3")
+    ]
+    for path in paths_to_save:
+        path.mkdir(parents=True, exist_ok=True)
+        (path / "simple.docx").unlink(missing_ok=True)
+    file_to_convert = example_doc_path("simple.doc")
+
+    with Pool(3) as pool:
+        pool.starmap(
+            common.convert_office_doc,
+            # set timeout to wait for soffice to be available to 0 so only one process can convert
+            # the doc file on the first try; then the catch all
+            [(file_to_convert, path, "docx", None, 0) for path in paths_to_save],
+        )
+
+    # because this test file is very small we could have occasions where two files are converted
+    # when one of the processes spawned just a little
+    assert np.sum([(path / "simple.docx").is_file() for path in paths_to_save]) < 3
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("<table><tbody><tr><td>👨\\U+1F3FB🔧</td></tr></tbody></table>", True),
+        ("<table><tbody><tr><td>Hello!</td></tr></tbody></table>", False),
+    ],
+)
+def test_contains_emoji(text, expected):
+    assert common.contains_emoji(text) is expected
+
+
+def test_get_page_image_metadata_and_coordinate_system():
+    doc = MockDocumentLayout()
+    metadata = common.get_page_image_metadata(doc.pages[0])
+    assert isinstance(metadata, dict)
+
+
+def test_ocr_data_to_elements():
+    filename = example_doc_path("img/layout-parser-paper-fast.jpg")
+    text_regions = [
+        TextRegion.from_coords(
+            163.0,
+            115.0,
+            452.0,
+            129.0,
+            text="LayoutParser: A Unified Toolkit for Deep",
+        ),
+        TextRegion.from_coords(
+            156.0,
+            132.0,
+            457.0,
+            147.0,
+            text="Learning Based Document Image Analysis",
+        ),
+    ]
+    ocr_data = [
+        LayoutElement(
+            bbox=r.bbox,
+            text=r.text,
+            source=r.source,
+            type=ElementType.UNCATEGORIZED_TEXT,
+        )
+        for r in text_regions
+    ]
+    image = Image.open(filename)
+
+    elements = common.ocr_data_to_elements(
+        ocr_data=ocr_data,
+        image_size=image.size,
+    )
+
+    assert len(ocr_data) == len(elements)
+    assert {el.category for el in elements} == {ElementType.UNCATEGORIZED_TEXT}
+
+    # check coordinates metadata
+    image_width, image_height = image.size
+    coordinate_system = PixelSpace(width=image_width, height=image_height)
+    for el, layout_el in zip(elements, ocr_data):
+        assert el.metadata.coordinates == CoordinatesMetadata(
+            points=layout_el.bbox.coordinates,
+            system=coordinate_system,
+        )
+
+
+def test_normalize_layout_element_layout_element_text_source_metadata():
+    layout_element = LayoutElement.from_coords(
+        type="NarrativeText",
+        x1=1,
+        y1=2,
+        x2=3,
+        y2=4,
+        text="Some lovely text",
+        is_extracted=IsExtracted.TRUE,
+    )
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert hasattr(element, "metadata")
+    assert hasattr(element.metadata, "is_extracted")
+    assert element.metadata.is_extracted == "true"

test_unstructured/partition/common/test_lang.py

@@ -0,0 +1,321 @@
+# pyright: reportPrivateUsage=false
+
+"""Unit-test suite for the `unstructured.partition.lang` module."""
+
+from __future__ import annotations
+
+import os
+import pathlib
+
+import pytest
+
+from test_unstructured.unit_utils import LogCaptureFixture
+from unstructured.documents.elements import (
+    NarrativeText,
+    PageBreak,
+)
+from unstructured.partition.common.lang import (
+    _clean_ocr_languages_arg,
+    _convert_language_code_to_pytesseract_lang_code,
+    apply_lang_metadata,
+    check_language_args,
+    detect_languages,
+    prepare_languages_for_tesseract,
+    tesseract_to_paddle_language,
+)
+
+DIRECTORY = pathlib.Path(__file__).parent.resolve()
+EXAMPLE_DOCS_DIRECTORY = os.path.join(DIRECTORY, "..", "..", "example-docs")
+
+
+def test_prepare_languages_for_tesseract_with_one_language():
+    languages = ["en"]
+    assert prepare_languages_for_tesseract(languages) == "eng"
+
+
+def test_prepare_languages_for_tesseract_with_duplicated_languages():
+    languages = ["en", "eng"]
+    assert prepare_languages_for_tesseract(languages) == "eng"
+
+
+def test_prepare_languages_for_tesseract_special_case():
+    languages = ["osd"]
+    assert prepare_languages_for_tesseract(languages) == "osd"
+
+    languages = ["equ"]
+    assert prepare_languages_for_tesseract(languages) == "equ"
+
+
+def test_prepare_languages_for_tesseract_removes_empty_inputs():
+    languages = ["kbd", "es"]
+    assert prepare_languages_for_tesseract(languages) == "spa+spa_old"
+
+
+def test_prepare_languages_for_tesseract_includes_variants():
+    languages = ["chi"]
+    assert prepare_languages_for_tesseract(languages) == "chi_sim+chi_sim_vert+chi_tra+chi_tra_vert"
+
+
+def test_prepare_languages_for_tesseract_with_multiple_languages():
+    languages = ["ja", "afr", "en", "equ"]
+    assert prepare_languages_for_tesseract(languages) == "jpn+jpn_vert+afr+eng+equ"
+
+
+def test_prepare_languages_for_tesseract_warns_nonstandard_language(caplog: LogCaptureFixture):
+    languages = ["zzz", "chi"]
+    assert prepare_languages_for_tesseract(languages) == "chi_sim+chi_sim_vert+chi_tra+chi_tra_vert"
+    assert "not a valid standard language code" in caplog.text
+
+
+def test_prepare_languages_for_tesseract_warns_non_tesseract_language(caplog: LogCaptureFixture):
+    languages = ["kbd", "eng"]
+    assert prepare_languages_for_tesseract(languages) == "eng"
+    assert "not a language supported by Tesseract" in caplog.text
+
+
+def test_prepare_languages_for_tesseract_None_languages():
+    with pytest.raises(ValueError, match="`languages` can not be `None`"):
+        languages = None
+        prepare_languages_for_tesseract(languages)
+
+
+def test_prepare_languages_for_tesseract_no_valid_languages(caplog: LogCaptureFixture):
+    languages = [""]
+    assert prepare_languages_for_tesseract(languages) == "eng"
+    assert "Failed to find any valid standard language code from languages" in caplog.text
+
+
+@pytest.mark.parametrize(
+    ("tesseract_lang", "expected_lang"),
+    [
+        ("eng", "en"),
+        ("chi_sim", "ch"),
+        ("chi_tra", "chinese_cht"),
+        ("deu", "german"),
+        ("jpn", "japan"),
+        ("kor", "korean"),
+    ],
+)
+def test_tesseract_to_paddle_language_valid_codes(tesseract_lang: str, expected_lang: str):
+    assert expected_lang == tesseract_to_paddle_language(tesseract_lang)
+
+
+def test_tesseract_to_paddle_language_invalid_codes(caplog: LogCaptureFixture):
+    tesseract_lang = "unsupported_lang"
+    assert tesseract_to_paddle_language(tesseract_lang) == "en"
+    assert "unsupported_lang is not a language code supported by PaddleOCR," in caplog.text
+
+
+@pytest.mark.parametrize(
+    ("tesseract_lang", "expected_lang"),
+    [
+        ("ENG", "en"),
+        ("Fra", "fr"),
+        ("DEU", "german"),
+    ],
+)
+def test_tesseract_to_paddle_language_case_sensitivity(tesseract_lang: str, expected_lang: str):
+    assert expected_lang == tesseract_to_paddle_language(tesseract_lang)
+
+
+def test_detect_languages_english_auto():
+    text = "This is a short sentence."
+    assert detect_languages(text) == ["eng"]
+
+
+def test_detect_languages_english_provided():
+    text = "This is another short sentence."
+    languages = ["en"]
+    assert detect_languages(text, languages) == ["eng"]
+
+
+def test_detect_languages_korean_auto():
+    text = "안녕하세요"
+    assert detect_languages(text) == ["kor"]
+
+
+def test_detect_languages_gets_multiple_languages():
+    text = "My lubimy mleko i chleb."
+    assert detect_languages(text) == ["ces", "pol", "slk"]
+
+
+def test_detect_languages_warns_for_auto_and_other_input(caplog: LogCaptureFixture):
+    text = "This is another short sentence."
+    languages = ["en", "auto", "rus"]
+    assert detect_languages(text, languages) == ["eng"]
+    assert "rest of the inputted languages will be ignored" in caplog.text
+
+
+def test_detect_languages_raises_TypeError_for_invalid_languages():
+    with pytest.raises(TypeError):
+        text = "This is a short sentence."
+        detect_languages(text, languages="eng") == ["eng"]  # type: ignore
+
+
+def test_apply_lang_metadata_has_no_warning_for_PageBreak(caplog: LogCaptureFixture):
+    elements = [NarrativeText("Sample text."), PageBreak("")]
+    elements = list(
+        apply_lang_metadata(
+            elements=elements,
+            languages=["auto"],
+            detect_language_per_element=True,
+        ),
+    )
+    assert "No features in text." not in [rec.message for rec in caplog.records]
+
+
+@pytest.mark.parametrize(
+    ("lang_in", "expected_lang"),
+    [
+        ("en", "eng"),
+        ("fr", "fra"),
+    ],
+)
+def test_convert_language_code_to_pytesseract_lang_code(lang_in: str, expected_lang: str):
+    assert expected_lang == _convert_language_code_to_pytesseract_lang_code(lang_in)
+
+
+@pytest.mark.parametrize(
+    ("input_ocr_langs", "expected"),
+    [
+        (["eng"], "eng"),  # list
+        ('"deu"', "deu"),  # extra quotation marks
+        ("[deu]", "deu"),  # brackets
+        ("['deu']", "deu"),  # brackets and quotation marks
+        (["[deu]"], "deu"),  # list, brackets and quotation marks
+        (['"deu"'], "deu"),  # list and quotation marks
+        ("deu+spa", "deu+spa"),  # correct input
+    ],
+)
+def test_clean_ocr_languages_arg(input_ocr_langs: str, expected: str):
+    assert _clean_ocr_languages_arg(input_ocr_langs) == expected
+
+
+def test_detect_languages_handles_spelled_out_languages():
+    languages = detect_languages(text="Sample text longer than 5 words.", languages=["Spanish"])
+    assert languages == ["spa"]
+
+
+def test_detect_languages_short_text_fallback_returns_none():
+    """Short ASCII text with language_fallback returning None leaves language unspecified."""
+    result = detect_languages(
+        text="Hi there.",
+        language_fallback=lambda t: None,
+    )
+    assert result is None
+
+
+def test_detect_languages_short_text_fallback_returns_custom():
+    """Short ASCII text triggers fallback; we assert the fallback's return is used as-is."""
+    # Any short (<5 word) ASCII text would hit the fallback; content is irrelevant.
+    result = detect_languages(
+        text="Hi there.",
+        language_fallback=lambda t: ["fra"],
+    )
+    assert result == ["fra"]
+
+
+def test_detect_languages_short_text_default_eng_without_fallback():
+    """Short ASCII text without fallback still defaults to English (backward compat)."""
+    result = detect_languages(text="Hi there.")
+    assert result == ["eng"]
+
+
+def test_apply_lang_metadata_with_language_fallback():
+    """apply_lang_metadata passes language_fallback so short text can return None."""
+    elements = [NarrativeText("Hi.")]
+    result = list(
+        apply_lang_metadata(
+            elements=elements,
+            languages=["auto"],
+            language_fallback=lambda t: None,
+        )
+    )
+    assert len(result) == 1
+    assert result[0].metadata.languages is None
+
+
+@pytest.mark.parametrize(
+    ("languages", "ocr_languages", "expected_langs"),
+    [
+        (["spa"], "deu", ["spa"]),
+        (["spanish"], "english", ["spa"]),
+        (["spa"], "[deu]", ["spa"]),
+        (["spa"], '"deu"', ["spa"]),
+        (["spa"], ["deu"], ["spa"]),
+        (["spa"], ["[deu]"], ["spa"]),
+        (["spa+deu"], "eng+deu", ["spa", "deu"]),
+    ],
+)
+def test_check_language_args_uses_languages_when_ocr_languages_and_languages_are_both_defined(
+    languages: list[str],
+    ocr_languages: list[str] | str,
+    expected_langs: list[str],
+    caplog: LogCaptureFixture,
+):
+    returned_langs = check_language_args(
+        languages=languages,
+        ocr_languages=ocr_languages,
+    )
+    for lang in returned_langs:  # type: ignore
+        assert lang in expected_langs
+        assert "ocr_languages" in caplog.text
+
+
+@pytest.mark.parametrize(
+    ("languages", "ocr_languages", "expected_langs"),
+    [
+        # raise warning and use `ocr_languages` when `languages` is empty or None
+        ([], "deu", ["deu"]),
+        ([""], '"deu"', ["deu"]),
+        ([""], "deu", ["deu"]),
+        ([""], "[deu]", ["deu"]),
+    ],
+)
+def test_check_language_args_uses_ocr_languages_when_languages_is_empty_or_None(
+    languages: list[str],
+    ocr_languages: str,
+    expected_langs: list[str],
+    caplog: LogCaptureFixture,
+):
+    returned_langs = check_language_args(languages=languages, ocr_languages=ocr_languages)
+    for lang in returned_langs:  # type: ignore
+        assert lang in expected_langs
+        assert "ocr_languages" in caplog.text
+
+
+@pytest.mark.parametrize(
+    ("languages", "ocr_languages"),
+    [
+        ([], None),  # how check_language_args is called from auto.partition()
+        ([""], None),
+    ],
+)
+def test_check_language_args_returns_None(
+    languages: list[str],
+    ocr_languages: None,
+):
+    returned_langs = check_language_args(languages=languages, ocr_languages=ocr_languages)
+    assert returned_langs is None
+
+
+def test_check_language_args_returns_auto():
+    assert check_language_args(languages=["eng", "spa", "auto"], ocr_languages=None) == ["auto"]
+
+
+@pytest.mark.parametrize(
+    ("languages", "ocr_languages"),
+    [
+        ([], ["auto"]),
+        ([""], "eng+auto"),
+    ],
+)
+def test_check_language_args_raises_error_when_ocr_languages_contains_auto(
+    languages: list[str],
+    ocr_languages: str | list[str],
+):
+    with pytest.raises(ValueError):
+        check_language_args(
+            languages=languages,
+            ocr_languages=ocr_languages,
+        )

test_unstructured/partition/common/test_metadata.py

@@ -0,0 +1,540 @@
+"""Test-suite for `unstructured.partition.common.metadata` module."""
+
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+import copy
+import datetime as dt
+import os
+import pathlib
+from typing import Any, Callable
+
+import pytest
+
+from unstructured.documents.elements import (
+    CheckBox,
+    Element,
+    ElementMetadata,
+    FigureCaption,
+    Header,
+    ListItem,
+    NarrativeText,
+    Text,
+    Title,
+)
+from unstructured.file_utils.model import FileType
+from unstructured.partition.common.metadata import (
+    _assign_hash_ids,
+    apply_metadata,
+    get_last_modified_date,
+    set_element_hierarchy,
+)
+
+# ================================================================================================
+# LAST-MODIFIED
+# ================================================================================================
+
+
+class Describe_get_last_modified_date:
+    def it_gets_the_modified_time_of_a_file_identified_by_a_path(self, tmp_path: pathlib.Path):
+        modified_timestamp = dt.datetime(
+            year=2024, month=3, day=5, hour=17, minute=43, second=40
+        ).timestamp()
+        file_path = tmp_path / "some_file.txt"
+        file_path.write_text("abcdefg")
+        os.utime(file_path, (modified_timestamp, modified_timestamp))
+
+        last_modified_date = get_last_modified_date(str(file_path))
+
+        assert last_modified_date == "2024-03-05T17:43:40"
+
+    def but_it_returns_None_when_there_is_no_file_at_that_path(self, tmp_path: pathlib.Path):
+        file_path = tmp_path / "some_file_that_does_not_exist.txt"
+
+        last_modified_date = get_last_modified_date(str(file_path))
+
+        assert last_modified_date is None
+
+
+# ================================================================================================
+# ELEMENT HIERARCHY
+# ================================================================================================
+
+
+class Describe_set_element_hierarchy:
+    def it_applies_default_ruleset(self):
+        elements = [
+            Title(element_id="0", text="Title0"),
+            Text(element_id="1", text="Text0"),
+            Header(element_id="2", text="Header0"),
+            Text(element_id="3", text="Text1"),
+            Title(element_id="4", text="Title1"),
+            Text(element_id="5", text="Text2"),
+        ]
+
+        result = set_element_hierarchy(elements)
+
+        assert result[0].metadata.parent_id is None
+        assert result[1].metadata.parent_id == "0"  # Text0 is under Title0
+        assert result[2].metadata.parent_id is None  # Header0 is higher than Title0
+        assert result[3].metadata.parent_id == "2"  # Text1 is under Header0
+        assert result[4].metadata.parent_id == "2"  # Title1 is under Header0
+        assert result[5].metadata.parent_id == "4"  # Text2 is under Title1, which is under Header0
+
+    def it_applies_category_depth_when_element_category_is_the_same(self):
+        elements = [
+            Title(element_id="0", text="Title0", metadata=ElementMetadata(category_depth=1)),
+            ListItem(element_id="1", text="ListItem0", metadata=ElementMetadata(category_depth=0)),
+            ListItem(element_id="2", text="ListItem1", metadata=ElementMetadata(category_depth=1)),
+            ListItem(element_id="3", text="ListItem2", metadata=ElementMetadata(category_depth=0)),
+        ]
+
+        result = set_element_hierarchy(elements)
+
+        assert result[0].metadata.parent_id is None
+        assert result[1].metadata.parent_id == "0"  # category_depth=0
+        assert result[2].metadata.parent_id == "1"  # category_depth=1, so it is under ListItem0
+        assert result[3].metadata.parent_id == "0"  # category_depth=0
+
+    def but_it_ignores_category_depth_when_elements_are_of_different_categories(self):
+        elements = [
+            Title(element_id="0", text="Title", metadata=ElementMetadata(category_depth=2)),
+            Text(element_id="1", text="Text", metadata=ElementMetadata(category_depth=0)),
+            Header(element_id="2", text="Header", metadata=ElementMetadata(category_depth=2)),
+            Text(element_id="3", text="Text", metadata=ElementMetadata(category_depth=0)),
+            ListItem(element_id="4", text="ListItem", metadata=ElementMetadata(category_depth=1)),
+            NarrativeText(element_id="5", text="", metadata=ElementMetadata(category_depth=0)),
+        ]
+
+        result = set_element_hierarchy(elements)
+
+        assert result[0].metadata.parent_id is None
+        assert result[1].metadata.parent_id == "0"  # Text is under Title despite category_depth=0
+        assert result[2].metadata.parent_id is None
+        assert result[3].metadata.parent_id == "2"  # These are under Header despite category_depth
+        assert result[4].metadata.parent_id == "2"
+        assert result[5].metadata.parent_id == "2"
+
+    def it_skips_elements_with_pre_existing_parent_id(self):
+        elements = [
+            Title(element_id="0", text="Title", metadata=ElementMetadata(parent_id="10")),
+            Title(element_id="1", text="Title"),
+            Text(element_id="2", text="Text"),
+        ]
+
+        result = set_element_hierarchy(elements)
+
+        # Parent ID should not change and element is skipped in figuring out other elements' parents
+        assert result[0].metadata.parent_id == "10"
+        assert result[1].metadata.parent_id is None
+        assert result[2].metadata.parent_id == "1"
+
+    def it_sets_parent_id_for_each_element_in_elements(self):
+        elements_to_set = [
+            Title(text="Title"),  # 0
+            NarrativeText(text="NarrativeText"),  # 1
+            FigureCaption(text="FigureCaption"),  # 2
+            ListItem(text="ListItem"),  # 3
+            ListItem(text="ListItem", metadata=ElementMetadata(category_depth=1)),  # 4
+            ListItem(text="ListItem", metadata=ElementMetadata(category_depth=1)),  # 5
+            ListItem(text="ListItem"),  # 6
+            CheckBox(element_id="some-id-1", checked=True),  # 7
+            Title(text="Title 2"),  # 8
+            ListItem(text="ListItem"),  # 9
+            ListItem(text="ListItem"),  # 10
+            Text(text="Text"),  # 11
+        ]
+        elements = set_element_hierarchy(elements_to_set)
+
+        assert elements[1].metadata.parent_id == elements[0].id, (
+            "NarrativeText should be child of Title"
+        )
+        assert elements[2].metadata.parent_id == elements[0].id, (
+            "FigureCaption should be child of Title"
+        )
+        assert elements[3].metadata.parent_id == elements[0].id, "ListItem should be child of Title"
+        assert elements[4].metadata.parent_id == elements[3].id, "ListItem should be child of Title"
+        assert elements[5].metadata.parent_id == elements[3].id, "ListItem should be child of Title"
+        assert elements[6].metadata.parent_id == elements[0].id, "ListItem should be child of Title"
+        # NOTE(Hubert): moving the category field to Element, caused this to fail.
+        # Checkboxes will soon be deprecated, then we can remove the test.
+        # assert (
+        #         elements[7].metadata.parent_id is None
+        # ), "CheckBox should be None, as it's not a Text based element"
+        assert elements[8].metadata.parent_id is None, "Title 2 should be child of None"
+        assert elements[9].metadata.parent_id == elements[8].id, (
+            "ListItem should be child of Title 2"
+        )
+        assert elements[10].metadata.parent_id == elements[8].id, (
+            "ListItem should be child of Title 2"
+        )
+        assert elements[11].metadata.parent_id == elements[8].id, "Text should be child of Title 2"
+
+    def it_applies_custom_rule_set(self):
+        elements_to_set = [
+            Header(text="Header"),  # 0
+            Title(text="Title"),  # 1
+            NarrativeText(text="NarrativeText"),  # 2
+            Text(text="Text"),  # 3
+            Title(text="Title 2"),  # 4
+            FigureCaption(text="FigureCaption"),  # 5
+        ]
+
+        custom_rule_set = {
+            "Header": ["Title", "Text"],
+            "Title": ["NarrativeText", "UncategorizedText", "FigureCaption"],
+        }
+
+        elements = set_element_hierarchy(
+            elements=elements_to_set,
+            ruleset=custom_rule_set,
+        )
+
+        assert elements[1].metadata.parent_id == elements[0].id, "Title should be child of Header"
+        assert elements[2].metadata.parent_id == elements[1].id, (
+            "NarrativeText should be child of Title"
+        )
+        assert elements[3].metadata.parent_id == elements[1].id, "Text should be child of Title"
+        assert elements[4].metadata.parent_id == elements[0].id, "Title 2 should be child of Header"
+        assert elements[5].metadata.parent_id == elements[4].id, (
+            "FigureCaption should be child of Title 2"
+        )
+
+
+# ================================================================================================
+# APPLY METADATA DECORATOR
+# ================================================================================================
+
+
+class Describe_apply_metadata:
+    """Unit-test suite for `unstructured.partition.common.metadata.apply_metadata()` decorator."""
+
+    # -- unique-ify elements and metadata ---------------------------------
+
+    def it_produces_unique_elements_and_metadata_when_input_reuses_element_instances(self):
+        element = Text(text="Element", metadata=ElementMetadata(filename="foo.bar", page_number=1))
+
+        def fake_partitioner(**kwargs: Any) -> list[Element]:
+            return [element, element, element]
+
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition()
+
+        # -- all elements are unique instances --
+        assert len({id(e) for e in elements}) == len(elements)
+        # -- all metadatas are unique instances --
+        assert len({id(e.metadata) for e in elements}) == len(elements)
+
+    def and_it_produces_unique_elements_and_metadata_when_input_reuses_metadata_instances(self):
+        metadata = ElementMetadata(filename="foo.bar", page_number=1)
+
+        def fake_partitioner(**kwargs: Any) -> list[Element]:
+            return [
+                Text(text="foo", metadata=metadata),
+                Text(text="bar", metadata=metadata),
+                Text(text="baz", metadata=metadata),
+            ]
+
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition()
+
+        # -- all elements are unique instances --
+        assert len({id(e) for e in elements}) == len(elements)
+        # -- all metadatas are unique instances --
+        assert len({id(e.metadata) for e in elements}) == len(elements)
+
+    # -- unique-ids -------------------------------------------------------
+
+    def it_assigns_hash_element_ids_when_unique_ids_arg_is_not_specified(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition()
+        elements_2 = partition()
+
+        # -- SHA1 hash is 32 characters long, no hyphens --
+        assert all(len(e.id) == 32 for e in elements)
+        assert all("-" not in e.id for e in elements)
+        # -- SHA1 hashes are deterministic --
+        assert all(e.id == e2.id for e, e2 in zip(elements, elements_2))
+
+    def it_assigns_hash_element_ids_when_unique_ids_arg_is_False(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition(unique_element_ids=False)
+        elements_2 = partition(unique_element_ids=False)
+
+        # -- SHA1 hash is 32 characters long, no hyphens --
+        assert all(len(e.id) == 32 for e in elements)
+        assert all("-" not in e.id for e in elements)
+        # -- SHA1 hashes are deterministic --
+        assert all(e.id == e2.id for e, e2 in zip(elements, elements_2))
+
+    def it_leaves_UUID_element_ids_when_unique_ids_arg_is_True(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition(unique_element_ids=True)
+        elements_2 = partition(unique_element_ids=True)
+
+        # -- UUID is 36 characters long with four hyphens --
+        assert all(len(e.id) == 36 for e in elements)
+        assert all(e.id.count("-") == 4 for e in elements)
+        # -- UUIDs are non-deterministic, different every time --
+        assert all(e.id != e2.id for e, e2 in zip(elements, elements_2))
+
+    # -- parent-id --------------------------------------------------------
+
+    def it_computes_and_assigns_parent_id(self, fake_partitioner: Callable[..., list[Element]]):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition()
+
+        title = elements[0]
+        assert title.metadata.category_depth == 1
+        narr_text = elements[1]
+        assert narr_text.metadata.parent_id == title.id
+
+    # -- languages --------------------------------------------------------
+
+    def it_applies_language_metadata(self, fake_partitioner: Callable[..., list[Element]]):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition(languages=["auto"], detect_language_per_element=True)
+
+        assert all(e.metadata.languages == ["eng"] for e in elements)
+
+    # -- filetype (MIME-type) ---------------------------------------------
+
+    def it_assigns_the_value_of_a_metadata_file_type_arg_when_there_is_one(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        """A `metadata_file_type` arg overrides the file-type specified in the decorator.
+
+        This is used for example by a delegating partitioner to preserve the original file-type in
+        the metadata, like EPUB instead of the HTML that partitioner converts the .epub file to.
+        """
+        partition = apply_metadata(file_type=FileType.DOCX)(fake_partitioner)
+
+        elements = partition(metadata_file_type=FileType.ODT)
+
+        assert all(
+            e.metadata.filetype == "application/vnd.oasis.opendocument.text" for e in elements
+        )
+
+    def and_it_assigns_the_decorator_file_type_when_the_metadata_file_type_arg_is_omitted(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        """The `file_type=...` decorator arg is the "normal" way to specify the file-type.
+
+        This is used for principal (non-delegating) partitioners.
+        """
+        partition = apply_metadata(file_type=FileType.DOCX)(fake_partitioner)
+
+        elements = partition()
+
+        DOCX_MIME_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
+        assert all(e.metadata.filetype == DOCX_MIME_TYPE for e in elements)
+
+    def and_it_does_not_assign_file_type_metadata_when_both_are_omitted(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        """A partitioner can elect to assign `.metadata.filetype` for itself.
+
+        This is done in `partition_image()` for example where the same partitioner is used for
+        multiple file-types.
+        """
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition()
+
+        assert all(e.metadata.filetype == "image/jpeg" for e in elements)
+
+    # -- filename ---------------------------------------------------------
+
+    def it_uses_metadata_filename_arg_value_when_present(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        """A `metadata_filename` arg overrides all other sources."""
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition(metadata_filename="a/b/c.xyz")
+
+        assert all(e.metadata.filename == "c.xyz" for e in elements)
+        assert all(e.metadata.file_directory == "a/b" for e in elements)
+
+    def and_it_uses_filename_arg_value_when_metadata_filename_arg_not_present(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition(filename="a/b/c.xyz")
+
+        assert all(e.metadata.filename == "c.xyz" for e in elements)
+        assert all(e.metadata.file_directory == "a/b" for e in elements)
+
+    def and_it_does_not_assign_filename_metadata_when_neither_are_present(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition()
+
+        assert all(e.metadata.filename == "image.jpeg" for e in elements)
+        assert all(e.metadata.file_directory == "x/y/images" for e in elements)
+
+    # -- last_modified ----------------------------------------------------
+
+    def it_uses_metadata_last_modified_arg_value_when_present(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        """A `metadata_last_modified` arg overrides all other sources."""
+        partition = apply_metadata()(fake_partitioner)
+        metadata_last_modified = "2024-09-26T15:17:53"
+
+        elements = partition(metadata_last_modified=metadata_last_modified)
+
+        assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+    @pytest.mark.parametrize("kwargs", [{}, {"metadata_last_modified": None}])
+    def but_it_does_not_update_last_modified_when_metadata_last_modified_arg_absent_or_None(
+        self, kwargs: dict[str, Any], fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition(**kwargs)
+
+        assert all(e.metadata.last_modified == "2020-01-06T05:07:03" for e in elements)
+
+    # -- url --------------------------------------------------------------
+
+    def it_assigns_url_metadata_field_when_url_arg_is_present(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition(url="https://adobe.com/stock/54321")
+
+        assert all(e.metadata.url == "https://adobe.com/stock/54321" for e in elements)
+
+    def and_it_does_not_assign_url_metadata_when_url_arg_is_not_present(
+        self, fake_partitioner: Callable[..., list[Element]]
+    ):
+        partition = apply_metadata()(fake_partitioner)
+
+        elements = partition()
+
+        assert all(e.metadata.url == "http://images.com" for e in elements)
+
+    # -- fixtures --------------------------------------------------------------------------------
+
+    @pytest.fixture
+    def fake_partitioner(self) -> Callable[..., list[Element]]:
+        def fake_partitioner(**kwargs: Any) -> list[Element]:
+            title = Title("Introduction")
+            title.metadata.category_depth = 1
+            title.metadata.file_directory = "x/y/images"
+            title.metadata.filename = "image.jpeg"
+            title.metadata.filetype = "image/jpeg"
+            title.metadata.last_modified = "2020-01-06T05:07:03"
+            title.metadata.url = "http://images.com"
+
+            narr_text = NarrativeText("To understand bar you must first understand foo.")
+            narr_text.metadata.file_directory = "x/y/images"
+            narr_text.metadata.filename = "image.jpeg"
+            narr_text.metadata.filetype = "image/jpeg"
+            narr_text.metadata.last_modified = "2020-01-06T05:07:03"
+            narr_text.metadata.url = "http://images.com"
+
+            return [title, narr_text]
+
+        return fake_partitioner
+
+
+# ================================================================================================
+# HASH IDS
+# ================================================================================================
+
+
+def test_assign_hash_ids_produces_unique_and_deterministic_SHA1_ids_even_for_duplicate_elements():
+    elements: list[Element] = [
+        Text(text="Element", metadata=ElementMetadata(filename="foo.bar", page_number=1)),
+        Text(text="Element", metadata=ElementMetadata(filename="foo.bar", page_number=1)),
+        Text(text="Element", metadata=ElementMetadata(filename="foo.bar", page_number=1)),
+    ]
+    # -- default ids are UUIDs --
+    assert all(len(e.id) == 36 for e in elements)
+
+    elements = _assign_hash_ids(copy.deepcopy(elements))
+    elements_2 = _assign_hash_ids(copy.deepcopy(elements))
+
+    ids = [e.id for e in elements]
+    # -- ids are now SHA1 --
+    assert all(len(e.id) == 32 for e in elements)
+    # -- each id is unique --
+    assert len(ids) == len(set(ids))
+    # -- ids are deterministic, same value is computed each time --
+    assert all(e.id == e2.id for e, e2 in zip(elements, elements_2))
+
+
+def test_assign_hash_ids_remaps_parent_id_to_new_hash_id():
+    """parent_id values (originally UUIDs) are updated to the corresponding hash IDs."""
+    title = Title(text="Title", metadata=ElementMetadata(filename="foo.bar", page_number=1))
+    child = Text(
+        text="Child",
+        metadata=ElementMetadata(filename="foo.bar", page_number=1, parent_id=title.id),
+    )
+    # -- sanity-check: ids are UUIDs before hashing --
+    assert len(title.id) == 36
+    assert child.metadata.parent_id == title.id
+
+    _assign_hash_ids([title, child])
+
+    # -- ids are now SHA1 hashes --
+    assert len(title.id) == 32
+    # -- parent_id has been updated to the new hash id, not the old UUID --
+    assert child.metadata.parent_id == title.id
+
+
+def test_assign_hash_ids_leaves_unknown_parent_id_unchanged():
+    """A parent_id that has no matching element (e.g. filtered out) is left as-is, not KeyError."""
+    external_parent_id = "some-external-or-filtered-id"
+    orphan = Text(
+        text="Orphan",
+        metadata=ElementMetadata(filename="foo.bar", page_number=1, parent_id=external_parent_id),
+    )
+
+    # -- should not raise KeyError even though external_parent_id is not in id_mapping --
+    _assign_hash_ids([orphan])
+
+    # -- parent_id is left unchanged because it wasn't in the mapping --
+    assert orphan.metadata.parent_id == external_parent_id
+
+
+def test_partition_html_parent_child_relationships_preserved_with_hash_ids():
+    """Integration: partition_html with unique_element_ids=False preserves parent-child links."""
+    from unstructured.partition.html import partition_html
+
+    html = "<html><body><h1>My Title</h1><p>My paragraph</p></body></html>"
+    elements = partition_html(text=html, unique_element_ids=False)
+
+    # -- all element ids should be SHA1 hashes (32 hex chars), not UUIDs (36 chars) --
+    assert all(len(e.id) == 32 for e in elements), "Expected SHA1 hash IDs"
+
+    # -- find the title and its child paragraph --
+    title = next((e for e in elements if isinstance(e, Title)), None)
+    child = next(
+        (e for e in elements if e.metadata.parent_id is not None),
+        None,
+    )
+    assert title is not None, "Expected a Title element"
+    assert child is not None, "Expected at least one element with a parent_id"
+    # -- parent_id must point to the hashed title id, not an old UUID --
+    assert child.metadata.parent_id == title.id

test_unstructured/partition/conftest.py

@@ -0,0 +1,18 @@
+import pytest
+
+from unstructured.partition.utils.constants import OCR_AGENT_PADDLE, OCR_AGENT_TESSERACT
+
+
+@pytest.fixture
+def mock_ocr_get_instance(mocker):
+    """Fixture that mocks OCRAgent.get_instance to prevent real OCR agent instantiation."""
+
+    def mock_get_instance(ocr_agent_module, language):
+        if ocr_agent_module in (OCR_AGENT_TESSERACT, OCR_AGENT_PADDLE):
+            return mocker.MagicMock()
+        else:
+            raise ValueError(f"Unknown OCR agent: {ocr_agent_module}")
+
+    from unstructured.partition.pdf_image.ocr import OCRAgent
+
+    return mocker.patch.object(OCRAgent, "get_instance", side_effect=mock_get_instance)

test_unstructured/partition/html/test_convert.py

@@ -0,0 +1,511 @@
+# pyright: reportPrivateUsage=false, reportUnknownMemberType=false, reportOptionalMemberAccess=false
+# pyright: reportAttributeAccessIssue=false, reportUnknownLambdaType=false
+from collections import defaultdict
+from typing import Any, Optional
+
+import pytest
+from bs4 import BeautifulSoup, Tag
+from pytest_mock import MockerFixture
+
+from unstructured.documents.elements import Element, ElementMetadata, ElementType
+from unstructured.partition.html.convert import (
+    HTML_PARSER,
+    ElementHtml,
+    ImageElementHtml,
+    LinkElementHtml,
+    ListItemElementHtml,
+    TableElementHtml,
+    TextElementHtml,
+    TitleElementHtml,
+    UnorderedListElementHtml,
+    _elements_to_html_tags,
+    _elements_to_html_tags_by_page,
+    _elements_to_html_tags_by_parent,
+    _group_element_children,
+    elements_to_html,
+    group_elements_by_page,
+)
+
+
+class MockElement(Element):
+    def __init__(
+        self,
+        text: str = "",
+        metadata: Optional[ElementMetadata] = None,
+        category: str = "",
+        id: str = "",
+    ) -> None:
+        self.text = text
+        self.metadata = metadata or ElementMetadata()
+        self.category = category
+        self._element_id = id
+
+
+class MockElementMetadata(ElementMetadata):
+    def __init__(
+        self,
+        text_as_html: Optional[str] = None,
+        category_depth: Optional[int] = None,
+        image_base64: Optional[str] = None,
+        image_mime_type: Optional[str] = None,
+        url: Optional[str] = None,
+        parent_id: Optional[str] = None,
+        page_number: Optional[int] = None,
+    ) -> None:
+        self.text_as_html = text_as_html
+        self.category_depth = category_depth
+        self.image_base64 = image_base64
+        self.image_mime_type = image_mime_type
+        self.url = url
+        self.parent_id = parent_id
+        self.page_number = page_number
+
+
+@pytest.fixture
+def mock_element() -> MockElement:
+    metadata = MockElementMetadata(text_as_html="<p>Test Text</p>")
+    return MockElement(text="Test Text", metadata=metadata, category="test-category", id="test-id")
+
+
+@pytest.fixture
+def element_html(mock_element: MockElement) -> ElementHtml:
+    return ElementHtml(mock_element)
+
+
+@pytest.fixture
+def title_element_html(mock_element: MockElement) -> TitleElementHtml:
+    metadata = MockElementMetadata(text_as_html="<p>Test HTML</p>")
+    MockElement(text="Test Text", metadata=metadata, category="test-category", id="test-id")
+    return TitleElementHtml(mock_element)
+
+
+@pytest.fixture
+def image_element_html(mock_element: MockElement) -> ImageElementHtml:
+    return ImageElementHtml(mock_element)
+
+
+@pytest.fixture
+def table_element_html(mock_element: MockElement) -> TableElementHtml:
+    return TableElementHtml(mock_element)
+
+
+@pytest.fixture
+def link_element_html(mock_element: MockElement) -> LinkElementHtml:
+    return LinkElementHtml(mock_element)
+
+
+@pytest.fixture
+def unordered_list_element_html(mock_element: MockElement) -> UnorderedListElementHtml:
+    return UnorderedListElementHtml(mock_element)
+
+
+@pytest.fixture
+def elements_html() -> list[ElementHtml]:
+    return [
+        ListItemElementHtml(
+            MockElement(
+                text="Test List Item",
+                metadata=MockElementMetadata(page_number=1),
+                category=ElementType.LIST_ITEM,
+                id="test-element-1",
+            )
+        ),
+        TextElementHtml(
+            MockElement(
+                text="Test Text",
+                metadata=MockElementMetadata(page_number=None),
+                category=ElementType.TEXT,
+                id="test-element-2",
+            )
+        ),
+        TextElementHtml(
+            MockElement(
+                text="Test Text",
+                metadata=MockElementMetadata(page_number=2),
+                category=ElementType.TEXT,
+                id="test-element-3",
+            )
+        ),
+        ListItemElementHtml(
+            MockElement(
+                text="Test List Item",
+                metadata=MockElementMetadata(parent_id="test-element-3", page_number=2),
+                category=ElementType.LIST_ITEM,
+                id="test-element-4",
+            )
+        ),
+        ListItemElementHtml(
+            MockElement(
+                text="Test List Item",
+                metadata=MockElementMetadata(parent_id="test-element-3", page_number=2),
+                category=ElementType.LIST_ITEM,
+                id="test-element-5",
+            )
+        ),
+        TextElementHtml(
+            MockElement(
+                text="Test Text",
+                metadata=MockElementMetadata(page_number=3),
+                category=ElementType.TEXT,
+                id="test-element-6",
+            )
+        ),
+        ListItemElementHtml(
+            MockElement(
+                text="Test List Item Other",
+                metadata=MockElementMetadata(parent_id="test-element-6", page_number=3),
+                category=ElementType.LIST_ITEM_OTHER,
+                id="test-element-7",
+            )
+        ),
+        ListItemElementHtml(
+            MockElement(
+                text="Test List Item",
+                metadata=MockElementMetadata(parent_id="test-element-7", page_number=3),
+                category=ElementType.LIST_ITEM,
+                id="test-element-8",
+            )
+        ),
+        ListItemElementHtml(
+            MockElement(
+                text="Test List Item Other",
+                metadata=MockElementMetadata(parent_id="test-element-6", page_number=3),
+                category=ElementType.LIST_ITEM_OTHER,
+                id="test-element-9",
+            )
+        ),
+        TextElementHtml(
+            MockElement(
+                text="Test Text",
+                metadata=MockElementMetadata(parent_id="test-element-6", page_number=3),
+                category=ElementType.TEXT,
+                id="test-element-10",
+            )
+        ),
+    ]
+
+
+@pytest.fixture
+def elements(elements_html: list[ElementHtml]) -> list[Element]:
+    return [el.element for el in elements_html]
+
+
+@pytest.fixture
+def elements_small() -> list[Element]:
+    return [
+        MockElement(
+            text="Test Text 1",
+            category=ElementType.TEXT,
+            id="test-element-1",
+            metadata=MockElementMetadata(page_number=1),
+        ),
+        MockElement(
+            text="Test Text 2",
+            category=ElementType.TEXT,
+            id="test-element-2",
+            metadata=MockElementMetadata(page_number=2),
+        ),
+    ]
+
+
+def test_inject_html_element_content(element_html: ElementHtml) -> None:
+    soup = BeautifulSoup("", HTML_PARSER)
+    tag = soup.new_tag("div")
+    element_html._inject_html_element_content(tag)
+    assert tag.string == "Test Text"
+
+
+def test_get_text_as_html(element_html: ElementHtml) -> None:
+    tag = element_html.get_text_as_html()
+    assert isinstance(tag, Tag)
+    assert tag.name == "p"
+    assert tag.string == "Test Text"
+
+
+def test_get_children_html(element_html: ElementHtml) -> None:
+    soup = BeautifulSoup("", HTML_PARSER)
+    parent_tag = soup.new_tag("div")
+    child_element = MockElement(text="Child Text")
+    child_element_html = ElementHtml(child_element)
+    element_html.set_children([child_element_html])
+    result_tag = element_html._get_children_html(soup, parent_tag)
+    assert result_tag.name == "div"
+    assert len(result_tag.contents) == 2
+    assert result_tag.contents[1].string == "Child Text"
+
+
+def test_get_html_element_text_as_html(element_html: ElementHtml) -> None:
+    tag = element_html.get_html_element()
+    assert isinstance(tag, Tag)
+    assert tag.name == "p"
+    assert tag.string == "Test Text"
+    assert tag["class"] == "test-category"
+    assert tag["id"] == "test-id"
+
+
+def test_get_html_element_no_text_as_html(element_html: ElementHtml) -> None:
+    element_html.element.metadata.text_as_html = None
+    tag = element_html.get_html_element()
+    assert isinstance(tag, Tag)
+    assert tag.name == "div"
+    assert tag.string == "Test Text"
+    assert tag["class"] == "test-category"
+    assert tag["id"] == "test-id"
+
+
+def test_set_children(element_html: ElementHtml) -> None:
+    child_element = MockElement(text="Child Text")
+    child_element_html = ElementHtml(child_element)
+    element_html.set_children([child_element_html])
+    assert len(element_html.children) == 1
+    assert element_html.children[0].element.text == "Child Text"
+
+
+def test_title_element_html_tag(title_element_html: TitleElementHtml) -> None:
+    assert title_element_html.html_tag == "h1"
+
+
+def test_image_element_html_content(image_element_html: ImageElementHtml) -> None:
+    soup = BeautifulSoup("", HTML_PARSER)
+    tag = soup.new_tag("img")
+    image_element_html._inject_html_element_content(tag)
+    assert tag["alt"] == "Test Text"
+
+
+def test_image_element_html_content_with_base64(image_element_html: ImageElementHtml) -> None:
+    image_element_html.element.metadata.image_base64 = "base64data"
+    image_element_html.element.metadata.image_mime_type = "image/png"
+    soup = BeautifulSoup("", HTML_PARSER)
+    tag = soup.new_tag("img")
+    image_element_html._inject_html_element_content(tag)
+    assert tag["src"] == "data:image/png;base64,base64data"
+    assert tag["alt"] == "Test Text"
+
+
+def test_table_element_html_attrs(table_element_html: TableElementHtml) -> None:
+    soup = BeautifulSoup("", HTML_PARSER)
+    tag = soup.new_tag("table")
+    table_element_html._inject_html_element_attrs(tag)
+    assert tag["style"] == "border: 1px solid black; border-collapse: collapse;"
+
+
+def test_table_element_html_wraps_raw_text_in_a_cell(
+    table_element_html: TableElementHtml,
+) -> None:
+    # -- raw table text must sit inside a `<td>` so HTML5 parsers (e.g. the nh3
+    # -- sanitization pass) don't foster-parent it out of the empty table --
+    table_element_html.element.metadata.text_as_html = None
+    table_element_html.element.text = "cell text"
+    tag = table_element_html.get_html_element()
+    cell = tag.find("td")
+    assert cell is not None
+    assert cell.get_text() == "cell text"
+
+
+def test_table_element_html_leaves_empty_table_without_a_cell(
+    table_element_html: TableElementHtml,
+) -> None:
+    table_element_html.element.metadata.text_as_html = None
+    table_element_html.element.text = ""
+    tag = table_element_html.get_html_element()
+    assert tag.find("td") is None
+
+
+def test_link_element_html_attrs(link_element_html: LinkElementHtml) -> None:
+    link_element_html.element.metadata.url = "http://example.com"
+    soup = BeautifulSoup("", HTML_PARSER)
+    tag = soup.new_tag("a")
+    link_element_html._inject_html_element_attrs(tag)
+    assert tag["href"] == "http://example.com"
+
+
+def test_unordered_list_element_html(unordered_list_element_html: UnorderedListElementHtml) -> None:
+    soup = BeautifulSoup("", HTML_PARSER)
+    tag = soup.new_tag("ul")
+    child_element = MockElement(text="Child Text")
+    child_element_html = ListItemElementHtml(child_element)
+    unordered_list_element_html.set_children([child_element_html])
+    result_tag = unordered_list_element_html._get_children_html(soup, tag)
+    assert result_tag.name == "ul"
+    assert len(result_tag.contents) == 1
+    assert result_tag.contents[0].name == "li"
+    assert result_tag.contents[0].string == "Child Text"
+
+
+def test_group_element_children(elements_html: list[ElementHtml]) -> None:
+    grouped_children = _group_element_children(elements_html)
+    assert len(grouped_children) == 7
+    assert len(grouped_children[0].children) == 1
+    assert grouped_children[0].children[0].element.category == ElementType.LIST_ITEM
+    assert len(grouped_children[3].children) == 2
+    assert grouped_children[3].children[0].element.category == ElementType.LIST_ITEM
+    assert grouped_children[3].children[1].element.category == ElementType.LIST_ITEM
+    assert len(grouped_children[5].children) == 3
+    assert grouped_children[5].children[0].element.category == ElementType.LIST_ITEM_OTHER
+    assert grouped_children[5].children[1].element.category == ElementType.LIST_ITEM
+    assert grouped_children[5].children[2].element.category == ElementType.LIST_ITEM_OTHER
+
+
+def test_elements_to_html_tags_by_parent(
+    mocker: MockerFixture, elements_html: list[ElementHtml]
+) -> None:
+    mocker.patch(
+        "unstructured.partition.html.convert._group_element_children",
+        side_effect=lambda children: children,
+    )
+    result = _elements_to_html_tags_by_parent(elements_html)
+    assert len(result) == 4
+    assert result[0].element.id == "test-element-1"
+    assert len(result[0].children) == 0
+    assert result[1].element.id == "test-element-2"
+    assert len(result[1].children) == 0
+    assert result[2].element.id == "test-element-3"
+    assert len(result[2].children) == 2
+    assert result[2].children[0].element.id == "test-element-4"
+    assert result[2].children[1].element.id == "test-element-5"
+    assert result[3].element.id == "test-element-6"
+    assert len(result[3].children) == 3
+    assert result[3].children[0].element.id == "test-element-7"
+    assert len(result[3].children[0].children) == 1
+    assert result[3].children[0].children[0].element.id == "test-element-8"
+    assert result[3].children[1].element.id == "test-element-9"
+    assert result[3].children[2].element.id == "test-element-10"
+
+
+def test_elements_to_html_tags(mocker: MockerFixture, elements: list[Element]) -> None:
+    def _mock_get_html_element(self: ElementHtml, **kwargs: Any):
+        return BeautifulSoup(f"<div>{self.element.id}</div>", HTML_PARSER).find()
+
+    mocker.patch(
+        "unstructured.partition.html.convert._elements_to_html_tags_by_parent",
+        side_effect=lambda elements: elements,
+    )
+    mocker.patch(
+        "unstructured.partition.html.convert.ElementHtml.get_html_element",
+        side_effect=_mock_get_html_element,
+        autospec=True,
+    )
+    result = _elements_to_html_tags(elements)
+    assert len(result) == 10
+    assert all(isinstance(tag, Tag) for tag in result)
+    for i, el in enumerate(result, start=1):
+        assert el.string == f"test-element-{i}"
+
+
+def test_elements_to_html_tags_by_page(mocker: MockerFixture, elements: list[Element]) -> None:
+    def _mock_elements_to_html_tags(elements: list[Element], _: bool):
+        return [
+            BeautifulSoup(f"<div>{element.id}</div>", HTML_PARSER).find() for element in elements
+        ]
+
+    def _mock_group_elements_by_page(elements: list[Element]) -> list[list[Element]]:
+        pages_dict: defaultdict[int, list[Element]] = defaultdict(list)
+        for element in elements:
+            if element.metadata.page_number is not None:
+                pages_dict[element.metadata.page_number].append(element)
+        return list(pages_dict.values())
+
+    mocker.patch(
+        "unstructured.partition.html.convert._elements_to_html_tags",
+        side_effect=_mock_elements_to_html_tags,
+    )
+    mocker.patch(
+        "unstructured.partition.html.convert.group_elements_by_page",
+        side_effect=_mock_group_elements_by_page,
+    )
+    result = _elements_to_html_tags_by_page(elements)
+    assert len(result) == 3
+    assert all(isinstance(tag, Tag) for tag in result)
+    assert result[0].name == "div"
+    assert result[0]["data-page_number"] == 1
+    assert len(result[0].contents) == 1
+    assert result[0].contents[0].string == "test-element-1"
+    assert result[1].name == "div"
+    assert result[1]["data-page_number"] == 2
+    assert len(result[1].contents) == 3
+    for i, el in enumerate(result[1].contents, start=3):
+        assert el.string == f"test-element-{i}"
+    assert result[2]["data-page_number"] == 3
+    assert len(result[2].contents) == 5
+    for i, el in enumerate(result[2].contents, start=6):
+        assert el.string == f"test-element-{i}"
+
+
+def test_group_elements_by_page(caplog: pytest.LogCaptureFixture, elements: list[Element]) -> None:
+    result = group_elements_by_page(elements)
+    assert len(result) == 3
+    assert len(result[0]) == 1
+    assert len(result[1]) == 3
+    assert len(result[2]) == 5
+    assert result[0][0].id == "test-element-1"
+    for i, el in enumerate(result[1], start=3):
+        assert el.id == f"test-element-{i}"
+    for i, el in enumerate(result[2], start=6):
+        assert el.id == f"test-element-{i}"
+    assert "Page number is not set for an element test-element-2. Skipping." in caplog.text
+
+
+def test_elements_to_html_no_group_by_page(
+    mocker: MockerFixture, elements_small: list[Element]
+) -> None:
+    def _mock_elements_to_html_tags(elements: list[Element], _: bool):
+        return [
+            BeautifulSoup(f"<div>{element.id}</div>", HTML_PARSER).find() for element in elements
+        ]
+
+    mock_elements_to_html_tags = mocker.patch(
+        "unstructured.partition.html.convert._elements_to_html_tags",
+        side_effect=_mock_elements_to_html_tags,
+    )
+    mock_elements_to_html_tags_by_page = mocker.patch(
+        "unstructured.partition.html.convert._elements_to_html_tags_by_page",
+    )
+
+    result = elements_to_html(elements_small, exclude_binary_image_data=True, no_group_by_page=True)
+    assert "<div>\n   test-element-1\n  </div>" in result
+    assert "<div>\n   test-element-2\n  </div>" in result
+    assert "data-page_number" not in result
+    mock_elements_to_html_tags.assert_called_once_with(elements_small, True)
+    mock_elements_to_html_tags_by_page.assert_not_called()
+
+
+def test_elements_to_html_group_by_page(
+    mocker: MockerFixture, elements_small: list[Element]
+) -> None:
+    def _mock_elements_to_html_tags_by_page(elements: list[Element], _: bool):
+        return [
+            BeautifulSoup(
+                f"<div data-page_number='{element.metadata.page_number}'>{element.id}</div>",
+                HTML_PARSER,
+            ).find()
+            for element in elements
+        ]
+
+    mock_elements_to_html_tags_by_page = mocker.patch(
+        "unstructured.partition.html.convert._elements_to_html_tags_by_page",
+        side_effect=_mock_elements_to_html_tags_by_page,
+    )
+    mock_elements_to_html_tags = mocker.patch(
+        "unstructured.partition.html.convert._elements_to_html_tags"
+    )
+
+    result = elements_to_html(
+        elements_small, exclude_binary_image_data=True, no_group_by_page=False
+    )
+    soup = BeautifulSoup(result, HTML_PARSER)
+    assert soup.find("div", {"data-page_number": "1"}).string.strip() == "test-element-1"
+    assert soup.find("div", {"data-page_number": "2"}).string.strip() == "test-element-2"
+    mock_elements_to_html_tags_by_page.assert_called_once_with(elements_small, True)
+    mock_elements_to_html_tags.assert_not_called()
+
+
+def test_elements_to_html_invalid_html_template(
+    mocker: MockerFixture, elements: list[Element]
+) -> None:
+    mocker.patch(
+        "unstructured.partition.html.convert.HTML_TEMPLATE",
+        "<html><head><title>Test</title></head></html>",
+    )
+
+    with pytest.raises(ValueError, match="Body tag not found in the HTML template"):
+        elements_to_html(elements)

test_unstructured/partition/html/test_html_to_ontology_parsing.py

@@ -0,0 +1,697 @@
+from typing import Optional, Type
+
+import pytest
+from bs4 import BeautifulSoup
+
+from unstructured.documents.ontology import (
+    Checkbox,
+    Form,
+    FormFieldValue,
+    Image,
+    OntologyElement,
+    Page,
+    RadioButton,
+)
+from unstructured.partition.html.html_utils import indent_html
+from unstructured.partition.html.transformations import RECURSION_LIMIT, parse_html_to_ontology
+
+
+def _wrap_with_body(html: str) -> str:
+    return f'<body class="Document">{html}</body>'
+
+
+def remove_all_ids(html_str):
+    soup = BeautifulSoup(html_str, "html.parser")
+    for tag in soup.find_all(True):
+        if tag.has_attr("id"):
+            del tag["id"]
+    return str(soup)
+
+
+def test_parsing_header_and_footer_into_correct_ontologyelement():
+    input_html = """
+    <div class="Page">
+    <header class="Header">
+     this is a header
+    </header>
+    <footer class="Footer">
+     this is a footer
+    </footer>
+    </div>
+    """
+    page = parse_html_to_ontology(input_html)
+    assert len(page.children) == 2
+    header, footer = page.children
+    assert header.text == "this is a header"
+    assert header.html_tag_name == "header"
+    assert footer.text == "this is a footer"
+    assert footer.html_tag_name == "footer"
+
+
+def test_wrong_html_parser_causes_paragraph_to_be_nested_in_div():
+    # This test would fail if html5lib parser would be applied on the input HTML.
+    # It would result in Page: <p></p> <address></address>
+    #         instead of Page: <p><address></address></p>
+
+    # language=HTML
+    input_html = """
+    <div class="Page">
+    <p class="NarrativeText">
+     <address class="Address">
+      Mountain View, California
+     </address>
+    </p>
+    </div>
+    """
+    page = parse_html_to_ontology(input_html)
+
+    assert len(page.children) == 1
+    narrative_text = page.children[0]
+
+    assert len(narrative_text.children) == 1
+    address = narrative_text.children[0]
+
+    assert address.text == "Mountain View, California"
+
+
+def test_when_class_is_missing_it_can_be_inferred_from_type():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <aside>Some text</aside>
+    </div>
+    """)
+
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <aside class='Sidebar'><p class='Paragraph'>Some text</p></aside>
+    </div>
+    """)
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_when_class_is_wrong_tag_name_is_overwritten():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <p class='Sidebar'>Some text</p>
+    </div>
+    """)
+
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <aside class='Sidebar'><p class='Paragraph'>Some text</p></aside>
+    </div>
+    """)
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_when_tag_not_supported_by_ontology_and_wrong_then_consider_them_text():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <newtag class="wrongclass">Some text
+        </newtag>
+    </div>
+    """)
+
+    base_html = indent_html(base_html)
+
+    # TODO (Pluto): Maybe it should be considered as plain text?
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <span class="UncategorizedText">Some text
+        </span>
+    </div>
+    """)
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_div_are_ignored_when_no_attrs():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+      <div>
+           <input class="RadioButton" name="health-comparison" type="radio"/>
+      </div>
+    </div>
+    """)
+
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+       <input class="RadioButton" name="health-comparison" type="radio"/>
+    </div>
+    """)
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_ids_are_preserved():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+      <div style="background-color: lightblue" id="important_div">
+           <input class="RadioButton" name="health-comparison" type="radio"/>
+      </div>
+    </div>
+    """)
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+      <div class="UncategorizedText" style="background-color: lightblue" id="important_div">
+           <input class="RadioButton" name="health-comparison" type="radio"/>
+      </div>
+    </div>
+    """)
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+
+    page = ontology.children[0]
+    div_obj = page.children[0]
+    assert div_obj.additional_attributes["id"] == "important_div"
+
+
+def test_br_is_not_considered_uncategorized_text():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <br/>
+    </div>
+    """)
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <br/>
+    </div>
+    """)
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_text_without_tag_is_marked_as_uncategorized_text_when_there_are_other_elements():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        About the same
+        <input class="RadioButton" name="health-comparison" type="radio"/>
+        Some text
+    </div>
+    """)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <p class="Paragraph">
+            About the same
+        </p>
+        <input class="RadioButton" name="health-comparison" type="radio"/>
+        <p class="Paragraph">
+            Some text
+        </p>
+    </div>
+    """)
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_keyword_only_attributes_are_preserved_during_mapping():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <input class="FormFieldValue" type="radio" name="options" value="2" checked>
+    """)  # noqa: E501
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <input class="FormFieldValue" type="radio" name="options" value="2" checked>
+    """)  # noqa: E501
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_when_unknown_element_keyword_only_attributes_are_preserved_during_mapping():
+    # <input> can be assigned to multiple classes so it is not clear what it is
+    #  thus we assign it to UncategorizedText
+
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <form class="Form">
+            <label class="FormField" for="option1">
+                <input type="radio" name="option1" value="2" checked>
+                <span class="UncategorizedText">
+                    Option 1 (Checked)
+                </span>
+            </label>
+        </form>
+    </div>
+    """)
+    base_html = indent_html(base_html)
+
+    # TODO(Pluto): Maybe tag also should be overwritten? Or just leave it as it is?
+    #  We classify <input> as UncategorizedText but all the text is preserved
+    #  for UnstructuredElement so it make sense now as well
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <form class="Form">
+            <label class="FormField" for="option1">
+                <input class="RadioButton" type="radio" name="option1" value="2" checked />
+                <span class="UncategorizedText">
+                    Option 1 (Checked)
+                </span>
+            </label>
+        </form>
+    </div>
+    """)  # noqa: E501
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_broken_cell_is_not_raising_error():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <table class="Table">
+            <tbody class="TableBody">
+                <tr class="TableRow">
+                   <td class="TableCell&gt;11,442,231&lt;/td&gt;&lt;td class=" tablecell"="">
+                    83.64 GiB
+                   </td>
+                  <th class="TableCellHeader" rowspan="2">
+                    Fair Value
+                 </th>
+               </tr>
+           </tbody>
+        </table>
+    </div>
+    """)
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <table class="Table">
+            <tbody>
+                <tr>
+                   <td>
+                    83.64 GiB
+                   </td>
+                  <th rowspan="2">
+                    Fair Value
+                 </th>
+               </tr>
+           </tbody>
+        </table>
+    </div>
+    """)
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_table():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <table class="Table">
+            <tbody class="TableBody">
+                <tr class="TableRow">
+                  <td class="TableCell">
+                    Fair Value1
+                  </td>
+                  <th class="TableCellHeader" rowspan="2">
+                    Fair Value2
+                 </th>
+               </tr>
+           </tbody>
+        </table>
+    </div>
+    """)
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <table class="Table">
+            <tbody>
+                <tr>
+                  <td>
+                    Fair Value1
+                  </td>
+                  <th rowspan="2">
+                    Fair Value2
+                 </th>
+               </tr>
+           </tbody>
+        </table>
+    </div>
+    """)
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_table_and_time():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <table class="Table">
+            <thead class='TableHeader'>
+                <tr class="TableRow">
+                    <th class="TableCellHeader"  colspan="6">
+                        Carrying Value
+                    </th>
+                </tr>
+            </thead>
+            <tbody class='TableBody'>
+                <tr class="TableRow">
+                    <td class="TableCell" colspan="5">
+                        <time class="CalendarDate">
+                            June 30, 2023
+                        </time>
+                    </td>
+                <td class="TableCell">
+                    <span class="Currency">
+                        $—
+                    </span>
+                </td>
+                </tr>
+            </tbody>
+        </table>
+    </div>
+    """)
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <table class="Table">
+            <thead>
+                <tr>
+                    <th colspan="6">
+                        Carrying Value
+                    </th>
+                </tr>
+            </thead>
+            <tbody>
+                <tr>
+                    <td colspan="5">
+                        <time>
+                            June 30, 2023
+                        </time>
+                    </td>
+                <td>
+                    <span>
+                        $—
+                    </span>
+                </td>
+                </tr>
+            </tbody>
+        </table>
+    </div>
+    """)
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_malformed_html():
+    # language=HTML
+    input_html = """
+    <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+    <html>
+      <head>
+        <title>Super Malformed HTML</title>
+      </head>
+      <body class="Document">
+        <!-- Unclosed comment
+        <div class=>
+          <p>Paragraph with missing closing angle bracket
+        <div>
+          <span>
+            <p>Improperly nested paragraph within a span</span>
+          </p>
+        </div>
+        <script>
+          var x = "Unclosed script tag example;
+        </script>
+        <p>Paragraph with invalid characters: � � �</p>
+      </div>
+    </html>
+    """
+
+    # Such malformed HTML won't be returned by html_partitioning as it uses html5lib parser
+    # to imitate the same behaviour it will be first parsed the same way
+
+    input_html = indent_html(input_html, html_parser="html5lib")
+
+    # Ontology has 1 element and everything inside is just Text.
+    # SECURITY (GHSA-v5mq-3xhg-98m9): the collapsed markup stored as this element's
+    # text is HTML-escaped on output, so the embedded <script>/<div>/<p> survive as
+    # inert escaped text (&lt;script&gt; ...) instead of being re-materialized as live
+    # tags. Previously this test asserted a live <script> tag in the output.
+    # language=HTML
+    expected_html = """
+    <body class="Document">
+      <p class="Paragraph">
+       Unclosed comment
+        &lt;div class=&gt;
+          &lt;p&gt;Paragraph with missing closing angle bracket
+        &lt;div&gt;
+          &lt;span&gt;
+            &lt;p&gt;Improperly nested paragraph within a span&lt;/span&gt;
+          &lt;/p&gt;
+        &lt;/div&gt;
+        &lt;script&gt;
+          var x = "Unclosed script tag example;
+        &lt;/script&gt;
+        &lt;p&gt;Paragraph with invalid characters: � � �&lt;/p&gt;
+      &lt;/div&gt;
+    &lt;/html&gt;
+      </p>
+    </body>
+    """
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(input_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_text_is_wrapped_inside_layout_element():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        Text
+    </div>
+    """)
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <p class='Paragraph'>Text</p>
+    </div>
+    """)
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_text_in_form_field_value():
+    # language=HTML
+    input_html = """
+    <div class="Page">
+    <input class="FormFieldValue" value="Random Input Value"/>
+    </div>
+    """
+    page = parse_html_to_ontology(input_html)
+
+    assert len(page.children) == 1
+    form_field_value = page.children[0]
+    assert form_field_value.text == ""
+    assert form_field_value.to_text() == "Random Input Value"
+
+
+def test_text_in_form_field_value_with_null_value():
+    # language=HTML
+    input_html = """
+    <div class="Page">
+    <input class="FormFieldValue" value=""/>
+    </div>
+    """
+    page = parse_html_to_ontology(input_html)
+
+    assert len(page.children) == 1
+    form_field_value = page.children[0]
+    assert form_field_value.text == ""
+    assert form_field_value.to_text() == ""
+
+
+def test_to_text_when_form_field():
+    ontology = Page(
+        children=[
+            Form(
+                tag="input",
+                additional_attributes={"value": "Random Input Value"},
+                children=[
+                    FormFieldValue(
+                        tag="input",
+                        additional_attributes={"value": "Random Input Value"},
+                    )
+                ],
+            )
+        ]
+    )
+    assert ontology.to_text(add_children=True) == "Random Input Value"
+
+
+def test_recursion_limit_is_limiting_parsing():
+    # language=HTML
+    broken_html = "some text"
+    for i in range(100):
+        broken_html = f"<p class='Paragraph'>{broken_html}</p>"
+    broken_html = _wrap_with_body(broken_html)
+    ontology = parse_html_to_ontology(broken_html)
+
+    iterator = 1
+    last_child = ontology.children[0]
+    while last_child.children:
+        last_child = last_child.children[0]
+        iterator += 1
+    assert last_child.text.startswith('<p class="Paragraph">')
+    assert iterator == RECURSION_LIMIT
+
+
+def test_get_text_when_recursion_limit_activated():
+    broken_html = "some text"
+    for i in range(100):
+        broken_html = f"<p class='Paragraph'>{broken_html}</p>"
+    broken_html = _wrap_with_body(broken_html)
+    ontology = parse_html_to_ontology(broken_html)
+
+    last_child = ontology.children[0]
+    while last_child.children:
+        last_child = last_child.children[0]
+
+    assert last_child.to_text() == "some text"
+
+
+def test_uncategorizedtest_has_image_and_no_text():
+    # language=HTML
+    base_html = _wrap_with_body("""
+        <div class="Page">
+    <div class="UncategorizedText">
+        <img src="https://www.example.com/image.jpg"/>
+    </div>
+    </div>
+    """)
+
+    base_html = indent_html(base_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+
+    element = ontology.children[0].children[0]
+    assert type(element) is Image
+    assert element.css_class_name == "Image"
+
+
+@pytest.mark.parametrize(
+    ("input_type", "expected_class"),
+    [
+        ("checkbox", Checkbox),
+        ("radio", RadioButton),
+        ("text", FormFieldValue),  # explicit non-specialised type
+        (None, FormFieldValue),  # missing type attribute
+    ],
+)
+def test_input_tag_type_is_mapped_to_correct_ontology_class(
+    input_type: Optional[str], expected_class: Type[OntologyElement]
+) -> None:
+    """Ensure bare <input> tags are classified based on their *type* attribute."""
+
+    type_attr = f' type="{input_type}"' if input_type is not None else ""
+    html_snippet = f'<div class="Page"><input{type_attr} name="field" /></div>'
+
+    page = parse_html_to_ontology(html_snippet)
+    assert len(page.children) == 1
+    element = page.children[0]
+
+    # Validate chosen ontology class and preserved HTML semantics
+    assert isinstance(element, expected_class)
+    assert element.html_tag_name == "input"
+    assert element.css_class_name == expected_class.__name__

test_unstructured/partition/html/test_html_to_unstructured_and_back_parsing.py

@@ -0,0 +1,480 @@
+# End 2 End tests for 15 types
+
+from unstructured.documents.elements import (
+    Element,
+    ElementMetadata,
+    Header,
+    NarrativeText,
+    Table,
+    Text,
+)
+from unstructured.documents.ontology import Address, Paragraph
+from unstructured.partition.html.html_utils import indent_html
+from unstructured.partition.html.partition import partition_html
+from unstructured.partition.html.transformations import (
+    ontology_to_unstructured_elements,
+    parse_html_to_ontology,
+    unstructured_elements_to_ontology,
+)
+
+
+def _wrap_in_body_and_page(html_code):
+    return f'<body class="Document"><div class="Page" data-page-number="1">{html_code}</div></body>'
+
+
+_page_elements = [
+    Text(
+        text="",
+        metadata=ElementMetadata(text_as_html='<div class="Page" data-page-number="1" />'),
+    ),
+]
+
+
+def _assert_elements_equal(actual_elements: list[Element], expected_elements: list[Element]):
+    assert len(actual_elements) == len(expected_elements)
+    for actual, expected in zip(actual_elements, expected_elements):
+        assert actual == expected, f"Actual: {actual}, Expected: {expected}"
+        # Not all elements are considered be __eq__ Elements method
+        actual_html = indent_html(actual.metadata.text_as_html, html_parser="html.parser")
+        expected_html = indent_html(expected.metadata.text_as_html, html_parser="html.parser")
+        assert actual_html == expected_html, f"Actual: {actual_html}, Expected: {expected_html}"
+
+
+def _parse_to_unstructured_elements_and_back_to_html(html_as_str: str):
+    unstructured_elements = partition_html(
+        text=html_as_str, add_img_alt_text=False, html_parser_version="v2", unique_element_ids=True
+    )
+    parsed_ontology = unstructured_elements_to_ontology(unstructured_elements)
+    return unstructured_elements, parsed_ontology
+
+
+def test_simple_narrative_text_with_id():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+    <p class="NarrativeText">
+     DEALER ONLY
+    </p>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+
+    assert expected_html == parsed_html
+    expected_elements = _page_elements + [
+        NarrativeText(
+            text="DEALER ONLY",
+            metadata=ElementMetadata(
+                text_as_html='<p class="NarrativeText">DEALER ONLY</p>',
+            ),
+        )
+    ]
+
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_input_with_radio_button_checked():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+       <input class="RadioButton" name="health-comparison" type="radio" checked/>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+
+    assert expected_html == parsed_html
+    expected_elements = _page_elements + [
+        Text(
+            text="",
+            metadata=ElementMetadata(
+                text_as_html=(
+                    '<input class="RadioButton" name="health-comparison" type="radio" checked />'
+                ),
+            ),
+        )
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_multiple_elements():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+    <p class="Paragraph">
+        About the same
+    </p>
+    <input class="RadioButton" name="health-comparison" type="radio"/>
+    <p class="Paragraph">
+        Some text
+    </p>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+
+    assert expected_html == parsed_html
+    expected_elements = _page_elements + [
+        NarrativeText(
+            text="About the same",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph">About the same</p>',
+            ),
+        ),
+        Text(
+            text="",
+            metadata=ElementMetadata(
+                text_as_html='<input class="RadioButton" name="health-comparison" type="radio" />',
+            ),
+        ),
+        NarrativeText(
+            text="Some text",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph">Some text</p>',
+            ),
+        ),
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_multiple_pages():
+    # language=HTML
+    html_as_str = """
+    <body class="Document">
+        <div class="Page" data-page-number="1">
+            <p class="Paragraph">
+                Some text
+            </p>
+        </div>
+        <div class="Page" data-page-number="2">
+            <p class="Paragraph">
+                Another text
+            </p>
+        </div>
+    </body>
+    """
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+
+    assert expected_html == parsed_html
+
+    expected_elements = [
+        Text(
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page" data-page-number="1" />'),
+        ),
+        NarrativeText(
+            text="Some text",
+            metadata=ElementMetadata(text_as_html='<p class="Paragraph">Some text</p>'),
+        ),
+        Text(
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page" data-page-number="2" />'),
+        ),
+        NarrativeText(
+            text="Another text",
+            metadata=ElementMetadata(text_as_html='<p class="Paragraph">Another text</p>'),
+        ),
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_forms():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+        <form class="Form">
+            <label class="FormField" for="option1">
+                <input class="FormFieldValue" type="radio"
+                 name="options" value="2" checked>
+                <p class="Paragraph">
+                    Option 1 (Checked)
+                </p>
+            </label>
+        </form>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+    assert expected_html == parsed_html
+    expected_elements = _page_elements + [
+        Text(
+            text="2 Option 1 (Checked)",
+            metadata=ElementMetadata(
+                text_as_html=""
+                '<form class="Form">'
+                '<label class="FormField" '
+                'for="option1">'
+                '<input class="FormFieldValue" type="radio" '
+                'name="options" value="2" checked />'
+                '<p class="Paragraph">'
+                "Option 1 (Checked)"
+                "</p></label></form>",
+            ),
+        )
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_table():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+    <table class="Table">
+        <tbody class="TableBody">
+            <tr class="TableRow">
+              <td class="TableCell">
+                Fair Value1
+              </td>
+              <th class="TableCellHeader" rowspan="2">
+                Fair Value2
+             </th>
+           </tr>
+       </tbody>
+    </table>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+
+    expected_elements = _page_elements + [
+        Table(
+            text="Fair Value1 Fair Value2",
+            metadata=ElementMetadata(
+                text_as_html='<table class="Table">'
+                "<tbody>"
+                "<tr>"
+                "<td>"
+                "Fair Value1"
+                "</td>"
+                '<th rowspan="2">'
+                "Fair Value2"
+                "</th></tr></tbody></table>",
+            ),
+        )
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_very_nested_structure_is_preserved():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+    <section class='Section'>
+        <div class='Column'>
+            <header class='Header'>
+                <h1 class='Title'>
+                    Title
+                </h1>
+            </header>
+        </div>
+    </section>
+    <div class='Column'>
+            <header class='Header'>
+                Page 1
+            </header>
+            <blockquote class="Quote">
+                <p class="Paragraph">
+                    Clever Quote
+                </p>
+            </blockquote>
+            <div class='Footnote'>
+                <span class='UncategorizedText'>
+                    Uncategorized footnote text
+                </span>
+            </div>
+    </div>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+
+    assert expected_html == parsed_html
+    expected_elements = _page_elements + [
+        Text(
+            text="",
+            metadata=ElementMetadata(text_as_html='<section class="Section" />'),
+        ),
+        Text(
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Column" />'),
+        ),
+        Header(
+            text="Title",
+            metadata=ElementMetadata(
+                text_as_html='<header class="Header"><h1 class="Title">Title</h1></header>'
+            ),
+        ),
+        Text(
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Column" />'),
+        ),
+        Header(
+            text="Page 1",
+            metadata=ElementMetadata(text_as_html='<header class="Header">Page 1</header>'),
+        ),
+        NarrativeText(
+            text="Clever Quote",
+            metadata=ElementMetadata(
+                text_as_html='<blockquote class="Quote">'
+                '<p class="Paragraph">'
+                "Clever Quote"
+                "</p>"
+                "</blockquote>",
+            ),
+        ),
+        Text(
+            text="Uncategorized footnote text",
+            metadata=ElementMetadata(
+                text_as_html='<div class="Footnote">'
+                '<span class="UncategorizedText">'
+                "Uncategorized footnote text"
+                "</span>"
+                "</div>",
+            ),
+        ),
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_ordered_list():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+    <ul class="UnorderedList">
+        <li class="ListItem">
+            Item 1
+        </li>
+        <li class="ListItem">
+            Item 2
+        </li>
+        <li class="ListItem">
+            Item 3
+        </li>
+    </ul>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+
+    assert expected_html == parsed_html
+    expected_elements = _page_elements + [
+        Text(
+            text="Item 1 Item 2 Item 3",
+            metadata=ElementMetadata(
+                text_as_html='<ul class="UnorderedList">'
+                '<li class="ListItem">'
+                "Item 1"
+                "</li>"
+                '<li class="ListItem">'
+                "Item 2</li>"
+                '<li class="ListItem">'
+                "Item 3"
+                "</li></ul>",
+            ),
+        )
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_squeezed_elements_are_parsed_back():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+       <p class="NarrativeText">
+        Table of Contents
+       </p>
+       <address class="Address">
+        68 Prince Street Palmdale, CA 93550
+       </address>
+       <a class="Hyperlink">
+        www.google.com
+       </a>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+
+    assert expected_html == parsed_html
+    expected_elements = _page_elements + [
+        NarrativeText(
+            text="Table of Contents 68 Prince Street Palmdale, CA 93550 www.google.com",
+            metadata=ElementMetadata(
+                text_as_html='<p class="NarrativeText">Table of Contents</p>'
+                '<address class="Address">'
+                "68 Prince Street Palmdale, CA 93550"
+                "</address>"
+                '<a class="Hyperlink">www.google.com</a>',
+            ),
+        )
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_inline_elements_are_squeezed_when_text_wrapped_into_paragraphs():
+    # language=HTML
+    base_html = """
+        <div class="Page">
+            About the same
+            <address class="Address">
+                1356 Hornor Avenue Oklahoma
+            </address>
+            Some text
+        </div>
+        """
+    # Such HTML is transformed into Page: [Pargraph, Address, Paragraph]
+    # We would like it to be parsed to UnstructuredElements as [Page, NarrativeText]
+
+    ontology = parse_html_to_ontology(base_html)
+
+    p1, address, p2 = ontology.children
+    assert isinstance(p1, Paragraph)
+    assert isinstance(address, Address)
+    assert isinstance(p2, Paragraph)
+
+    unstructured_elements = ontology_to_unstructured_elements(ontology)
+
+    assert len(unstructured_elements) == 2
+    assert isinstance(unstructured_elements[0], Text)
+    assert isinstance(unstructured_elements[1], NarrativeText)
+
+
+def test_alternate_text_from_image_is_passed():
+    # language=HTML
+    input_html = """
+    <div class="Page">
+    <table>
+        <tr>
+            <td rowspan="2">Example image nested in the table:</td>
+            <td rowspan="2"><img src="my-logo.png" alt="ALT TEXT Logo"></td>
+        </tr>
+    </table>
+    </div>add_img_alt_text
+    """
+    page = parse_html_to_ontology(input_html)
+    unstructured_elements = ontology_to_unstructured_elements(page)
+    assert len(unstructured_elements) == 2
+    assert "ALT TEXT Logo" in unstructured_elements[1].text

test_unstructured/partition/html/test_html_utils.py

@@ -0,0 +1,26 @@
+import pytest
+
+from unstructured.partition.html.transformations import remove_empty_tags_from_html_content
+
+
+@pytest.mark.parametrize(
+    ("html_content, expected_output"),  # noqa PT006
+    [
+        ("<div></div>", ""),
+        ("<div><p></p></div>", "<div></div>"),
+        ("<div><input/></div>", "<div><input/></div>"),
+        ("<div><br/></div>", "<div><br/></div>"),
+        ('<div><p id="1"></p></div>', '<div><p id="1"></p></div>'),
+        ("<div><p>Content</p></div>", "<div><p>Content</p></div>"),
+        ("<div><p> </p></div>", "<div></div>"),
+        ("<div><p></p><span></span></div>", "<div></div>"),
+        ("<div><p>Content</p><span></span></div>", "<div><p>Content</p></div>"),
+        ("<div><p>Content</p><span> </span></div>", "<div><p>Content</p></div>"),
+        (
+            "<div><p>Content</p><span>Text</span></div>",
+            "<div><p>Content</p><span>Text</span></div>",
+        ),
+    ],
+)
+def test_removes_empty_tags(html_content, expected_output):
+    assert remove_empty_tags_from_html_content(html_content) == expected_output

test_unstructured/partition/html/test_partition_v2.py

@@ -0,0 +1,164 @@
+from unstructured.partition.html import partition_html
+
+
+def _by_text(elements):
+    return {(e.text or ""): e for e in elements}
+
+
+def test_category_depth_follows_heading_level_on_a_multi_level_page():
+    """ML-1328 AC1/AC3: a heading's `category_depth` is its heading level (h1->0, h2->1, h3->2),
+    and `parent_id` chains each subsection under its enclosing section heading."""
+    # language=HTML
+    html = """
+    <body class="Document">
+        <div class="Page" data-page-number="1">
+            <h1 class="Title">Cost Share Summary</h1>
+            <p class="NarrativeText">Intro paragraph.</p>
+            <h2 class="Heading">Accumulation Period</h2>
+            <p class="NarrativeText">Accumulation body.</p>
+            <h2 class="Heading">Cost Share Summary Tables by Benefit</h2>
+            <h3 class="Heading">How to read the Cost Share summary tables</h3>
+            <p class="NarrativeText">How-to body.</p>
+        </div>
+    </body>
+    """
+    elements = partition_html(text=html, html_parser_version="v2", unique_element_ids=False)
+    by_text = _by_text(elements)
+
+    section = by_text["Cost Share Summary"]
+    subsection_a = by_text["Accumulation Period"]
+    subsection_b = by_text["Cost Share Summary Tables by Benefit"]
+    sub_subsection = by_text["How to read the Cost Share summary tables"]
+
+    # -- depth tracks heading level --
+    assert section.metadata.category_depth == 0
+    assert subsection_a.metadata.category_depth == 1
+    assert subsection_b.metadata.category_depth == 1
+    assert sub_subsection.metadata.category_depth == 2
+
+    # -- parent_id chains section -> subsection -> sub-subsection --
+    assert subsection_a.metadata.parent_id == section.id
+    assert subsection_b.metadata.parent_id == section.id
+    assert sub_subsection.metadata.parent_id == subsection_b.id
+
+    # -- body text parents to its enclosing heading, not the page/column container --
+    assert by_text["How-to body."].metadata.parent_id == sub_subsection.id
+
+    # -- v2's defining behavior is preserved: every element keeps its text_as_html --
+    assert all(e.metadata.text_as_html for e in elements)
+
+
+def test_category_depth_does_not_change_with_multi_column_layout():
+    """ML-1328 AC2: the same heading reads the same `category_depth` whether the page is single- or
+    multi-column. Layout nesting (Page -> Column) must not bump the depth."""
+    # language=HTML
+    single_column = """
+    <body class="Document">
+        <div class="Page" data-page-number="1">
+            <h1 class="Title">Introduction</h1>
+            <h2 class="Heading">About</h2>
+        </div>
+    </body>
+    """
+    two_column = """
+    <body class="Document">
+        <div class="Page" data-page-number="1">
+            <div class="Column">
+                <h1 class="Title">Introduction</h1>
+                <h2 class="Heading">About</h2>
+            </div>
+            <div class="Column">
+                <h2 class="Heading">Contact</h2>
+            </div>
+        </div>
+    </body>
+    """
+    single = _by_text(
+        partition_html(text=single_column, html_parser_version="v2", unique_element_ids=False)
+    )
+    multi = _by_text(
+        partition_html(text=two_column, html_parser_version="v2", unique_element_ids=False)
+    )
+
+    # -- identical depth across layouts: h1 -> 0, h2 -> 1 -- column wrapping is irrelevant --
+    assert single["Introduction"].metadata.category_depth == 0
+    assert multi["Introduction"].metadata.category_depth == 0
+    assert single["About"].metadata.category_depth == 1
+    assert multi["About"].metadata.category_depth == 1
+    assert multi["Contact"].metadata.category_depth == 1
+
+
+def test_alternative_image_text_can_be_included():
+    # language=HTML
+    html = """
+    <div class="Page">
+        <img src="my-logo.png" alt="ALT TEXT Logo"/>
+    </div>
+    """
+    _, image_to_text_alt_mode = partition_html(
+        text=html,
+        image_alt_mode="to_text",
+        html_parser_version="v2",
+    )
+    assert "ALT TEXT Logo" in image_to_text_alt_mode.text
+
+    _, image_none_alt_mode = partition_html(
+        text=html,
+        image_alt_mode=None,
+        html_parser_version="v2",
+    )
+    assert "ALT TEXT Logo" not in image_none_alt_mode.text
+
+
+def test_alternative_image_text_can_be_included_when_nested_in_paragraph():
+    # language=HTML
+    html = """
+    <div class="Page">
+        <p class="Paragraph">
+            <img src="my-logo.png" alt="ALT TEXT Logo"/>
+        </p>
+    </div>
+    """
+    _, paragraph_to_text_alt_mode = partition_html(
+        text=html,
+        image_alt_mode="to_text",
+        html_parser_version="v2",
+    )
+    assert "ALT TEXT Logo" in paragraph_to_text_alt_mode.text
+
+    _, paragraph_none_alt_mode = partition_html(
+        text=html,
+        image_alt_mode=None,
+        html_parser_version="v2",
+    )
+    assert "ALT TEXT Logo" not in paragraph_none_alt_mode.text
+
+
+def test_attr_and_html_inside_table_cell_is_kept():
+    # language=HTML
+    html = """
+    <div class="Page">
+        <table class="Table">
+            <tbody>
+                <tr>
+                    <td colspan="2">
+                        Some text
+                    </td>
+                    <td>
+                        <input checked="" class="Checkbox" type="checkbox"/>
+                    </td>
+                </tr>
+            </tbody>
+        </table>
+    </div>
+    """
+    page, table = partition_html(
+        text=html,
+        image_alt_mode="to_text",
+        html_parser_version="v2",
+    )
+
+    assert (
+        '<input checked="" class="Checkbox" type="checkbox"/>' in table.metadata.text_as_html
+    )  # class is removed
+    assert 'colspan="2"' in table.metadata.text_as_html

test_unstructured/partition/html/test_unstructured_elements_to_ontology_parsing.py

@@ -0,0 +1,294 @@
+from unstructured.documents.elements import ElementMetadata, NarrativeText, Text
+from unstructured.documents.ontology import Column, Document, Page, Paragraph
+from unstructured.partition.html.transformations import unstructured_elements_to_ontology
+
+
+def test_when_first_elements_does_not_have_id():
+    unstructured_elements = [
+        Text(
+            element_id="1",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        NarrativeText(
+            element_id="2",
+            text="Example text",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Example text </p>', parent_id="1"
+            ),
+        ),
+    ]
+    ontology = unstructured_elements_to_ontology(unstructured_elements)
+
+    assert isinstance(ontology, Document)
+
+    assert len(ontology.children) == 1
+    page = ontology.children[0]
+
+    assert isinstance(page, Page)
+    assert len(page.children) == 1
+    paragraph = page.children[0]
+
+    assert isinstance(paragraph, Paragraph)
+    assert paragraph.text == "Example text"
+
+
+def test_elements_without_text_as_html_are_skipped_not_fatal():
+    # An element with no HTML payload (text_as_html=None) carries nothing to rebuild the
+    # ontology tree from. It must be skipped per-element, not crash the whole conversion.
+    unstructured_elements = [
+        Text(
+            element_id="1",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        NarrativeText(
+            element_id="2",
+            text="no html payload",
+            metadata=ElementMetadata(text_as_html=None, parent_id="1"),
+        ),
+        NarrativeText(
+            element_id="3",
+            text="Example text",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Example text </p>', parent_id="1"
+            ),
+        ),
+    ]
+
+    ontology = unstructured_elements_to_ontology(unstructured_elements)  # must not raise
+
+    assert isinstance(ontology, Document)
+    page = ontology.children[0]
+    assert isinstance(page, Page)
+    # the None-payload element is skipped; the valid one still reconstructs
+    assert len(page.children) == 1
+    assert isinstance(page.children[0], Paragraph)
+    assert page.children[0].text == "Example text"
+
+
+def test_when_two_combined_elements_have_the_same_parent():
+    unstructured_elements = [
+        Text(
+            element_id="1",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        NarrativeText(
+            element_id="2",
+            text="Example text",
+            metadata=ElementMetadata(
+                text_as_html=(
+                    '<p class="Paragraph"> Example text </p>'
+                    '<p class="Paragraph"> Example text 2 </p>'
+                ),
+                parent_id="1",
+            ),
+        ),
+        NarrativeText(
+            element_id="3",
+            text="Example text 2",
+            metadata=ElementMetadata(
+                text_as_html=(
+                    '<p class="Paragraph"> Example text 3 </p>'
+                    '<p class="Paragraph"> Example text 4 </p>'
+                ),
+                parent_id="1",
+            ),
+        ),
+    ]
+    ontology = unstructured_elements_to_ontology(unstructured_elements)
+
+    assert isinstance(ontology, Document)
+
+    assert len(ontology.children) == 1
+    page = ontology.children[0]
+
+    assert isinstance(page, Page)
+    assert len(page.children) == 4
+
+
+def test_content_element_without_parent_nests_in_current_container():
+    """ML-1328: tree reconstruction is layout-container driven, not content-`parent_id` driven.
+
+    Content elements nest in the innermost open layout container regardless of their (now
+    heading-based, possibly absent) `parent_id`. A paragraph with no `parent_id` therefore lands in
+    the current Page rather than being lifted to a Document-level sibling -- and is not lost.
+    """
+    unstructured_elements = [
+        Text(
+            element_id="1",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        NarrativeText(
+            element_id="2",
+            text="Example text",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Example text </p>', parent_id="1"
+            ),
+        ),
+        NarrativeText(
+            element_id="3",
+            text="Example text without parent",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Example text without parent </p>'
+            ),
+        ),
+    ]
+    ontology = unstructured_elements_to_ontology(unstructured_elements)
+
+    assert isinstance(ontology, Document)
+    assert len(ontology.children) == 1
+    page = ontology.children[0]
+    assert isinstance(page, Page)
+    assert len(page.children) == 2
+    first, second = page.children
+    assert first.text == "Example text"
+    assert second.text == "Example text without parent"
+
+
+def test_multiple_pages_can_be_combined():
+    unstructured_elements = [
+        Text(
+            element_id="1",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        NarrativeText(
+            element_id="2",
+            text="Example text on page 1",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Example text on page 1 </p>', parent_id="1"
+            ),
+        ),
+        Text(
+            element_id="3",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        NarrativeText(
+            element_id="4",
+            text="Example text on page 2",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Example text on page 2 </p>', parent_id="3"
+            ),
+        ),
+    ]
+    ontology = unstructured_elements_to_ontology(unstructured_elements)
+
+    assert isinstance(ontology, Document)
+    assert len(ontology.children) == 2
+    page1 = ontology.children[0]
+    page2 = ontology.children[1]
+    assert isinstance(page1, Page)
+    assert isinstance(page2, Page)
+    assert len(page1.children) == 1
+    assert len(page2.children) == 1
+    paragraph1 = page1.children[0]
+    paragraph2 = page2.children[0]
+    assert isinstance(paragraph1, Paragraph)
+    assert isinstance(paragraph2, Paragraph)
+    assert paragraph1.text == "Example text on page 1"
+    assert paragraph2.text == "Example text on page 2"
+
+
+def test_empty_input_returns_empty_document():
+    """ML-1328: empty input must return an empty Document, not raise IndexError."""
+    ontology = unstructured_elements_to_ontology([])
+
+    assert isinstance(ontology, Document)
+    assert ontology.children == []
+
+
+def test_nested_layout_containers_rebuild_column_nesting():
+    """ML-1328: a Column nests inside its Page (container `parent_id` drives the layout tree).
+
+    Two columns under one page; content nests in the innermost open container. This is the
+    multi-column round-trip that exercises container-under-container nesting.
+    """
+    unstructured_elements = [
+        Text(
+            element_id="page",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        Text(
+            element_id="col1",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Column"/>', parent_id="page"),
+        ),
+        NarrativeText(
+            element_id="c1",
+            text="Left column text",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Left column text </p>', parent_id="col1"
+            ),
+        ),
+        Text(
+            element_id="col2",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Column"/>', parent_id="page"),
+        ),
+        NarrativeText(
+            element_id="c2",
+            text="Right column text",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Right column text </p>', parent_id="col2"
+            ),
+        ),
+    ]
+    ontology = unstructured_elements_to_ontology(unstructured_elements)
+
+    assert isinstance(ontology, Document)
+    assert len(ontology.children) == 1
+    page = ontology.children[0]
+    assert isinstance(page, Page)
+    # -- both columns nest under the page; the second column did not pop past the page to root --
+    assert len(page.children) == 2
+    col1, col2 = page.children
+    assert isinstance(col1, Column)
+    assert isinstance(col2, Column)
+    assert [c.text for c in col1.children] == ["Left column text"]
+    assert [c.text for c in col2.children] == ["Right column text"]
+
+
+def test_layout_container_with_unknown_parent_id_does_not_pop_to_root():
+    """ML-1328: a container whose `parent_id` matches no open container nests in the current one.
+
+    Malformed/reordered input (violating the documented parent-before-child precondition) must not
+    pop past valid ancestors to the Document root and mis-nest subsequent content. The Column here
+    references a non-existent parent; it should stay inside the open Page, and nothing is lost.
+    """
+    unstructured_elements = [
+        Text(
+            element_id="page",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        Text(
+            element_id="col",
+            text="",
+            metadata=ElementMetadata(
+                text_as_html='<div class="Column"/>', parent_id="DOES_NOT_EXIST"
+            ),
+        ),
+        NarrativeText(
+            element_id="c1",
+            text="Body text",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Body text </p>', parent_id="col"
+            ),
+        ),
+    ]
+    ontology = unstructured_elements_to_ontology(unstructured_elements)
+
+    assert isinstance(ontology, Document)
+    assert len(ontology.children) == 1
+    page = ontology.children[0]
+    assert isinstance(page, Page)
+    # -- the Column stayed nested in the Page rather than being lifted to a Document-level sibling
+    assert len(page.children) == 1
+    column = page.children[0]
+    assert isinstance(column, Column)
+    assert [c.text for c in column.children] == ["Body text"]

test_unstructured/partition/html/test_xss_sanitization.py

@@ -0,0 +1,254 @@
+"""End-to-end regression tests for the stored-XSS fix (GHSA-v5mq-3xhg-98m9).
+
+These exercise the two production sinks:
+
+* ``elements_to_html`` — the assembled HTML document,
+* ``ElementMetadata.text_as_html`` — which some callers return to clients verbatim,
+
+feeding them the exact proof-of-concept from the advisory and asserting every
+vector is neutralized, while legitimate formatting is preserved.
+"""
+
+from __future__ import annotations
+
+import re
+from urllib.parse import urlparse
+
+from bs4 import BeautifulSoup
+
+from unstructured.documents.elements import ElementMetadata, Text
+from unstructured.partition.html import partition_html
+from unstructured.partition.html.convert import elements_to_html
+
+# -- The advisory's proof-of-concept document. --
+MALICIOUS_HTML = (
+    '<body class="Document">'
+    '<p class="Paragraph" onmouseover="alert(1)">hello'
+    '<img src=x onerror="alert(document.domain)"></p>'
+    '<a class="Hyperlink" href="javascript:alert(document.cookie)">click me</a>'
+    "</body>"
+)
+
+
+def _partition_v2(html_text: str):
+    return list(partition_html(text=html_text, html_parser_version="v2"))
+
+
+def _all_text_as_html(elements) -> str:
+    return " ".join(e.metadata.text_as_html or "" for e in elements)
+
+
+def _href_values(html: str) -> list[str]:
+    return [
+        str(anchor["href"])
+        for anchor in BeautifulSoup(html, "html.parser").find_all("a", href=True)
+    ]
+
+
+class DescribeElementsToHtmlIsInert:
+    """The four advisory vectors must not survive into elements_to_html output."""
+
+    def it_strips_event_handler_attributes(self):
+        out = elements_to_html(_partition_v2(MALICIOUS_HTML), no_group_by_page=True)
+        assert "onmouseover" not in out
+        assert "onerror" not in out
+        assert "onload" not in out
+
+    def it_neutralizes_javascript_hrefs(self):
+        out = elements_to_html(_partition_v2(MALICIOUS_HTML), no_group_by_page=True)
+        assert "javascript:" not in out.lower()
+
+    def it_has_no_live_svg_from_attribute_breakout(self):
+        # -- title='"><svg onload=...>' must not break out of the quoted value --
+        breakout = (
+            '<body class="Document">'
+            '<p class="Paragraph" title=\'"><svg onload=alert(1)>\'>x</p>'
+            "</body>"
+        )
+        out = elements_to_html(_partition_v2(breakout), no_group_by_page=True)
+        # -- no live <svg> tag; the payload is escaped to inert text --
+        assert not re.search(r"<svg", out)
+        assert "&lt;svg" in out
+
+    def it_adds_safe_rel_for_blank_targets(self):
+        blank_target = (
+            '<body class="Document">'
+            '<a class="Hyperlink" href="https://example.com" target="_blank">x</a>'
+            "</body>"
+        )
+        out = elements_to_html(_partition_v2(blank_target), no_group_by_page=True)
+        assert 'target="_blank"' in out
+        assert "noopener" in out
+        assert "noreferrer" in out
+
+    def it_filters_overlay_styles(self):
+        styled = (
+            '<body class="Document">'
+            '<p class="Paragraph" style="position:fixed;inset:0;z-index:9999;color:red">x</p>'
+            "</body>"
+        )
+        out = elements_to_html(_partition_v2(styled), no_group_by_page=True)
+        assert "position" not in out
+        assert "inset" not in out
+        assert "z-index" not in out
+        assert "color:red" in out
+
+
+class DescribeTextAsHtmlIsInert:
+    """`text_as_html` is a value some callers return to clients directly."""
+
+    def it_strips_event_handlers_from_text_as_html(self):
+        tah = _all_text_as_html(_partition_v2(MALICIOUS_HTML))
+        assert "onmouseover" not in tah
+        assert "onerror" not in tah
+
+    def it_neutralizes_javascript_hrefs_in_text_as_html(self):
+        tah = _all_text_as_html(_partition_v2(MALICIOUS_HTML))
+        assert "javascript:" not in tah.lower()
+
+    def it_escapes_the_attribute_breakout_quote(self):
+        breakout = (
+            '<body class="Document">'
+            '<p class="Paragraph" title=\'"><svg onload=alert(1)>\'>x</p>'
+            "</body>"
+        )
+        tah = _all_text_as_html(_partition_v2(breakout))
+        # -- the double-quote that would break out is encoded --
+        assert "&quot;" in tah
+        assert "&lt;svg" in tah
+        assert not re.search(r"<svg", tah)
+
+    def it_drops_meta_refresh_attributes_from_text_as_html(self):
+        meta_refresh = (
+            '<body class="Document">'
+            '<meta class="Keywords" http-equiv="refresh" content="0;url=javascript:alert(1)">'
+            "</body>"
+        )
+        tah = _all_text_as_html(_partition_v2(meta_refresh))
+        assert "http-equiv" not in tah
+        assert "refresh" not in tah
+
+    def it_drops_form_navigation_attributes_from_text_as_html(self):
+        form_payload = (
+            '<body class="Document">'
+            '<form class="Form" action="javascript:alert(1)">'
+            '<input class="FormFieldValue" value="x" formaction="javascript:alert(2)">'
+            "</form>"
+            "</body>"
+        )
+        tah = _all_text_as_html(_partition_v2(form_payload))
+        assert "action=" not in tah
+        assert "formaction" not in tah
+        assert "javascript:" not in tah.lower()
+
+    def it_drops_srcset_from_text_as_html(self):
+        srcset_payload = (
+            '<body class="Document">'
+            '<img class="Image" src="https://example.com/ok.png" '
+            'srcset="javascript:alert(1) 1x">'
+            "</body>"
+        )
+        tah = _all_text_as_html(_partition_v2(srcset_payload))
+        assert "srcset" not in tah
+        assert "javascript:" not in tah.lower()
+
+    def it_rejects_svg_data_images_in_text_as_html(self):
+        svg_payload = (
+            '<body class="Document">'
+            '<img class="Image" src="data:image/svg+xml;base64,PHN2Zz48L3N2Zz4=">'
+            "</body>"
+        )
+        tah = _all_text_as_html(_partition_v2(svg_payload))
+        assert "data:image/svg" not in tah
+
+    def it_adds_safe_rel_for_blank_targets_in_text_as_html(self):
+        blank_target = (
+            '<body class="Document">'
+            '<a class="Hyperlink" href="https://example.com" target="_blank">x</a>'
+            "</body>"
+        )
+        tah = _all_text_as_html(_partition_v2(blank_target))
+        assert 'target="_blank"' in tah
+        assert "noopener" in tah
+        assert "noreferrer" in tah
+
+    def it_filters_overlay_styles_in_text_as_html(self):
+        styled = (
+            '<body class="Document">'
+            '<p class="Paragraph" style="position:fixed;inset:0;z-index:9999;color:red">x</p>'
+            "</body>"
+        )
+        tah = _all_text_as_html(_partition_v2(styled))
+        assert "position" not in tah
+        assert "inset" not in tah
+        assert "z-index" not in tah
+        assert "color: red" in tah
+
+
+class DescribeLinkUrlInjectionViaMetadata:
+    """`elements_to_html` builds `href` from `metadata.url` outside the ontology
+    emitter, so the nh3 sweep must cover it too."""
+
+    def it_drops_a_javascript_url_from_link_metadata(self):
+        el = Text(
+            text="click me",
+            metadata=ElementMetadata(url="javascript:alert(1)"),
+        )
+        el.category = "Link"  # type: ignore[assignment]
+        out = elements_to_html([el], no_group_by_page=True)
+        assert "javascript:" not in out.lower()
+
+    def it_keeps_an_https_url_from_link_metadata(self):
+        el = Text(
+            text="click me",
+            metadata=ElementMetadata(url="https://example.com"),
+        )
+        el.category = "Link"  # type: ignore[assignment]
+        out = elements_to_html([el], no_group_by_page=True)
+        hrefs = _href_values(out)
+        assert any(
+            (p := urlparse(href)).scheme == "https" and p.hostname == "example.com"
+            for href in hrefs
+        )
+
+
+class DescribeLegitimateFormattingPreserved:
+    """The fix must not strip benign structure/formatting."""
+
+    def it_preserves_tables_with_colspan_and_borders(self):
+        tbl = (
+            '<body class="Document"><table class="Table"><tbody>'
+            '<tr><td colspan="2">A&amp;B</td></tr>'
+            "<tr><td>1</td><td>2</td></tr></tbody></table></body>"
+        )
+        out = elements_to_html(_partition_v2(tbl), no_group_by_page=True)
+        assert "<table" in out
+        assert 'colspan="2"' in out
+        # -- the ampersand text is HTML-escaped, not dropped --
+        assert "A&amp;B" in out
+
+    def it_preserves_safe_links_and_headings(self):
+        doc = (
+            '<body class="Document">'
+            '<h1 class="Title">Heading</h1>'
+            '<a class="Hyperlink" href="https://example.com/page">link</a>'
+            "</body>"
+        )
+        out = elements_to_html(_partition_v2(doc), no_group_by_page=True)
+        assert "https://example.com/page" in _href_values(out)
+        assert "Heading" in out
+
+    def it_preserves_mailto_scheme_in_text_as_html(self):
+        # -- `mailto:` is a safe scheme and must survive into `text_as_html`
+        # -- (a value some callers return to clients directly) --
+        doc = '<body class="Document"><a class="Hyperlink" href="mailto:a@b.com">email</a></body>'
+        tah = _all_text_as_html(_partition_v2(doc))
+        assert "mailto:a@b.com" in tah
+
+    def it_preserves_base64_images(self):
+        img_src = "data:image/png;base64,iVBORw0KGgo="
+        el = Text(text="", metadata=ElementMetadata(image_base64="iVBORw0KGgo="))
+        el.category = "Image"  # type: ignore[assignment]
+        el.metadata.image_mime_type = "image/png"
+        out = elements_to_html([el], no_group_by_page=True)
+        assert img_src in out

test_unstructured/partition/pdf_image/conftest.py

@@ -0,0 +1,78 @@
+import pytest
+from unstructured_inference.inference.elements import EmbeddedTextRegion
+
+
+@pytest.fixture()
+def mock_embedded_text_regions():
+    return [
+        EmbeddedTextRegion.from_coords(
+            x1=453.00277777777774,
+            y1=317.319341111111,
+            x2=711.5338541666665,
+            y2=358.28571222222206,
+            text="LayoutParser:",
+        ),
+        EmbeddedTextRegion.from_coords(
+            x1=726.4778125,
+            y1=317.319341111111,
+            x2=760.3308594444444,
+            y2=357.1698966666667,
+            text="A",
+        ),
+        EmbeddedTextRegion.from_coords(
+            x1=775.2748177777777,
+            y1=317.319341111111,
+            x2=917.3579885555555,
+            y2=357.1698966666667,
+            text="Unified",
+        ),
+        EmbeddedTextRegion.from_coords(
+            x1=932.3019468888888,
+            y1=317.319341111111,
+            x2=1071.8426522222221,
+            y2=357.1698966666667,
+            text="Toolkit",
+        ),
+        EmbeddedTextRegion.from_coords(
+            x1=1086.7866105555556,
+            y1=317.319341111111,
+            x2=1141.2105142777777,
+            y2=357.1698966666667,
+            text="for",
+        ),
+        EmbeddedTextRegion.from_coords(
+            x1=1156.154472611111,
+            y1=317.319341111111,
+            x2=1256.334784222222,
+            y2=357.1698966666667,
+            text="Deep",
+        ),
+        EmbeddedTextRegion.from_coords(
+            x1=437.83888888888885,
+            y1=367.13322999999986,
+            x2=610.0171992222222,
+            y2=406.9837855555556,
+            text="Learning",
+        ),
+        EmbeddedTextRegion.from_coords(
+            x1=624.9611575555555,
+            y1=367.13322999999986,
+            x2=741.6754646666665,
+            y2=406.9837855555556,
+            text="Based",
+        ),
+        EmbeddedTextRegion.from_coords(
+            x1=756.619423,
+            y1=367.13322999999986,
+            x2=958.3867708333332,
+            y2=406.9837855555556,
+            text="Document",
+        ),
+        EmbeddedTextRegion.from_coords(
+            x1=973.3307291666665,
+            y1=367.13322999999986,
+            x2=1092.0535042777776,
+            y2=406.9837855555556,
+            text="Image",
+        ),
+    ]

test_unstructured/partition/pdf_image/test_analysis.py

@@ -0,0 +1,154 @@
+import numpy as np
+import pytest
+from PIL import Image
+from unstructured_inference.inference.elements import Rectangle
+from unstructured_inference.inference.layout import DocumentLayout, PageLayout
+from unstructured_inference.inference.layoutelement import LayoutElement
+
+from unstructured.partition.pdf_image.analysis.bbox_visualisation import (
+    TextAlignment,
+    get_bbox_text_size,
+    get_bbox_thickness,
+    get_label_rect_and_coords,
+    get_rgb_color,
+    get_text_color,
+)
+from unstructured.partition.pdf_image.analysis.layout_dump import ObjectDetectionLayoutDumper
+
+
+@pytest.mark.parametrize("color", ["red", "green", "blue", "yellow", "black", "white"])
+def test_get_rgb_color(color: str):
+    color_tuple = get_rgb_color(color)
+
+    assert isinstance(color_tuple, tuple)
+    assert len(color_tuple) == 3
+    assert all(isinstance(c, int) for c in color_tuple)
+    assert all(0 <= c <= 255 for c in color_tuple)
+
+
+@pytest.mark.parametrize(
+    ("bbox", "expected_text_size"),
+    [
+        ((0, 0, 90, 90), 17),
+        ((0, 0, 500, 200), 21),
+        ((0, 0, 10000, 10000), 32),
+    ],
+)
+def test_get_bbox_text_size(bbox: tuple[int, int, int, int], expected_text_size):
+    page_size = (1700, 2200)  # standard size of a page
+    text_size = get_bbox_text_size(bbox, page_size)
+
+    assert text_size == expected_text_size
+
+
+@pytest.mark.parametrize(
+    ("bbox", "expected_box_thickness"),
+    [
+        ((0, 0, 90, 90), 1),
+        ((0, 0, 450, 250), 2),
+        ((0, 0, 600, 1000), 3),
+    ],
+)
+def test_get_bbox_thickness(bbox: tuple[int, int, int, int], expected_box_thickness):
+    page_size = (1700, 2200)  # standard size of a page
+    box_thickness = get_bbox_thickness(bbox, page_size)
+
+    assert box_thickness == expected_box_thickness
+
+
+@pytest.mark.parametrize(
+    ("color", "expected_text_color"),
+    [
+        ("navy", "white"),
+        ("crimson", "white"),
+        ("maroon", "white"),
+        ("dimgray", "white"),
+        ("darkgreen", "white"),
+        ("darkcyan", "white"),
+        ("fuchsia", "white"),
+        ("violet", "black"),
+        ("gold", "black"),
+        ("aqua", "black"),
+        ("greenyellow", "black"),
+    ],
+)
+def test_best_text_color(color, expected_text_color):
+    color_tuple = get_rgb_color(color)
+    expected_text_color_tuple = get_rgb_color(expected_text_color)
+
+    _, text_color_tuple = get_text_color(color_tuple)
+    assert text_color_tuple == expected_text_color_tuple
+
+
+@pytest.mark.parametrize(
+    ("alignment", "expected_text_bbox"),
+    [
+        (TextAlignment.CENTER, ((145, 145), (155, 155))),
+        (TextAlignment.TOP_LEFT, ((100, 90), (120, 100))),
+        (TextAlignment.TOP_RIGHT, ((180, 100), (200, 110))),
+        (TextAlignment.BOTTOM_LEFT, ((100, 190), (120, 200))),
+        (TextAlignment.BOTTOM_RIGHT, ((180, 190), (200, 200))),
+    ],
+)
+def test_get_text_bbox(alignment, expected_text_bbox):
+    text_bbox, text_xy = get_label_rect_and_coords(
+        alignment=alignment, bbox_points=(100, 100, 200, 200), text_width=10, text_height=10
+    )
+    # adding high atol to account for the text-based extending of the bbox
+    assert np.allclose(text_bbox, expected_text_bbox, atol=10)
+
+
+def test_od_document_layout_dump():
+    page1 = PageLayout(
+        number=1,
+        image=Image.new("1", (1, 1)),
+        image_metadata={"width": 100, "height": 100},
+    )
+    page1.elements = [
+        LayoutElement(type="Title", bbox=Rectangle(x1=0, y1=0, x2=10, y2=10), prob=0.7),
+        LayoutElement(type="Paragraph", bbox=Rectangle(x1=0, y1=100, x2=10, y2=110), prob=0.8),
+    ]
+    page2 = PageLayout(
+        number=2,
+        image=Image.new("1", (1, 1)),
+        image_metadata={"width": 100, "height": 100},
+    )
+    page2.elements = [
+        LayoutElement(type="Table", bbox=Rectangle(x1=0, y1=0, x2=10, y2=10), prob=0.9),
+        LayoutElement(type="Image", bbox=Rectangle(x1=0, y1=100, x2=10, y2=110), prob=1.0),
+    ]
+    od_document_layout = DocumentLayout(pages=[page1, page2])
+
+    expected_dump = {
+        "pages": [
+            {
+                "number": 1,
+                "size": {
+                    "width": 100,
+                    "height": 100,
+                },
+                "elements": [
+                    {"bbox": [0, 0, 10, 10], "type": "Title", "prob": 0.7},
+                    {"bbox": [0, 100, 10, 110], "type": "Paragraph", "prob": 0.8},
+                ],
+            },
+            {
+                "number": 2,
+                "size": {
+                    "width": 100,
+                    "height": 100,
+                },
+                "elements": [
+                    {"bbox": [0, 0, 10, 10], "type": "Table", "prob": 0.9},
+                    {"bbox": [0, 100, 10, 110], "type": "Image", "prob": 1.0},
+                ],
+            },
+        ]
+    }
+    od_layout_dump = ObjectDetectionLayoutDumper(od_document_layout).dump()
+
+    assert expected_dump == {"pages": od_layout_dump.get("pages")}
+
+    # check OD model classes are attached but do not depend on a specific model instance
+    assert "object_detection_classes" in od_layout_dump
+    assert len(od_layout_dump["object_detection_classes"]) > 0

test_unstructured/partition/pdf_image/test_image.py

@@ -0,0 +1,674 @@
+from __future__ import annotations
+
+import os
+import pathlib
+import tempfile
+from unittest import mock
+
+import pytest
+from PIL import Image
+from pytest_mock import MockFixture
+from unstructured_inference.inference import layout
+from unstructured_pytesseract import TesseractError
+
+from test_unstructured.partition.pdf_image.test_pdf import assert_element_extraction
+from test_unstructured.unit_utils import assert_round_trips_through_JSON, example_doc_path
+from unstructured.chunking.title import chunk_by_title
+from unstructured.documents.elements import ElementType
+from unstructured.partition import image, pdf
+from unstructured.partition.pdf_image import ocr
+from unstructured.partition.utils.constants import (
+    UNSTRUCTURED_INCLUDE_DEBUG_METADATA,
+    PartitionStrategy,
+)
+from unstructured.utils import only
+
+DIRECTORY = pathlib.Path(__file__).parent.resolve()
+
+
+class MockResponse:
+    def __init__(self, status_code, response):
+        self.status_code = status_code
+        self.response = response
+
+    def json(self):
+        return self.response
+
+
+def mock_healthy_get(url, **kwargs):
+    return MockResponse(status_code=200, response={})
+
+
+def mock_unhealthy_get(url, **kwargs):
+    return MockResponse(status_code=500, response={})
+
+
+def mock_unsuccessful_post(url, **kwargs):
+    return MockResponse(status_code=500, response={})
+
+
+def mock_successful_post(url, **kwargs):
+    response = {
+        "pages": [
+            {
+                "number": 0,
+                "elements": [
+                    {"type": "Title", "text": "Charlie Brown and the Great Pumpkin"},
+                ],
+            },
+            {
+                "number": 1,
+                "elements": [{"type": "Title", "text": "A Charlie Brown Christmas"}],
+            },
+        ],
+    }
+    return MockResponse(status_code=200, response=response)
+
+
+class MockPageLayout(layout.PageLayout):
+    def __init__(self, number: int, image: Image):
+        self.number = number
+        self.image = image
+        self.image_metadata = {"pdf_rotation": 0}
+        self.elements = [
+            layout.LayoutElement.from_coords(
+                type="Title",
+                x1=0,
+                y1=0,
+                x2=2,
+                y2=2,
+                text="Charlie Brown and the Great Pumpkin",
+            ),
+        ]
+        self.elements_array = layout.LayoutElements.from_list(self.elements)
+
+
+class MockDocumentLayout(layout.DocumentLayout):
+    @property
+    def pages(self):
+        return [
+            MockPageLayout(number=0, image=Image.new("1", (1, 1))),
+        ]
+
+
+@pytest.mark.parametrize(
+    ("filename", "file"),
+    [
+        (example_doc_path("img/example.jpg"), None),
+        (None, b"0000"),
+    ],
+)
+def test_partition_image_local(monkeypatch, filename, file):
+    monkeypatch.setattr(
+        layout,
+        "process_data_with_model",
+        lambda *args, **kwargs: MockDocumentLayout(),
+    )
+    monkeypatch.setattr(
+        layout,
+        "process_file_with_model",
+        lambda *args, **kwargs: MockDocumentLayout(),
+    )
+    monkeypatch.setattr(
+        ocr,
+        "process_data_with_ocr",
+        lambda *args, **kwargs: MockDocumentLayout(),
+    )
+    monkeypatch.setattr(
+        ocr,
+        "process_data_with_ocr",
+        lambda *args, **kwargs: MockDocumentLayout(),
+    )
+
+    partition_image_response = pdf._partition_pdf_or_image_local(
+        filename,
+        file,
+        is_image=True,
+    )
+    assert partition_image_response[0].text == "Charlie Brown and the Great Pumpkin"
+
+
+@pytest.mark.skip("Needs to be fixed upstream in unstructured-inference")
+def test_partition_image_local_raises_with_no_filename():
+    with pytest.raises(FileNotFoundError):
+        pdf._partition_pdf_or_image_local(filename="", file=None, is_image=True)
+
+
+def test_partition_image_with_auto_strategy():
+    filename = example_doc_path("img/layout-parser-paper-fast.jpg")
+    elements = image.partition_image(filename=filename, strategy=PartitionStrategy.AUTO)
+    titles = [
+        el for el in elements if el.category == ElementType.TITLE and len(el.text.split(" ")) > 10
+    ]
+    title = "LayoutParser: A Unified Toolkit for Deep Learning Based Document Image Analysis"
+    idx = 3
+    assert titles[0].text == title
+    assert elements[idx].metadata.detection_class_prob is not None
+    assert isinstance(elements[idx].metadata.detection_class_prob, float)
+
+
+def test_partition_image_with_table_extraction():
+    filename = example_doc_path("img/layout-parser-paper-with-table.jpg")
+    elements = image.partition_image(
+        filename=filename,
+        strategy=PartitionStrategy.HI_RES,
+        infer_table_structure=True,
+    )
+    table = [el.metadata.text_as_html for el in elements if el.metadata.text_as_html]
+    assert len(table) == 1
+    assert "<table><thead><tr>" in table[0]
+    assert "</thead><tbody><tr>" in table[0]
+
+
+def test_partition_image_with_multipage_tiff():
+    filename = example_doc_path("img/layout-parser-paper-combined.tiff")
+    elements = image.partition_image(filename=filename, strategy=PartitionStrategy.AUTO)
+    assert elements[-1].metadata.page_number == 2
+
+
+def test_partition_image_with_bmp(tmpdir):
+    filename = example_doc_path("img/layout-parser-paper-with-table.jpg")
+    bmp_filename = os.path.join(tmpdir.dirname, "example.bmp")
+    img = Image.open(filename)
+    img.save(bmp_filename)
+
+    elements = image.partition_image(
+        filename=bmp_filename,
+        strategy=PartitionStrategy.HI_RES,
+        infer_table_structure=True,
+    )
+    table = [el.metadata.text_as_html for el in elements if el.metadata.text_as_html]
+    assert len(table) == 1
+    assert "<table><thead><tr>" in table[0]
+    assert "</thead><tbody><tr>" in table[0]
+
+
+def test_partition_image_with_language_passed():
+    filename = example_doc_path("img/example.jpg")
+    with mock.patch.object(
+        ocr,
+        "process_file_with_ocr",
+        mock.MagicMock(),
+    ) as mock_partition:
+        image.partition_image(
+            filename=filename,
+            strategy=PartitionStrategy.HI_RES,
+            ocr_languages="eng+swe",
+        )
+
+    assert mock_partition.call_args.kwargs.get("ocr_languages") == "eng+swe"
+
+
+def test_partition_image_from_file_with_language_passed():
+    filename = example_doc_path("img/example.jpg")
+    with (
+        mock.patch.object(
+            ocr,
+            "process_data_with_ocr",
+            mock.MagicMock(),
+        ) as mock_partition,
+        open(filename, "rb") as f,
+    ):
+        image.partition_image(file=f, strategy=PartitionStrategy.HI_RES, ocr_languages="eng+swe")
+
+    assert mock_partition.call_args.kwargs.get("ocr_languages") == "eng+swe"
+
+
+# NOTE(crag): see https://github.com/Unstructured-IO/unstructured/issues/1086
+@pytest.mark.skip(reason="Current catching too many tesseract errors")
+def test_partition_image_raises_with_invalid_language():
+    filename = example_doc_path("img/example.jpg")
+    with pytest.raises(TesseractError):
+        image.partition_image(
+            filename=filename,
+            strategy=PartitionStrategy.HI_RES,
+            ocr_languages="fakeroo",
+        )
+
+
+@pytest.mark.parametrize(
+    "strategy",
+    [
+        PartitionStrategy.HI_RES,
+        PartitionStrategy.OCR_ONLY,
+    ],
+)
+def test_partition_image_strategies_keep_languages_metadata(strategy):
+    filename = example_doc_path("img/english-and-korean.png")
+    elements = image.partition_image(
+        filename=filename,
+        languages=["eng", "kor"],
+        strategy=strategy,
+    )
+
+    assert elements[0].metadata.languages == ["eng", "kor"]
+
+
+def test_partition_image_with_ocr_detects_korean():
+    filename = example_doc_path("img/english-and-korean.png")
+    elements = image.partition_image(
+        filename=filename,
+        ocr_languages="eng+kor",
+        strategy=PartitionStrategy.OCR_ONLY,
+    )
+
+    assert elements[0].text == "RULES AND INSTRUCTIONS"
+    # FIXME (yao): revisit this lstrip after refactoring merging logics; right now on docker and
+    # local testing yield different results and on docker there is a "," at the start of the Korean
+    # text line
+    assert elements[3].text.replace(" ", "").lstrip(",").startswith("안녕하세요")
+
+
+def test_partition_image_with_ocr_detects_korean_from_file():
+    filename = example_doc_path("img/english-and-korean.png")
+    with open(filename, "rb") as f:
+        elements = image.partition_image(
+            file=f,
+            ocr_languages="eng+kor",
+            strategy=PartitionStrategy.OCR_ONLY,
+        )
+
+    assert elements[0].text == "RULES AND INSTRUCTIONS"
+    assert elements[3].text.replace(" ", "").lstrip(",").startswith("안녕하세요")
+
+
+def test_partition_image_raises_with_bad_strategy():
+    filename = example_doc_path("img/english-and-korean.png")
+    with pytest.raises(ValueError):
+        image.partition_image(filename=filename, strategy="fakeroo")
+
+
+def test_partition_image_default_strategy_hi_res():
+    filename = example_doc_path("img/layout-parser-paper-fast.jpg")
+    with open(filename, "rb") as f:
+        elements = image.partition_image(file=f)
+
+    title = "LayoutParser: A Unified Toolkit for Deep Learning Based Document Image Analysis"
+    idx = 2
+    assert elements[idx].text == title
+    assert elements[idx].metadata.coordinates is not None
+    assert elements[idx].metadata.detection_class_prob is not None
+    assert isinstance(elements[idx].metadata.detection_class_prob, float)
+    if UNSTRUCTURED_INCLUDE_DEBUG_METADATA:
+        # A bug in partition_groups_from_regions in unstructured-inference losses some sources
+        assert {element.metadata.detection_origin for element in elements} == {
+            "yolox",
+            "ocr_tesseract",
+        }
+
+
+# -- .metadata.last_modified ---------------------------------------------------------------------
+
+
+def test_partition_image_from_file_path_gets_last_modified_from_filesystem(mocker: MockFixture):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.pdf.get_last_modified_date",
+        return_value=filesystem_last_modified,
+    )
+
+    elements = image.partition_image(example_doc_path("img/english-and-korean.png"))
+
+    assert all(e.metadata.last_modified == filesystem_last_modified for e in elements)
+
+
+def test_partition_image_from_file_path_with_hi_res_strategy_gets_last_modified_from_filesystem(
+    mocker: MockFixture,
+):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.pdf.get_last_modified_date",
+        return_value=filesystem_last_modified,
+    )
+
+    elements = image.partition_image(
+        example_doc_path("img/english-and-korean.png"), strategy=PartitionStrategy.HI_RES
+    )
+
+    assert all(e.metadata.last_modified == filesystem_last_modified for e in elements)
+
+
+def test_partition_image_from_file_path_prefers_metadata_last_modified(mocker: MockFixture):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    metadata_last_modified = "2009-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.pdf.get_last_modified_date",
+        return_value=filesystem_last_modified,
+    )
+
+    elements = image.partition_image(
+        example_doc_path("img/english-and-korean.png"),
+        metadata_last_modified=metadata_last_modified,
+    )
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+def test_partition_image_from_file_path_with_hi_res_strategy_prefers_metadata_last_modified(
+    mocker: MockFixture,
+):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    metadata_last_modified = "2009-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.pdf.get_last_modified_date",
+        return_value=filesystem_last_modified,
+    )
+
+    elements = image.partition_image(
+        example_doc_path("img/english-and-korean.png"),
+        strategy=PartitionStrategy.HI_RES,
+        metadata_last_modified=metadata_last_modified,
+    )
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+def test_partition_image_from_file_gets_last_modified_None():
+    with open(example_doc_path("img/english-and-korean.png"), "rb") as f:
+        elements = image.partition_image(file=f)
+
+    assert all(e.metadata.last_modified is None for e in elements)
+
+
+def test_partition_image_from_file_with_hi_res_strategy_gets_last_modified_None(
+    mocker: MockFixture,
+):
+    with open(example_doc_path("img/english-and-korean.png"), "rb") as f:
+        elements = image.partition_image(file=f, strategy=PartitionStrategy.HI_RES)
+
+    assert all(e.metadata.last_modified is None for e in elements)
+
+
+def test_partition_image_from_file_prefers_metadata_last_modified():
+    metadata_last_modified = "2009-07-05T09:24:28"
+
+    with open(example_doc_path("img/english-and-korean.png"), "rb") as f:
+        elements = image.partition_image(file=f, metadata_last_modified=metadata_last_modified)
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+def test_partition_image_from_file_with_hi_res_strategy_prefers_metadata_last_modified():
+    metadata_last_modified = "2009-07-05T09:24:28"
+
+    with open(example_doc_path("img/english-and-korean.png"), "rb") as f:
+        elements = image.partition_image(
+            file=f,
+            metadata_last_modified=metadata_last_modified,
+            strategy=PartitionStrategy.HI_RES,
+        )
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+# ------------------------------------------------------------------------------------------------
+
+
+def test_partition_msg_with_json():
+    elements = image.partition_image(
+        example_doc_path("img/layout-parser-paper-fast.jpg"),
+        strategy=PartitionStrategy.AUTO,
+    )
+    assert_round_trips_through_JSON(elements)
+
+
+def test_partition_image_with_ocr_has_coordinates_from_filename():
+    filename = example_doc_path("img/english-and-korean.png")
+    elements = image.partition_image(filename=filename, strategy=PartitionStrategy.OCR_ONLY)
+    int_coordinates = [(int(x), int(y)) for x, y in elements[0].metadata.coordinates.points]
+    assert int_coordinates == [(14, 16), (14, 37), (381, 37), (381, 16)]
+
+
+@pytest.mark.parametrize(
+    "filename",
+    [
+        "img/layout-parser-paper-with-table.jpg",
+        "img/english-and-korean.png",
+        "img/layout-parser-paper-fast.jpg",
+    ],
+)
+def test_partition_image_with_ocr_coordinates_are_not_nan_from_filename(
+    filename,
+):
+    import math
+
+    elements = image.partition_image(
+        filename=example_doc_path(filename), strategy=PartitionStrategy.OCR_ONLY
+    )
+    for element in elements:
+        # TODO (jennings) One or multiple elements is an empty string
+        # without coordinates. This should be fixed in a new issue
+        if element.text:
+            box = element.metadata.coordinates.points
+            for point in box:
+                assert point[0] is not math.nan
+                assert point[1] is not math.nan
+
+
+def test_partition_image_formats_languages_for_tesseract():
+    filename = example_doc_path("img/jpn-vert.jpeg")
+    with mock.patch(
+        "unstructured.partition.pdf_image.ocr.process_file_with_ocr",
+    ) as mock_process_file_with_ocr:
+        image.partition_image(
+            filename=filename, strategy=PartitionStrategy.HI_RES, languages=["jpn_vert"]
+        )
+        _, kwargs = mock_process_file_with_ocr.call_args_list[0]
+        assert "ocr_languages" in kwargs
+        assert kwargs["ocr_languages"] == "jpn_vert"
+
+
+def test_partition_image_warns_with_ocr_languages(caplog):
+    filename = example_doc_path("img/layout-parser-paper-fast.jpg")
+    image.partition_image(filename=filename, strategy=PartitionStrategy.HI_RES, ocr_languages="eng")
+    assert "The ocr_languages kwarg will be deprecated" in caplog.text
+
+
+def test_add_chunking_strategy_on_partition_image():
+    filename = example_doc_path("img/layout-parser-paper-fast.jpg")
+    elements = image.partition_image(filename=filename)
+    chunk_elements = image.partition_image(filename, chunking_strategy="by_title")
+    chunks = chunk_by_title(elements)
+    assert chunk_elements != elements
+    assert chunk_elements == chunks
+
+
+def test_add_chunking_strategy_on_partition_image_hi_res():
+    filename = example_doc_path("img/layout-parser-paper-with-table.jpg")
+    elements = image.partition_image(
+        filename=filename,
+        strategy=PartitionStrategy.HI_RES,
+        infer_table_structure=True,
+    )
+    chunk_elements = image.partition_image(
+        filename,
+        strategy=PartitionStrategy.HI_RES,
+        infer_table_structure=True,
+        chunking_strategy="by_title",
+    )
+    chunks = chunk_by_title(elements)
+    assert chunk_elements != elements
+    assert chunk_elements == chunks
+
+
+def test_partition_image_uses_model_name():
+    with mock.patch.object(
+        pdf,
+        "_partition_pdf_or_image_local",
+    ) as mockpartition:
+        image.partition_image(
+            example_doc_path("img/layout-parser-paper-fast.jpg"), model_name="test"
+        )
+        print(mockpartition.call_args)
+        assert "model_name" in mockpartition.call_args.kwargs
+        assert mockpartition.call_args.kwargs["model_name"]
+
+
+def test_partition_image_uses_hi_res_model_name():
+    with mock.patch.object(
+        pdf,
+        "_partition_pdf_or_image_local",
+    ) as mockpartition:
+        image.partition_image(
+            example_doc_path("img/layout-parser-paper-fast.jpg"), hi_res_model_name="test"
+        )
+        print(mockpartition.call_args)
+        assert "model_name" not in mockpartition.call_args.kwargs
+        assert "hi_res_model_name" in mockpartition.call_args.kwargs
+        assert mockpartition.call_args.kwargs["hi_res_model_name"] == "test"
+
+
+@pytest.mark.parametrize(
+    ("ocr_mode", "idx_title_element"),
+    [
+        ("entire_page", 2),
+        ("individual_blocks", 1),
+    ],
+)
+def test_partition_image_hi_res_ocr_mode(ocr_mode, idx_title_element):
+    filename = example_doc_path("img/layout-parser-paper-fast.jpg")
+    elements = image.partition_image(
+        filename=filename, ocr_mode=ocr_mode, strategy=PartitionStrategy.HI_RES
+    )
+    # Note(yuming): idx_title_element is different based on xy-cut and ocr mode
+    assert elements[idx_title_element].category == ElementType.TITLE
+
+
+def test_partition_image_hi_res_invalid_ocr_mode():
+    filename = example_doc_path("img/layout-parser-paper-fast.jpg")
+    with pytest.raises(ValueError):
+        _ = image.partition_image(
+            filename=filename, ocr_mode="invalid_ocr_mode", strategy=PartitionStrategy.HI_RES
+        )
+
+
+@pytest.mark.parametrize(
+    "ocr_mode",
+    [
+        "entire_page",
+        "individual_blocks",
+    ],
+)
+def test_partition_image_hi_res_ocr_mode_with_table_extraction(ocr_mode):
+    filename = example_doc_path("img/layout-parser-paper-with-table.jpg")
+    elements = image.partition_image(
+        filename=filename,
+        ocr_mode=ocr_mode,
+        strategy=PartitionStrategy.HI_RES,
+        infer_table_structure=True,
+    )
+    table = [el.metadata.text_as_html for el in elements if el.metadata.text_as_html]
+    assert len(table) == 1
+    assert "<table><thead><tr>" in table[0]
+    assert "</thead><tbody><tr>" in table[0]
+    assert "Layouts of history Japanese documents" in table[0]
+    assert "Layouts of scanned modern magazines and scientific reports" in table[0]
+
+
+def test_partition_image_raises_type_error_for_invalid_languages():
+    filename = example_doc_path("img/layout-parser-paper-fast.jpg")
+    with pytest.raises(TypeError):
+        image.partition_image(filename=filename, strategy=PartitionStrategy.HI_RES, languages="eng")
+
+
+@pytest.fixture()
+def inference_results():
+    page = layout.PageLayout(
+        number=1,
+        image=mock.MagicMock(format="JPEG"),
+    )
+    page.elements = [layout.LayoutElement.from_coords(0, 0, 600, 800, text="hello")]
+    page.elements_array = layout.LayoutElements.from_list(page.elements)
+    doc = layout.DocumentLayout(pages=[page])
+    return doc
+
+
+def test_partition_image_has_filename(inference_results):
+    filename = "layout-parser-paper-fast.jpg"
+    # Mock inference call with known return results
+    with mock.patch(
+        "unstructured_inference.inference.layout.process_file_with_model",
+        return_value=inference_results,
+    ) as mock_inference_func:
+        elements = image.partition_image(
+            filename=example_doc_path(f"img/{filename}"),
+            strategy=PartitionStrategy.HI_RES,
+        )
+    # Make sure we actually went down the path we expect.
+    mock_inference_func.assert_called_once()
+    # Unpack element but also make sure there is only one
+    element = only(elements)
+    # This makes sure we are still getting the filetype metadata (should be translated from the
+    # fixtures)
+    assert element.metadata.filetype == "JPEG"
+    # This should be kept from the filename we originally gave
+    assert element.metadata.filename == filename
+
+
+@pytest.mark.parametrize("file_mode", ["filename", "rb"])
+@pytest.mark.parametrize("extract_image_block_to_payload", [False, True])
+def test_partition_image_element_extraction(
+    file_mode,
+    extract_image_block_to_payload,
+):
+    filename = example_doc_path("img/embedded-images-tables.jpg")
+    extract_image_block_types = ["Image", "Table"]
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        if file_mode == "filename":
+            elements = image.partition_image(
+                filename=filename,
+                extract_image_block_types=extract_image_block_types,
+                extract_image_block_to_payload=extract_image_block_to_payload,
+                extract_image_block_output_dir=tmpdir,
+            )
+        else:
+            with open(filename, "rb") as f:
+                elements = image.partition_image(
+                    file=f,
+                    extract_image_block_types=extract_image_block_types,
+                    extract_image_block_to_payload=extract_image_block_to_payload,
+                    extract_image_block_output_dir=tmpdir,
+                )
+
+        assert_element_extraction(
+            elements, extract_image_block_types, extract_image_block_to_payload, tmpdir
+        )
+
+
+def test_partition_image_works_on_heic_file():
+    filename = example_doc_path("img/DA-1p.heic")
+    elements = image.partition_image(filename=filename, strategy=PartitionStrategy.AUTO)
+    titles = [el.text for el in elements if el.category == ElementType.TITLE]
+    assert "CREATURES" in titles
+
+
+@pytest.mark.parametrize(
+    "strategy",
+    [PartitionStrategy.HI_RES, PartitionStrategy.OCR_ONLY],
+)
+def test_deterministic_element_ids(strategy: str):
+    elements_1 = image.partition_image(
+        example_doc_path("img/layout-parser-paper-with-table.jpg"),
+        strategy=strategy,
+        starting_page_number=2,
+    )
+    elements_2 = image.partition_image(
+        example_doc_path("img/layout-parser-paper-with-table.jpg"),
+        strategy=strategy,
+        starting_page_number=2,
+    )
+    ids_1 = [element.id for element in elements_1]
+    ids_2 = [element.id for element in elements_2]
+
+    assert ids_1 == ids_2
+
+
+def test_multi_page_tiff_starts_on_starting_page_number():
+    elements = image.partition_image(
+        example_doc_path("img/layout-parser-paper-combined.tiff"),
+        starting_page_number=2,
+    )
+    pages = {element.metadata.page_number for element in elements}
+
+    assert pages == {2, 3}

test_unstructured/partition/pdf_image/test_inference_utils.py

@@ -0,0 +1,169 @@
+from unstructured_inference.inference.elements import TextRegion, TextRegions
+from unstructured_inference.inference.layoutelement import LayoutElement, LayoutElements
+
+from unstructured.documents.elements import ElementType
+from unstructured.partition.pdf_image.inference_utils import (
+    build_layout_elements_from_ocr_regions,
+    merge_text_regions,
+)
+
+
+def test_merge_text_regions(mock_embedded_text_regions):
+    expected = TextRegion.from_coords(
+        x1=437.83888888888885,
+        y1=317.319341111111,
+        x2=1256.334784222222,
+        y2=406.9837855555556,
+        text="LayoutParser: A Unified Toolkit for Deep Learning Based Document Image",
+    )
+
+    merged_text_region = merge_text_regions(TextRegions.from_list(mock_embedded_text_regions))
+    assert merged_text_region == expected
+
+
+def test_build_layout_elements_from_ocr_regions(mock_embedded_text_regions):
+    expected = LayoutElements.from_list(
+        [
+            LayoutElement.from_coords(
+                x1=437.83888888888885,
+                y1=317.319341111111,
+                x2=1256.334784222222,
+                y2=406.9837855555556,
+                text="LayoutParser: A Unified Toolkit for Deep Learning Based Document Image",
+                type=ElementType.UNCATEGORIZED_TEXT,
+            ),
+        ]
+    )
+
+    elements = build_layout_elements_from_ocr_regions(
+        TextRegions.from_list(mock_embedded_text_regions)
+    )
+    assert elements == expected
+
+
+def test_build_layout_elements_from_ocr_regions_with_text(mock_embedded_text_regions):
+    text = "LayoutParser: A Unified Toolkit for Deep Learning Based Document Image"
+    expected = LayoutElements.from_list(
+        [
+            LayoutElement.from_coords(
+                x1=437.83888888888885,
+                y1=317.319341111111,
+                x2=1256.334784222222,
+                y2=406.9837855555556,
+                text=text,
+                type=ElementType.UNCATEGORIZED_TEXT,
+            ),
+        ]
+    )
+
+    elements = build_layout_elements_from_ocr_regions(
+        TextRegions.from_list(mock_embedded_text_regions),
+        text,
+        group_by_ocr_text=True,
+    )
+    assert elements == expected
+
+
+def test_build_layout_elements_from_ocr_regions_with_multi_line_text(mock_embedded_text_regions):
+    text = "LayoutParser: \n\nA Unified Toolkit for Deep Learning Based Document Image"
+    elements = build_layout_elements_from_ocr_regions(
+        TextRegions.from_list(mock_embedded_text_regions),
+        text,
+        group_by_ocr_text=True,
+    )
+    assert elements == LayoutElements.from_list(
+        [
+            LayoutElement.from_coords(
+                x1=453.00277777777774,
+                y1=317.319341111111,
+                x2=711.5338541666665,
+                y2=358.28571222222206,
+                text="LayoutParser:",
+                type=ElementType.UNCATEGORIZED_TEXT,
+            ),
+            LayoutElement.from_coords(
+                x1=437.83888888888885,
+                y1=317.319341111111,
+                x2=1256.334784222222,
+                y2=406.9837855555556,
+                text="A Unified Toolkit for Deep Learning Based Document Image",
+                type=ElementType.UNCATEGORIZED_TEXT,
+            ),
+        ]
+    )
+
+
+def test_build_layout_elements_from_ocr_regions_with_repeated_texts(mock_embedded_text_regions):
+    mock_embedded_text_regions.extend(
+        [
+            LayoutElement.from_coords(
+                x1=453.00277777777774,
+                y1=417.319341111111,
+                x2=711.5338541666665,
+                y2=458.28571222222206,
+                text="LayoutParser",
+                type=ElementType.UNCATEGORIZED_TEXT,
+            ),
+            LayoutElement.from_coords(
+                x1=453.00277777777774,
+                y1=468.319341111111,
+                x2=711.5338541666665,
+                y2=478.28571222222206,
+                text="for",
+                type=ElementType.UNCATEGORIZED_TEXT,
+            ),
+            LayoutElement.from_coords(
+                x1=453.00277777777774,
+                y1=488.319341111111,
+                x2=711.5338541666665,
+                y2=500.28571222222206,
+                text="Deep",
+                type=ElementType.UNCATEGORIZED_TEXT,
+            ),
+            LayoutElement.from_coords(
+                x1=453.00277777777774,
+                y1=510.319341111111,
+                x2=711.5338541666665,
+                y2=550.28571222222206,
+                text="Learning",
+                type=ElementType.UNCATEGORIZED_TEXT,
+            ),
+        ]
+    )
+    text = (
+        "LayoutParser: \n\nA Unified Toolkit for Deep Learning Based Document Image\n\n"
+        "LayoutParser for Deep Learning"
+    )
+    elements = build_layout_elements_from_ocr_regions(
+        TextRegions.from_list(mock_embedded_text_regions),
+        text,
+        group_by_ocr_text=True,
+    )
+    assert elements == LayoutElements.from_list(
+        [
+            LayoutElement.from_coords(
+                x1=453.00277777777774,
+                y1=317.319341111111,
+                x2=711.5338541666665,
+                y2=358.28571222222206,
+                text="LayoutParser:",
+                type=ElementType.UNCATEGORIZED_TEXT,
+            ),
+            LayoutElement.from_coords(
+                x1=437.83888888888885,
+                y1=317.319341111111,
+                x2=1256.334784222222,
+                y2=406.9837855555556,
+                text="A Unified Toolkit for Deep Learning Based Document Image",
+                type=ElementType.UNCATEGORIZED_TEXT,
+            ),
+            LayoutElement.from_coords(
+                x1=453.00277777777774,
+                y1=417.319341111111,
+                x2=711.5338541666665,
+                y2=550.28571222222206,
+                text="LayoutParser for Deep Learning",
+                type=ElementType.UNCATEGORIZED_TEXT,
+            ),
+        ]
+    )

test_unstructured/partition/pdf_image/test_merge_elements.py

@@ -0,0 +1,84 @@
+import numpy as np
+from PIL import Image
+from unstructured_inference.constants import IsExtracted
+from unstructured_inference.inference.elements import Rectangle
+from unstructured_inference.inference.layout import DocumentLayout, PageLayout
+from unstructured_inference.inference.layoutelement import LayoutElement, LayoutElements
+
+from unstructured.partition.pdf_image.pdfminer_processing import (
+    array_merge_inferred_layout_with_extracted_layout,
+    merge_inferred_with_extracted_layout,
+)
+
+
+def test_text_source_preserved_during_merge():
+    """Test that text_source property is preserved when elements are merged."""
+
+    # Create two simple LayoutElements with different text_source values
+    inferred_element = LayoutElement(
+        bbox=Rectangle(0, 0, 100, 50), text=None, is_extracted=IsExtracted.FALSE
+    )
+
+    extracted_element = LayoutElement(
+        bbox=Rectangle(0, 0, 100, 50), text="Extracted text", is_extracted=IsExtracted.TRUE
+    )
+
+    # Create LayoutElements arrays
+    inferred_layout_elements = LayoutElements.from_list([inferred_element])
+    extracted_layout_elements = LayoutElements.from_list([extracted_element])
+
+    # Create a PageLayout for the inferred layout
+    image = Image.new("RGB", (200, 200))
+    inferred_page = PageLayout(number=1, image=image)
+    inferred_page.elements_array = inferred_layout_elements
+
+    # Create DocumentLayout from the PageLayout
+    inferred_document_layout = DocumentLayout(pages=[inferred_page])
+
+    # Merge them
+    merged_layout = merge_inferred_with_extracted_layout(
+        inferred_document_layout=inferred_document_layout,
+        extracted_layout=[extracted_layout_elements],
+        hi_res_model_name="test_model",
+    )
+
+    # Verify text_source is preserved
+    # Check the merged page's elements_array
+    merged_page = merged_layout.pages[0]
+    assert "Extracted text" in merged_page.elements_array.texts
+    assert hasattr(merged_page.elements_array, "is_extracted_array")
+    assert IsExtracted.TRUE in merged_page.elements_array.is_extracted_array
+
+
+def test_single_extracted_region_at_index_zero_removes_inferred_subregion():
+    """A lone extracted text region (at index 0) must still absorb an inferred subregion.
+
+    Regression test: the subregion-removal rule was gated on ``any(extracted_to_keep)`` where
+    ``extracted_to_keep`` holds element *indices*. When the only kept extracted region was at
+    index 0, ``any([0])`` evaluated False and the rule was skipped, leaving the inferred box as
+    a duplicate of the extracted region. This happens with single-region pages such as a PDF
+    whose only text is a filled form field.
+    """
+    # Inferred (non-table) box fully contained by the single extracted text region.
+    inferred = LayoutElements(
+        element_coords=np.array([[10.0, 10.0, 40.0, 40.0]]),
+        texts=np.array([None], dtype=object),
+        element_class_ids=np.array([0]),
+        element_class_id_map={0: "Section-header"},
+    )
+    extracted = LayoutElements(
+        element_coords=np.array([[0.0, 0.0, 100.0, 50.0]]),
+        texts=np.array(["Extracted text"], dtype=object),
+        element_class_ids=np.array([0]),
+        element_class_id_map={0: "UncategorizedText"},
+    )
+
+    merged = array_merge_inferred_layout_with_extracted_layout(
+        inferred_layout=inferred,
+        extracted_layout=extracted,
+        page_image_size=(200, 200),
+    )
+
+    # The inferred subregion is removed; only the extracted region remains.
+    assert len(merged) == 1
+    assert list(merged.texts) == ["Extracted text"]

test_unstructured/partition/pdf_image/test_ocr.py

@@ -0,0 +1,675 @@
+from collections import namedtuple
+from typing import Optional
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+import pandas as pd
+import pytest
+import unstructured_pytesseract
+from lxml import etree
+from PIL import Image, UnidentifiedImageError
+from pypdfium2 import PdfiumError
+from unstructured_inference.inference.elements import EmbeddedTextRegion, TextRegion, TextRegions
+from unstructured_inference.inference.layout import DocumentLayout, PageLayout
+from unstructured_inference.inference.layoutelement import (
+    LayoutElement,
+    LayoutElements,
+)
+
+from test_unstructured.unit_utils import example_doc_path
+from unstructured.documents.elements import ElementType
+from unstructured.partition.pdf_image import ocr
+from unstructured.partition.pdf_image.pdf_image_utils import (
+    convert_pdf_to_images,
+    pad_element_bboxes,
+)
+from unstructured.partition.utils.config import env_config
+from unstructured.partition.utils.constants import (
+    OCR_AGENT_PADDLE,
+    OCR_AGENT_TESSERACT,
+    Source,
+)
+from unstructured.partition.utils.ocr_models.google_vision_ocr import OCRAgentGoogleVision
+from unstructured.partition.utils.ocr_models.ocr_interface import OCRAgent
+from unstructured.partition.utils.ocr_models.paddle_ocr import OCRAgentPaddle
+from unstructured.partition.utils.ocr_models.tesseract_ocr import (
+    OCRAgentTesseract,
+    zoom_image,
+)
+
+
+@pytest.mark.parametrize(
+    ("is_image", "expected_error"),
+    [
+        (True, UnidentifiedImageError),
+        (False, PdfiumError),
+    ],
+)
+def test_process_data_with_ocr_invalid_file(is_image, expected_error):
+    invalid_data = b"i am not a valid file"
+    with pytest.raises(expected_error):
+        _ = ocr.process_data_with_ocr(
+            data=invalid_data,
+            is_image=is_image,
+            out_layout=DocumentLayout(),
+            extracted_layout=[],
+        )
+
+
+@pytest.mark.parametrize("is_image", [True, False])
+def test_process_file_with_ocr_invalid_filename(is_image):
+    invalid_filename = "i am not a valid file name"
+    with pytest.raises(FileNotFoundError):
+        _ = ocr.process_file_with_ocr(
+            filename=invalid_filename,
+            is_image=is_image,
+            out_layout=DocumentLayout(),
+            extracted_layout=[],
+        )
+
+
+def test_supplement_page_layout_with_ocr_invalid_ocr():
+    with pytest.raises(ValueError):
+        _ = ocr.supplement_page_layout_with_ocr(
+            page_layout=None, image=None, ocr_agent="invliad_ocr"
+        )
+
+
+def test_get_ocr_layout_from_image_tesseract(monkeypatch):
+    monkeypatch.setattr(
+        OCRAgentTesseract,
+        "image_to_data_with_character_confidence_filter",
+        lambda *args, **kwargs: pd.DataFrame(
+            {
+                "left": [10, 20, 30, 0],
+                "top": [5, 15, 25, 0],
+                "width": [15, 25, 35, 0],
+                "height": [10, 20, 30, 0],
+                "text": ["Hello", "World", "!", ""],
+            },
+        ),
+    )
+
+    image = Image.new("RGB", (100, 100))
+
+    ocr_agent = OCRAgentTesseract()
+    ocr_layout = ocr_agent.get_layout_from_image(image)
+
+    expected_layout = TextRegions(
+        element_coords=np.array([[10.0, 5, 25, 15], [20, 15, 45, 35], [30, 25, 65, 55]]),
+        texts=np.array(["Hello", "World", "!"]),
+        sources=np.array([Source.OCR_TESSERACT] * 3),
+    )
+
+    assert ocr_layout.texts.tolist() == expected_layout.texts.tolist()
+    np.testing.assert_array_equal(ocr_layout.element_coords, expected_layout.element_coords)
+    np.testing.assert_array_equal(ocr_layout.sources, expected_layout.sources)
+
+
+def mock_ocr(*args, **kwargs):
+    return [
+        [
+            (
+                [(10, 5), (25, 5), (25, 15), (10, 15)],
+                ["Hello"],
+            ),
+        ],
+        [
+            (
+                [(20, 15), (45, 15), (45, 35), (20, 35)],
+                ["World"],
+            ),
+        ],
+        [
+            (
+                [(30, 25), (65, 25), (65, 55), (30, 55)],
+                ["!"],
+            ),
+        ],
+        [
+            (
+                [(0, 0), (0, 0), (0, 0), (0, 0)],
+                [""],
+            ),
+        ],
+    ]
+
+
+def monkeypatch_load_agent(*args):
+    class MockAgent:
+        def __init__(self):
+            self.ocr = mock_ocr
+
+    return MockAgent()
+
+
+def test_get_ocr_layout_from_image_paddle(monkeypatch):
+    monkeypatch.setattr(
+        OCRAgentPaddle,
+        "load_agent",
+        monkeypatch_load_agent,
+    )
+
+    image = Image.new("RGB", (100, 100))
+
+    ocr_layout = OCRAgentPaddle().get_layout_from_image(image)
+
+    expected_layout = TextRegions(
+        element_coords=np.array([[10.0, 5, 25, 15], [20, 15, 45, 35], [30, 25, 65, 55]]),
+        texts=np.array(["Hello", "World", "!"]),
+        sources=np.array([Source.OCR_PADDLE] * 3),
+    )
+
+    assert ocr_layout.texts.tolist() == expected_layout.texts.tolist()
+    np.testing.assert_array_equal(ocr_layout.element_coords, expected_layout.element_coords)
+    np.testing.assert_array_equal(ocr_layout.sources, expected_layout.sources)
+
+
+def test_get_ocr_text_from_image_tesseract(monkeypatch):
+    monkeypatch.setattr(
+        unstructured_pytesseract,
+        "image_to_string",
+        lambda *args, **kwargs: "Hello World",
+    )
+    image = Image.new("RGB", (100, 100))
+
+    ocr_agent = OCRAgentTesseract()
+    ocr_text = ocr_agent.get_text_from_image(image)
+
+    assert ocr_text == "Hello World"
+
+
+def test_get_ocr_text_from_image_paddle(monkeypatch):
+    monkeypatch.setattr(
+        OCRAgentPaddle,
+        "load_agent",
+        monkeypatch_load_agent,
+    )
+
+    image = Image.new("RGB", (100, 100))
+
+    ocr_agent = OCRAgentPaddle()
+    ocr_text = ocr_agent.get_text_from_image(image)
+
+    assert ocr_text == "Hello\n\nWorld\n\n!"
+
+
+@pytest.fixture()
+def google_vision_text_annotation():
+    from google.cloud.vision import (
+        Block,
+        BoundingPoly,
+        Page,
+        Paragraph,
+        Symbol,
+        TextAnnotation,
+        Vertex,
+        Word,
+    )
+
+    breaks = TextAnnotation.DetectedBreak.BreakType
+    symbols_hello = [Symbol(text=c) for c in "Hello"] + [
+        Symbol(
+            property=TextAnnotation.TextProperty(
+                detected_break=TextAnnotation.DetectedBreak(type_=breaks.SPACE)
+            )
+        )
+    ]
+    symbols_world = [Symbol(text=c) for c in "World!"] + [
+        Symbol(
+            property=TextAnnotation.TextProperty(
+                detected_break=TextAnnotation.DetectedBreak(type_=breaks.LINE_BREAK)
+            )
+        )
+    ]
+    words = [Word(symbols=symbols_hello), Word(symbols=symbols_world)]
+    bounding_box = BoundingPoly(
+        vertices=[Vertex(x=0, y=0), Vertex(x=0, y=10), Vertex(x=10, y=10), Vertex(x=10, y=0)]
+    )
+    paragraphs = [Paragraph(words=words, bounding_box=bounding_box)]
+    blocks = [Block(paragraphs=paragraphs)]
+    pages = [Page(blocks=blocks)]
+    return TextAnnotation(text="Hello World!", pages=pages)
+
+
+@pytest.fixture()
+def google_vision_client(google_vision_text_annotation):
+    Response = namedtuple("Response", "full_text_annotation")
+
+    class FakeGoogleVisionClient:
+        def document_text_detection(self, image, image_context):
+            return Response(full_text_annotation=google_vision_text_annotation)
+
+    class OCRAgentFakeGoogleVision(OCRAgentGoogleVision):
+        def __init__(self, language: Optional[str] = None):
+            self.client = FakeGoogleVisionClient()
+            self.language = language
+
+    return OCRAgentFakeGoogleVision()
+
+
+def test_get_ocr_from_image_google_vision(google_vision_client):
+    image = Image.new("RGB", (100, 100))
+
+    ocr_agent = google_vision_client
+    ocr_text = ocr_agent.get_text_from_image(image)
+
+    assert ocr_text == "Hello World!"
+
+
+def test_get_layout_from_image_google_vision(google_vision_client):
+    image = Image.new("RGB", (100, 100))
+
+    ocr_agent = google_vision_client
+    regions = ocr_agent.get_layout_from_image(image)
+    assert len(regions) == 1
+    assert regions.texts[0] == "Hello World!"
+    assert all(source == Source.OCR_GOOGLEVISION for source in regions.sources)
+    assert regions.x1[0] == 0
+    assert regions.y1[0] == 0
+    assert regions.x2[0] == 10
+    assert regions.y2[0] == 10
+
+
+def test_get_layout_elements_from_image_google_vision(google_vision_client):
+    image = Image.new("RGB", (100, 100))
+
+    ocr_agent = google_vision_client
+    layout_elements = ocr_agent.get_layout_elements_from_image(image)
+    assert len(layout_elements) == 1
+
+
+@pytest.fixture()
+def mock_ocr_regions():
+    return TextRegions.from_list(
+        [
+            EmbeddedTextRegion.from_coords(10, 10, 90, 90, text="0", source=None),
+            EmbeddedTextRegion.from_coords(200, 200, 300, 300, text="1", source=None),
+            EmbeddedTextRegion.from_coords(500, 320, 600, 350, text="3", source=None),
+        ]
+    )
+
+
+@pytest.fixture()
+def mock_out_layout(mock_embedded_text_regions):
+    return LayoutElements.from_list(
+        [
+            LayoutElement(
+                text="",
+                source=None,
+                type="Text",
+                bbox=r.bbox,
+            )
+            for r in mock_embedded_text_regions
+        ]
+    )
+
+
+def test_aggregate_ocr_text_by_block():
+    expected = "A Unified Toolkit"
+    ocr_layout = [
+        TextRegion.from_coords(0, 0, 20, 20, "A"),
+        TextRegion.from_coords(50, 50, 150, 150, "Unified"),
+        TextRegion.from_coords(150, 150, 300, 250, "Toolkit"),
+        TextRegion.from_coords(200, 250, 300, 350, "Deep"),
+    ]
+    region = TextRegion.from_coords(0, 0, 250, 350, "")
+
+    text = ocr.aggregate_ocr_text_by_block(ocr_layout, region, 0.5)
+    assert text == expected
+
+
+@pytest.mark.parametrize("zoom", [1, 0.1, 5, -1, 0])
+def test_zoom_image(zoom):
+    image = Image.new("RGB", (100, 100))
+    width, height = image.size
+    new_image = zoom_image(image, zoom)
+    new_w, new_h = new_image.size
+    if zoom <= 0:
+        zoom = 1
+    assert new_w == np.round(width * zoom, 0)
+    assert new_h == np.round(height * zoom, 0)
+
+
+@pytest.fixture()
+def mock_layout(mock_embedded_text_regions):
+    return LayoutElements.from_list(
+        [
+            LayoutElement(text=r.text, type=ElementType.UNCATEGORIZED_TEXT, bbox=r.bbox)
+            for r in mock_embedded_text_regions
+        ]
+    )
+
+
+def test_supplement_layout_with_ocr_elements(mock_layout, mock_ocr_regions):
+    ocr_elements = [
+        LayoutElement(text=r.text, source=None, type=ElementType.UNCATEGORIZED_TEXT, bbox=r.bbox)
+        for r in mock_ocr_regions.as_list()
+    ]
+
+    final_layout = ocr.supplement_layout_with_ocr_elements(mock_layout, mock_ocr_regions).as_list()
+
+    # Check if the final layout contains the original layout elements
+    for element in mock_layout.as_list():
+        assert element in final_layout
+
+    # Check if the final layout contains the OCR-derived elements
+    assert any(ocr_element in final_layout for ocr_element in ocr_elements)
+
+    # Check if the OCR-derived elements that are subregions of layout elements are removed
+    for element in mock_layout.as_list():
+        for ocr_element in ocr_elements:
+            if ocr_element.bbox.is_almost_subregion_of(
+                element.bbox,
+                env_config.OCR_LAYOUT_SUBREGION_THRESHOLD,
+            ):
+                assert ocr_element not in final_layout
+
+
+def test_merge_out_layout_with_ocr_layout(mock_out_layout, mock_ocr_regions):
+    ocr_elements = [
+        LayoutElement(text=r.text, source=None, type=ElementType.UNCATEGORIZED_TEXT, bbox=r.bbox)
+        for r in mock_ocr_regions.as_list()
+    ]
+    input_layout_elements = mock_out_layout.as_list()
+
+    final_layout = ocr.merge_out_layout_with_ocr_layout(
+        mock_out_layout,
+        mock_ocr_regions,
+    ).as_list()
+
+    # Check if the out layout's text attribute is updated with aggregated OCR text
+    assert final_layout[0].text == mock_ocr_regions.texts[2]
+
+    # Check if the final layout contains both original elements and OCR-derived elements
+    # The first element's text is modified by the ocr regions so it won't be the same as the input
+    assert all(element in final_layout for element in input_layout_elements[1:])
+    assert final_layout[0].bbox == input_layout_elements[0].bbox
+    assert any(element in final_layout for element in ocr_elements)
+
+
+@pytest.mark.parametrize(
+    ("padding", "expected_bbox"),
+    [
+        (5, (5, 15, 35, 45)),
+        (-3, (13, 23, 27, 37)),
+        (2.5, (7.5, 17.5, 32.5, 42.5)),
+        (-1.5, (11.5, 21.5, 28.5, 38.5)),
+    ],
+)
+def test_pad_element_bboxes(padding, expected_bbox):
+    element = LayoutElement.from_coords(
+        x1=10,
+        y1=20,
+        x2=30,
+        y2=40,
+        text="",
+        source=None,
+        type=ElementType.UNCATEGORIZED_TEXT,
+    )
+    expected_original_element_bbox = (10, 20, 30, 40)
+
+    padded_element = pad_element_bboxes(element, padding)
+
+    padded_element_bbox = (
+        padded_element.bbox.x1,
+        padded_element.bbox.y1,
+        padded_element.bbox.x2,
+        padded_element.bbox.y2,
+    )
+    assert padded_element_bbox == expected_bbox
+
+    # make sure the original element has not changed
+    original_element_bbox = (element.bbox.x1, element.bbox.y1, element.bbox.x2, element.bbox.y2)
+    assert original_element_bbox == expected_original_element_bbox
+
+
+@pytest.fixture()
+def table_element():
+    table = LayoutElement.from_coords(x1=10, y1=20, x2=50, y2=70, text="I am a table", type="Table")
+    return table
+
+
+@pytest.fixture()
+def mock_ocr_layout():
+    return TextRegions.from_list(
+        [
+            TextRegion.from_coords(x1=15, y1=25, x2=35, y2=45, text="Token1"),
+            TextRegion.from_coords(x1=40, y1=30, x2=45, y2=50, text="Token2"),
+        ]
+    )
+
+
+def test_supplement_element_with_table_extraction():
+    from unstructured_inference.models import tables
+
+    tables.load_agent()
+
+    image = next(convert_pdf_to_images(example_doc_path("pdf/single_table.pdf")))
+    elements = LayoutElements(
+        element_coords=np.array([[215.00109863, 731.89996338, 1470.07739258, 972.83129883]]),
+        texts=np.array(["foo"]),
+        sources=np.array(["yolox_sg"]),
+        element_class_ids=np.array([0]),
+        element_class_id_map={0: "Table"},
+    )
+    supplemented = ocr.supplement_element_with_table_extraction(
+        elements=elements,
+        image=image,
+        tables_agent=tables.tables_agent,
+        ocr_agent=ocr.OCRAgent.get_agent(language="eng"),
+    )
+    assert supplemented.text_as_html[0].startswith("<table>")
+
+
+def test_get_table_tokens(mock_ocr_layout):
+    with patch.object(OCRAgentTesseract, "get_layout_from_image", return_value=mock_ocr_layout):
+        ocr_agent = OCRAgent.get_agent(language="eng")
+        table_tokens = ocr.get_table_tokens(table_element_image=None, ocr_agent=ocr_agent)
+        expected_tokens = [
+            {
+                "bbox": [15, 25, 35, 45],
+                "text": "Token1",
+                "span_num": 0,
+                "line_num": 0,
+                "block_num": 0,
+            },
+            {
+                "bbox": [40, 30, 45, 50],
+                "text": "Token2",
+                "span_num": 1,
+                "line_num": 0,
+                "block_num": 0,
+            },
+        ]
+
+        assert table_tokens == expected_tokens
+
+
+def test_auto_zoom_not_exceed_tesseract_limit(monkeypatch):
+    monkeypatch.setenv("TESSERACT_MIN_TEXT_HEIGHT", "1000")
+    monkeypatch.setenv("TESSERACT_OPTIMUM_TEXT_HEIGHT", "100000")
+    monkeypatch.setattr(
+        OCRAgentTesseract,
+        "image_to_data_with_character_confidence_filter",
+        lambda *args, **kwargs: pd.DataFrame(
+            {
+                "left": [10, 20, 30, 0],
+                "top": [5, 15, 25, 0],
+                "width": [15, 25, 35, 0],
+                "height": [10, 20, 30, 0],
+                "text": ["Hello", "World", "!", ""],
+            },
+        ),
+    )
+
+    image = Image.new("RGB", (1000, 1000))
+    ocr_agent = OCRAgentTesseract()
+    # tests that the code can run instead of oom and OCR results make sense
+    assert ocr_agent.get_layout_from_image(image).texts.tolist() == [
+        "Hello",
+        "World",
+        "!",
+    ]
+
+
+def test_merge_out_layout_with_cid_code(mock_out_layout, mock_ocr_regions):
+    # the code should ignore this invalid text and use ocr region's text
+    mock_out_layout.texts = mock_out_layout.texts.astype(object)
+    mock_out_layout.texts[0] = "(cid:10)(cid:5)?"
+    ocr_elements = [
+        LayoutElement(text=r.text, source=None, type=ElementType.UNCATEGORIZED_TEXT, bbox=r.bbox)
+        for r in mock_ocr_regions.as_list()
+    ]
+    input_layout_elements = mock_out_layout.as_list()
+
+    # TODO (yao): refactor the tests to check the array data structure directly instead of
+    # converting them into lists first (this includes other tests in this file)
+    final_layout = ocr.merge_out_layout_with_ocr_layout(mock_out_layout, mock_ocr_regions).as_list()
+
+    # Check if the out layout's text attribute is updated with aggregated OCR text
+    assert final_layout[0].text == mock_ocr_regions.texts[2]
+
+    # Check if the final layout contains both original elements and OCR-derived elements
+    assert all(element in final_layout for element in input_layout_elements[1:])
+    assert any(element in final_layout for element in ocr_elements)
+
+
+def _create_hocr_word_span(
+    characters: list[tuple[str, str]], word_bbox: tuple[int, int, int, int], namespace_map: dict
+) -> etree.Element:
+    word_span = [
+        '<root xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">\n',
+        (
+            f"<span class='ocrx_word' title='"
+            f"bbox {word_bbox[0]} {word_bbox[1]} {word_bbox[2]} {word_bbox[3]}"
+            f"; x_wconf 64'>"
+        ),
+    ]
+    for char, x_conf in characters:
+        word_span.append(
+            f"<span class='ocrx_cinfo' title='x_bboxes 0 0 0 0; x_conf {x_conf}'>{char}</span>"
+        )
+    word_span.append("</span>")
+    word_span.append("</root>")
+    root = etree.fromstring("\n".join(word_span))
+    return root
+
+
+def test_extract_word_from_hocr():
+    characters = [
+        ("w", "99.0"),
+        ("o", "98.5"),
+        ("r", "97.5"),
+        ("d", "96.0"),
+        ("!", "50.0"),
+        ("@", "45.0"),
+    ]
+    word_bbox = (10, 9, 70, 22)
+    agent = OCRAgentTesseract()
+    word_span = _create_hocr_word_span(characters, word_bbox, agent.hocr_namespace)
+
+    text = agent.extract_word_from_hocr(word_span, 0.0)
+    assert text == "word!@"
+
+    text = agent.extract_word_from_hocr(word_span, 0.960)
+    assert text == "word"
+
+    text = agent.extract_word_from_hocr(word_span, 0.990)
+    assert text == "w"
+
+    text = agent.extract_word_from_hocr(word_span, 0.999)
+    assert text == ""
+
+
+def test_hocr_to_dataframe():
+    characters = [
+        ("w", "99.0"),
+        ("o", "98.5"),
+        ("r", "97.5"),
+        ("d", "96.0"),
+        ("!", "50.0"),
+        ("@", "45.0"),
+    ]
+    word_bbox = (10, 9, 70, 22)
+    agent = OCRAgentTesseract()
+    hocr = etree.tostring(_create_hocr_word_span(characters, word_bbox, agent.hocr_namespace))
+    df = agent.hocr_to_dataframe(hocr=hocr, character_confidence_threshold=0.960)
+
+    assert df.shape == (1, 5)
+    assert df["left"].iloc[0] == 10
+    assert df["top"].iloc[0] == 9
+    assert df["width"].iloc[0] == 60
+    assert df["height"].iloc[0] == 13
+    assert df["text"].iloc[0] == "word"
+
+
+def test_hocr_to_dataframe_when_no_prediction_empty_df():
+    df = OCRAgentTesseract().hocr_to_dataframe(hocr="")
+
+    assert df.shape == (0, 5)
+    assert "left" in df.columns
+    assert "top" in df.columns
+    assert "width" in df.columns
+    assert "height" in df.columns
+    assert "text" in df.columns
+
+
+@pytest.fixture
+def mock_page(mock_ocr_layout, mock_layout):
+    mock_page = MagicMock(PageLayout)
+    mock_page.elements_array = mock_layout
+    return mock_page
+
+
+def test_supplement_layout_with_ocr(mock_ocr_get_instance, mocker, mock_page):
+    from unstructured.partition.pdf_image.ocr import OCRAgent
+
+    mocker.patch.object(OCRAgent, "get_layout_from_image", return_value=mock_ocr_layout)
+
+    ocr.supplement_page_layout_with_ocr(
+        mock_page,
+        Image.new("RGB", (100, 100)),
+        infer_table_structure=True,
+        ocr_agent=OCR_AGENT_TESSERACT,
+        ocr_languages="eng",
+        table_ocr_agent=OCR_AGENT_PADDLE,
+    )
+
+    assert mock_ocr_get_instance.call_args_list[0][1] == {
+        "language": "eng",
+        "ocr_agent_module": OCR_AGENT_TESSERACT,
+    }
+    assert mock_ocr_get_instance.call_args_list[1][1] == {
+        "language": "en",
+        "ocr_agent_module": OCR_AGENT_PADDLE,
+    }
+
+
+def test_pass_down_agents(mock_ocr_get_instance, mocker, mock_page):
+    from unstructured.partition.pdf_image.ocr import OCRAgent, PILImage
+
+    mocker.patch.object(OCRAgent, "get_layout_from_image", return_value=mock_ocr_layout)
+    mocker.patch.object(PILImage, "open", return_value=Image.new("RGB", (100, 100)))
+    doc = MagicMock(DocumentLayout)
+    doc.pages = [mock_page]
+
+    ocr.process_file_with_ocr(
+        "foo",
+        doc,
+        [],
+        infer_table_structure=True,
+        is_image=True,
+        ocr_agent=OCR_AGENT_PADDLE,
+        ocr_languages="eng",
+        table_ocr_agent=OCR_AGENT_TESSERACT,
+    )
+
+    assert mock_ocr_get_instance.call_args_list[0][1] == {
+        "language": "en",
+        "ocr_agent_module": OCR_AGENT_PADDLE,
+    }
+    assert mock_ocr_get_instance.call_args_list[1][1] == {
+        "language": "eng",
+        "ocr_agent_module": OCR_AGENT_TESSERACT,
+    }

test_unstructured/partition/pdf_image/test_pdf_image_utils.py

@@ -0,0 +1,464 @@
+import base64
+import io
+import os
+import tempfile
+from unittest.mock import MagicMock, patch
+
+import numpy as np
+import pytest
+from PIL import Image as PILImg
+from unstructured_inference.inference import pdf_image
+
+from test_unstructured.unit_utils import example_doc_path
+from unstructured.documents.coordinates import PixelSpace
+from unstructured.documents.elements import ElementMetadata, ElementType, Image, Table
+from unstructured.errors import UnprocessableEntityError
+from unstructured.partition.pdf_image import pdf_image_utils
+
+
+@pytest.mark.parametrize("image_type", ["pil", "numpy_array"])
+def test_write_image(image_type):
+    mock_pil_image = PILImg.new("RGB", (50, 50))
+    mock_numpy_image = np.zeros((50, 50, 3), np.uint8)
+
+    image_map = {
+        "pil": mock_pil_image,
+        "numpy_array": mock_numpy_image,
+    }
+    image = image_map[image_type]
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        output_image_path = os.path.join(tmpdir, "test_image.jpg")
+        pdf_image_utils.write_image(image, output_image_path)
+        assert os.path.exists(output_image_path)
+
+        # Additional check to see if the written image can be read
+        read_image = PILImg.open(output_image_path)
+        assert read_image is not None
+
+
+@pytest.mark.parametrize("file_mode", ["filename", "rb"])
+@pytest.mark.parametrize("path_only", [True, False])
+def test_convert_pdf_to_image(file_mode, path_only):
+    filename = example_doc_path("pdf/embedded-images.pdf")
+    with tempfile.TemporaryDirectory() as tmpdir:
+        if file_mode == "filename":
+            images = pdf_image_utils.convert_pdf_to_image(
+                filename=filename,
+                file=None,
+                output_folder=tmpdir,
+                path_only=path_only,
+            )
+        else:
+            with open(filename, "rb") as f:
+                images = pdf_image_utils.convert_pdf_to_image(
+                    filename="",
+                    file=f,
+                    output_folder=tmpdir,
+                    path_only=path_only,
+                )
+
+        if path_only:
+            assert isinstance(images[0], str)
+        else:
+            assert isinstance(images[0], PILImg.Image)
+
+
+def test_convert_pdf_to_image_raises_unprocessable_when_render_too_large():
+    with patch.object(
+        pdf_image_utils,
+        "render_pdf_to_image",
+        side_effect=pdf_image.PdfRenderTooLargeError("too many pixels"),
+    ):
+        with pytest.raises(UnprocessableEntityError, match="too many pixels"):
+            pdf_image_utils.convert_pdf_to_image(filename="example.pdf")
+
+
+def test_convert_pdf_to_images_raises_unprocessable_when_render_too_large():
+    with (
+        patch.object(pdf_image_utils.pdf2image, "pdfinfo_from_path", return_value={"Pages": 1}),
+        patch.object(
+            pdf_image_utils,
+            "render_pdf_to_image",
+            side_effect=pdf_image.PdfRenderTooLargeError("too many pixels"),
+        ),
+    ):
+        with pytest.raises(UnprocessableEntityError, match="too many pixels"):
+            list(pdf_image_utils.convert_pdf_to_images(filename="example.pdf"))
+
+
+@pytest.mark.parametrize("file_mode", ["filename", "rb"])
+@pytest.mark.parametrize("path_only", [True, False])
+def test_convert_pdf_to_image_twice(file_mode, path_only):
+    filename = example_doc_path("pdf/embedded-images.pdf")
+    with tempfile.TemporaryDirectory() as tmpdir:
+        if file_mode == "filename":
+            images = pdf_image_utils.convert_pdf_to_image(
+                filename=filename,
+                file=None,
+                output_folder=tmpdir,
+                path_only=path_only,
+            )
+            images = pdf_image_utils.convert_pdf_to_image(
+                filename=filename,
+                file=None,
+                output_folder=tmpdir,
+                path_only=path_only,
+            )
+        else:
+            with open(filename, "rb") as f:
+                images = pdf_image_utils.convert_pdf_to_image(
+                    filename="",
+                    file=f,
+                    output_folder=tmpdir,
+                    path_only=path_only,
+                )
+                images = pdf_image_utils.convert_pdf_to_image(
+                    filename="",
+                    file=f,
+                    output_folder=tmpdir,
+                    path_only=path_only,
+                )
+
+        if path_only:
+            assert isinstance(images[0], str)
+        else:
+            assert isinstance(images[0], PILImg.Image)
+
+
+def test_convert_pdf_to_image_raises_error():
+    filename = example_doc_path("embedded-images.pdf")
+    with pytest.raises(ValueError) as exc_info:
+        pdf_image_utils.convert_pdf_to_image(filename=filename, path_only=True, output_folder=None)
+
+    assert str(exc_info.value) == "output_folder must be specified if path_only is true"
+
+
+def test_convert_pdf_to_image_rejects_both_filename_and_file():
+    filename = example_doc_path("pdf/embedded-images.pdf")
+    with tempfile.TemporaryDirectory() as tmpdir:
+        with open(filename, "rb") as f:
+            with pytest.raises(ValueError) as exc_info:
+                pdf_image_utils.convert_pdf_to_image(
+                    filename=filename,
+                    file=f,
+                    output_folder=tmpdir,
+                    path_only=True,
+                )
+    assert "Exactly one of" in str(exc_info.value)
+
+
+@pytest.mark.parametrize(
+    ("filename", "is_image"),
+    [
+        (example_doc_path("pdf/layout-parser-paper-fast.pdf"), False),
+        (example_doc_path("img/layout-parser-paper-fast.jpg"), True),
+        (example_doc_path("img/english-and-korean.png"), True),
+    ],
+)
+@pytest.mark.parametrize("element_category_to_save", [ElementType.IMAGE, ElementType.TABLE])
+@pytest.mark.parametrize("extract_image_block_to_payload", [False, True])
+@pytest.mark.parametrize("horizontal_padding", [0, 20])
+@pytest.mark.parametrize("vertical_padding", [0, 10])
+def test_save_elements(
+    element_category_to_save,
+    extract_image_block_to_payload,
+    filename,
+    is_image,
+    horizontal_padding,
+    vertical_padding,
+    monkeypatch,
+):
+    if horizontal_padding > 0:
+        monkeypatch.setenv("EXTRACT_IMAGE_BLOCK_CROP_HORIZONTAL_PAD", str(horizontal_padding))
+    if vertical_padding > 0:
+        monkeypatch.setenv("EXTRACT_IMAGE_BLOCK_CROP_VERTICAL_PAD", str(vertical_padding))
+    with tempfile.TemporaryDirectory() as tmpdir:
+        elements = [
+            Image(
+                text="Image Text 1",
+                coordinates=((78, 86), (78, 519), (512, 519), (512, 86)),
+                coordinate_system=PixelSpace(width=1575, height=1166),
+                metadata=ElementMetadata(page_number=1),
+            ),
+            Image(
+                text="Image Text 2",
+                coordinates=((570, 86), (570, 519), (1003, 519), (1003, 86)),
+                coordinate_system=PixelSpace(width=1575, height=1166),
+                metadata=ElementMetadata(page_number=1),
+            ),
+            Image(
+                text="Table 1",
+                coordinates=((1062, 86), (1062, 519), (1496, 519), (1496, 86)),
+                coordinate_system=PixelSpace(width=1575, height=1166),
+                metadata=ElementMetadata(page_number=1),
+            ),
+        ]
+        if not is_image:
+            # add a page 2 element
+            elements.append(
+                Table(
+                    text="Table 2",
+                    coordinates=((1062, 86), (1062, 519), (1496, 519), (1496, 86)),
+                    coordinate_system=PixelSpace(width=1575, height=1166),
+                    metadata=ElementMetadata(page_number=2),
+                ),
+            )
+
+        pdf_image_utils.save_elements(
+            elements=elements,
+            starting_page_number=1,
+            element_category_to_save=element_category_to_save,
+            pdf_image_dpi=200,
+            filename=filename,
+            is_image=is_image,
+            output_dir_path=str(tmpdir),
+            extract_image_block_to_payload=extract_image_block_to_payload,
+        )
+
+        saved_elements = [el for el in elements if el.category == element_category_to_save]
+        for i, el in enumerate(saved_elements):
+            basename = "table" if el.category == ElementType.TABLE else "figure"
+            expected_image_path = os.path.join(
+                str(tmpdir), f"{basename}-{el.metadata.page_number}-{i + 1}.jpg"
+            )
+            if extract_image_block_to_payload:
+                assert isinstance(el.metadata.image_base64, str)
+                assert isinstance(el.metadata.image_mime_type, str)
+                image_bytes = base64.b64decode(el.metadata.image_base64)
+                image = PILImg.open(io.BytesIO(image_bytes))
+                x1, y1 = el.metadata.coordinates.points[0]
+                x2, y2 = el.metadata.coordinates.points[2]
+                width = x2 - x1
+                height = y2 - y1
+                assert image.width == width + 2 * horizontal_padding
+                assert image.height == height + 2 * vertical_padding
+                assert not el.metadata.image_path
+                assert not os.path.isfile(expected_image_path)
+            else:
+                assert os.path.isfile(expected_image_path)
+                image = PILImg.open(expected_image_path)
+                x1, y1 = el.metadata.coordinates.points[0]
+                x2, y2 = el.metadata.coordinates.points[2]
+                width = x2 - x1
+                height = y2 - y1
+                assert image.width == width + 2 * horizontal_padding
+                assert image.height == height + 2 * vertical_padding
+                assert el.metadata.image_path == expected_image_path
+                assert not el.metadata.image_base64
+                assert not el.metadata.image_mime_type
+
+
+@pytest.mark.parametrize("storage_enabled", [False, True])
+def test_save_elements_with_output_dir_path_none(monkeypatch, storage_enabled):
+    monkeypatch.setenv(
+        "GLOBAL_WORKING_DIR_ENABLED",
+        "true" if storage_enabled else "false",
+    )
+    with (
+        patch("PIL.Image.open"),
+        patch("unstructured.partition.pdf_image.pdf_image_utils.write_image"),
+        patch("unstructured.partition.pdf_image.pdf_image_utils.convert_pdf_to_image"),
+        tempfile.TemporaryDirectory() as tmpdir,
+    ):
+        original_cwd = os.getcwd()
+        os.chdir(tmpdir)
+        pdf_image_utils.save_elements(
+            elements=[],
+            element_category_to_save="",
+            starting_page_number=1,
+            pdf_image_dpi=200,
+            filename="dummy.pdf",
+            output_dir_path=None,
+        )
+
+        # Verify that the images are saved in the expected directory
+        if storage_enabled:
+            from unstructured.partition.utils.config import env_config
+
+            expected_output_dir = os.path.join(env_config.GLOBAL_WORKING_PROCESS_DIR, "figures")
+        else:
+            expected_output_dir = os.path.join(tmpdir, "figures")
+        assert os.path.exists(expected_output_dir)
+        assert os.path.isdir(expected_output_dir)
+        os.chdir(original_cwd)
+
+
+def test_write_image_raises_error():
+    with pytest.raises(ValueError):
+        pdf_image_utils.write_image("invalid_type", "test_image.jpg")
+
+
+@pytest.mark.parametrize(
+    ("text", "outcome"), [("", False), ("foo", True), (None, False), ("(cid:10)boo", False)]
+)
+def test_valid_text(text, outcome):
+    assert pdf_image_utils.valid_text(text) == outcome
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("base", 0.0),
+        ("", 0.0),
+        ("(cid:2)", 1.0),
+        ("(cid:1)a", 0.5),
+        ("c(cid:1)ab", 0.25),
+    ],
+)
+def test_cid_ratio(text, expected):
+    assert pdf_image_utils.cid_ratio(text) == expected
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [
+        ("base", False),
+        ("(cid:2)", True),
+        ("(cid:1234567890)", True),
+        ("jkl;(cid:12)asdf", True),
+    ],
+)
+def test_is_cid_present(text, expected):
+    assert pdf_image_utils.is_cid_present(text) == expected
+
+
+def test_pad_bbox():
+    bbox = (100, 100, 200, 200)
+    padding = (10, 20)  # Horizontal padding 10, Vertical padding 20
+    expected = (90, 80, 210, 220)
+
+    result = pdf_image_utils.pad_bbox(bbox, padding)
+    assert result == expected
+
+
+@pytest.mark.parametrize(
+    ("input_types", "expected"),
+    [
+        (None, []),
+        (["table", "image"], ["Table", "Image"]),
+        (["unknown"], ["Unknown"]),
+        (["Table", "image", "UnknOwn"], ["Table", "Image", "Unknown"]),
+        (["NarrativeText", "narrativetext"], ["NarrativeText", "NarrativeText"]),
+    ],
+)
+def test_check_element_types_to_extract(input_types, expected):
+    assert pdf_image_utils.check_element_types_to_extract(input_types) == expected
+
+
+def test_check_element_types_to_extract_raises_error():
+    with pytest.raises(TypeError) as exc_info:
+        pdf_image_utils.check_element_types_to_extract("not a list")
+    assert "must be a list" in str(exc_info.value)
+
+
+class MockPageLayout:
+    def annotate(self, colors):
+        return "mock_image"
+
+
+class MockDocumentLayout:
+    pages = [MockPageLayout(), MockPageLayout]
+
+
+def test_annotate_layout_elements_with_image():
+    inferred_layout = MockPageLayout()
+    extracted_layout = MockPageLayout()
+    output_basename = "test_page"
+    page_number = 1
+
+    # Check if images for both layouts were saved
+    with (
+        tempfile.TemporaryDirectory() as tmpdir,
+        patch("unstructured.partition.pdf_image.pdf_image_utils.write_image") as mock_write_image,
+    ):
+        pdf_image_utils.annotate_layout_elements_with_image(
+            inferred_page_layout=inferred_layout,
+            extracted_page_layout=extracted_layout,
+            output_dir_path=str(tmpdir),
+            output_f_basename=output_basename,
+            page_number=page_number,
+        )
+
+        expected_filenames = [
+            f"{output_basename}_{page_number}_inferred.jpg",
+            f"{output_basename}_{page_number}_extracted.jpg",
+        ]
+        actual_calls = [call.args[1] for call in mock_write_image.call_args_list]
+        for expected_filename in expected_filenames:
+            assert any(expected_filename in actual_call for actual_call in actual_calls)
+
+    # Check if only the inferred layout image was saved if extracted layout is None
+    with (
+        tempfile.TemporaryDirectory() as tmpdir,
+        patch("unstructured.partition.pdf_image.pdf_image_utils.write_image") as mock_write_image,
+    ):
+        pdf_image_utils.annotate_layout_elements_with_image(
+            inferred_page_layout=inferred_layout,
+            extracted_page_layout=None,
+            output_dir_path=str(tmpdir),
+            output_f_basename=output_basename,
+            page_number=page_number,
+        )
+
+        expected_filename = f"{output_basename}_{page_number}_inferred.jpg"
+        actual_calls = [call.args[1] for call in mock_write_image.call_args_list]
+        assert any(expected_filename in actual_call for actual_call in actual_calls)
+        assert len(actual_calls) == 1  # Only one image should be saved
+
+
+@pytest.mark.parametrize(
+    ("filename", "is_image"),
+    [
+        (example_doc_path("pdf/layout-parser-paper-fast.pdf"), False),
+        (example_doc_path("img/layout-parser-paper-fast.jpg"), True),
+    ],
+)
+def test_annotate_layout_elements(filename, is_image):
+    inferred_document_layout = MockDocumentLayout
+    extracted_layout = [MagicMock(), MagicMock()]
+
+    with (
+        patch("PIL.Image.open"),
+        patch(
+            "unstructured.partition.pdf_image.pdf_image_utils.convert_pdf_to_image",
+            return_value=["/path/to/image1.jpg", "/path/to/image2.jpg"],
+        ) as mock_pdf2image,
+        patch(
+            "unstructured.partition.pdf_image.pdf_image_utils.annotate_layout_elements_with_image"
+        ) as mock_annotate_layout_elements_with_image,
+    ):
+        pdf_image_utils.annotate_layout_elements(
+            inferred_document_layout=inferred_document_layout,
+            extracted_layout=extracted_layout,
+            filename=filename,
+            output_dir_path="/output",
+            pdf_image_dpi=200,
+            is_image=is_image,
+        )
+        if is_image:
+            mock_annotate_layout_elements_with_image.assert_called_once()
+        else:
+            assert mock_annotate_layout_elements_with_image.call_count == len(
+                mock_pdf2image.return_value
+            )
+
+
+def test_annotate_layout_elements_file_not_found_error():
+    with pytest.raises(FileNotFoundError):
+        pdf_image_utils.annotate_layout_elements(
+            inferred_document_layout=MagicMock(),
+            extracted_layout=[],
+            filename="nonexistent.jpg",
+            output_dir_path="/output",
+            pdf_image_dpi=200,
+            is_image=True,
+        )
+
+
+@pytest.mark.parametrize(
+    ("text", "expected"),
+    [("test\tco\x0cn\ftrol\ncharacter\rs\b", "test control characters"), ("\"'\\", "\"'\\")],
+)
+def test_remove_control_characters(text, expected):
+    assert pdf_image_utils.remove_control_characters(text) == expected

test_unstructured/partition/pdf_image/test_pdfminer_processing.py

@@ -0,0 +1,682 @@
+import os
+import tempfile
+from unittest.mock import Mock, patch
+
+import numpy as np
+import pytest
+from pdfminer.layout import LAParams, LTChar, LTContainer
+from PIL import Image
+from unstructured_inference.constants import IsExtracted
+from unstructured_inference.constants import Source as InferenceSource
+from unstructured_inference.inference.elements import (
+    EmbeddedTextRegion,
+    Rectangle,
+    TextRegion,
+    TextRegions,
+)
+from unstructured_inference.inference.layout import DocumentLayout, LayoutElement, PageLayout
+from unstructured_inference.inference.layoutelement import LayoutElements
+
+from test_unstructured.unit_utils import example_doc_path
+from unstructured.partition.auto import partition
+from unstructured.partition.pdf_image.pdfminer_processing import (
+    _deduplicate_ltchars,
+    _rotate_bboxes,
+    _validate_bbox,
+    aggregate_embedded_text_by_block,
+    bboxes1_is_almost_subregion_of_bboxes2,
+    boxes_self_iou,
+    clean_pdfminer_inner_elements,
+    get_widget_text_from_annots,
+    process_file_with_pdfminer,
+    remove_duplicate_elements,
+    text_is_embedded,
+)
+from unstructured.partition.utils.constants import Source
+
+# A set of elements with pdfminer elements inside tables
+deletable_elements_inside_table = [
+    LayoutElement(
+        bbox=Rectangle(0, 0, 100, 100),
+        text="Table with inner elements",
+        type="Table",
+    ),
+    LayoutElement(bbox=Rectangle(50, 50, 70, 70), text="text1", source=Source.PDFMINER),
+    LayoutElement(bbox=Rectangle(70, 70, 80, 80), text="text2", source=Source.PDFMINER),
+]
+
+# A set of elements without pdfminer elements inside
+# tables (no elements with source=Source.PDFMINER)
+no_deletable_elements_inside_table = [
+    LayoutElement(
+        bbox=Rectangle(0, 0, 100, 100),
+        text="Table with inner elements",
+        type="Table",
+        source=InferenceSource.YOLOX,
+    ),
+    LayoutElement(bbox=Rectangle(50, 50, 70, 70), text="text1", source=InferenceSource.YOLOX),
+    LayoutElement(bbox=Rectangle(70, 70, 80, 80), text="text2", source=InferenceSource.YOLOX),
+]
+# A set of elements with pdfminer elements inside tables and other
+# elements with source=Source.PDFMINER
+# Note: there is some elements with source=Source.PDFMINER are not inside tables
+mix_elements_inside_table = [
+    LayoutElement(
+        bbox=Rectangle(0, 0, 100, 100),
+        text="Table1 with inner elements",
+        type="Table",
+        source=InferenceSource.YOLOX,
+    ),
+    LayoutElement(bbox=Rectangle(50, 50, 70, 70), text="Inside table1"),
+    LayoutElement(bbox=Rectangle(70, 70, 80, 80), text="Inside table1", source=Source.PDFMINER),
+    LayoutElement(
+        bbox=Rectangle(150, 150, 170, 170),
+        text="Outside tables",
+        source=Source.PDFMINER,
+    ),
+    LayoutElement(
+        bbox=Rectangle(180, 180, 200, 200),
+        text="Outside tables",
+        source=Source.PDFMINER,
+    ),
+    LayoutElement(
+        bbox=Rectangle(0, 500, 100, 700),
+        text="Table2 with inner elements",
+        type="Table",
+        source=InferenceSource.YOLOX,
+    ),
+    LayoutElement(bbox=Rectangle(0, 510, 50, 600), text="Inside table2", source=Source.PDFMINER),
+    LayoutElement(bbox=Rectangle(0, 550, 70, 650), text="Inside table2", source=Source.PDFMINER),
+]
+
+
+def test_rotate_bboxes_matches_pil_rotation_directions():
+    """_rotate_bboxes mirrors PIL.Image.rotate(angle, expand=True) (counter-clockwise)."""
+    W, H = 100.0, 200.0  # portrait display-frame canvas
+    coords = np.array([[10.0, 20.0, 30.0, 60.0]])
+
+    # 0 / 360 are no-ops
+    assert np.array_equal(_rotate_bboxes(coords, 0, W, H), coords)
+    assert np.array_equal(_rotate_bboxes(coords, 360, W, H), coords)
+
+    # 90 CCW (expand): x' = y, y' = W - x
+    r90 = _rotate_bboxes(coords, 90, W, H)
+    assert np.allclose(r90, [[20.0, W - 30.0, 60.0, W - 10.0]])
+    # 180
+    r180 = _rotate_bboxes(coords, 180, W, H)
+    assert np.allclose(r180, [[W - 30.0, H - 60.0, W - 10.0, H - 20.0]])
+    # 270 CCW
+    assert np.allclose(_rotate_bboxes(coords, 270, W, H), [[H - 60.0, 10.0, H - 20.0, 30.0]])
+
+    # rotating 90 then 270 (about the post-rotation H x W canvas) restores the original box
+    assert np.allclose(_rotate_bboxes(r90, 270, H, W), coords)
+
+    # outputs remain valid bboxes (x1 < x2, y1 < y2)
+    for angle in (90, 180, 270):
+        r = _rotate_bboxes(coords, angle, W, H)
+        assert r[0, 0] < r[0, 2]
+        assert r[0, 1] < r[0, 3]
+
+
+@pytest.mark.parametrize(
+    ("bbox", "is_valid"),
+    [
+        ([0, 1, 0, 1], False),
+        ([0, 1, 1, 2], True),
+        ([0, 1, 1, None], False),
+        ([0, 1, 1, np.nan], False),
+        ([0, 1, -1, 0], False),
+        ([0, 1, -1, 2], False),
+    ],
+)
+def test_valid_bbox(bbox, is_valid):
+    assert _validate_bbox(bbox) is is_valid
+
+
+@pytest.mark.parametrize(
+    ("elements", "length_extra_info", "expected_document_length"),
+    [
+        (deletable_elements_inside_table, 1, 1),
+        (no_deletable_elements_inside_table, 0, 3),
+        (mix_elements_inside_table, 2, 5),
+    ],
+)
+def test_clean_pdfminer_inner_elements(elements, length_extra_info, expected_document_length):
+    # create a sample document with pdfminer elements inside tables
+    page = PageLayout(number=1, image=Image.new("1", (1, 1)))
+    page.elements_array = LayoutElements.from_list(elements)
+    document_with_table = DocumentLayout(pages=[page])
+    document = document_with_table
+
+    # call the function to clean the pdfminer inner elements
+    cleaned_doc = clean_pdfminer_inner_elements(document)
+
+    # check that the pdfminer elements were stored in the extra_info dictionary
+    assert len(cleaned_doc.pages[0].elements_array) == expected_document_length
+
+
+elements_with_duplicate_images = [
+    LayoutElement(
+        bbox=Rectangle(0, 0, 100, 100),
+        text="Image1",
+        type="Image",
+        source=Source.PDFMINER,
+    ),
+    LayoutElement(
+        bbox=Rectangle(10, 10, 110, 110), text="Image1", type="Image", source=Source.PDFMINER
+    ),
+    LayoutElement(bbox=Rectangle(150, 150, 170, 170), text="Title1", type="Title"),
+]
+
+elements_without_duplicate_images = [
+    LayoutElement(
+        bbox=Rectangle(0, 0, 100, 100),
+        text="Sample image",
+        type="Image",
+        source=Source.PDFMINER,
+    ),
+    LayoutElement(
+        bbox=Rectangle(10, 10, 110, 110),
+        text="Sample image with similar bbox",
+        type="Image",
+        source=Source.PDFMINER,
+    ),
+    LayoutElement(
+        bbox=Rectangle(200, 200, 250, 250),
+        text="Sample image",
+        type="Image",
+        source=Source.PDFMINER,
+    ),
+    LayoutElement(bbox=Rectangle(150, 150, 170, 170), text="Title1", type="Title"),
+]
+
+
+def test_aggregate_by_block():
+    expected = "Inside region1 Inside region2"
+    embedded_regions = TextRegions.from_list(
+        [
+            TextRegion.from_coords(0, 0, 300, 20, "Inside region1"),
+            TextRegion.from_coords(0, 20, 300, 80, None),
+            TextRegion.from_coords(0, 80, 200, 300, "Inside region2"),
+            TextRegion.from_coords(250, 250, 350, 350, "Outside region"),
+        ]
+    )
+    embedded_regions.is_extracted_array = np.array([IsExtracted.TRUE] * 4)
+    target_region = TextRegions.from_list([TextRegion.from_coords(0, 0, 300, 300)])
+
+    text, extracted = aggregate_embedded_text_by_block(target_region, embedded_regions)
+    assert text == expected
+    assert extracted.value == "true"
+
+
+def test_aggregate_only_partially_fill_target():
+    expected = "Inside region1"
+    embedded_regions = TextRegions.from_list(
+        [
+            TextRegion.from_coords(0, 0, 20, 20, "Inside region1"),
+        ]
+    )
+    embedded_regions.is_extracted_array = np.array([IsExtracted.TRUE])
+    target_region = TextRegions.from_list([TextRegion.from_coords(0, 0, 300, 300)])
+
+    text, extracted = aggregate_embedded_text_by_block(target_region, embedded_regions)
+    assert text == expected
+    assert extracted.value == "partial"
+
+
+def test_aggregate_not_filling_target():
+    embedded_regions = TextRegions.from_list(
+        [
+            TextRegion.from_coords(300, 0, 400, 20, "outside"),
+        ]
+    )
+    embedded_regions.is_extracted_array = np.array([IsExtracted.TRUE])
+    target_region = TextRegions.from_list([TextRegion.from_coords(0, 0, 300, 300)])
+
+    text, extracted = aggregate_embedded_text_by_block(target_region, embedded_regions)
+    assert text == ""
+    assert extracted.value == "false"
+
+
+@pytest.mark.parametrize(
+    ("coords1", "coords2", "expected"),
+    [
+        (
+            [[0, 0, 10, 10], [10, 0, 20, 10], [10, 10, 20, 20]],
+            [[0, 0, 10, 10], [0, 0, 12, 12]],
+            [[True, True], [False, False], [False, False]],
+        ),
+        (
+            [[0, 0, 10, 10], [10, 0, 20, 10], [10, 10, 20, 20]],
+            [[0, 0, 10, 10], [10, 10, 22, 22], [0, 0, 5, 5]],
+            [[True, False, False], [False, False, False], [False, True, False]],
+        ),
+        (
+            [[0, 0, 10, 10], [10, 10, 10, 10]],
+            [[0, 0, 10, 10], [10, 10, 22, 22], [0, 0, 5, 5]],
+            [[True, False, False], [True, True, False]],
+        ),
+    ],
+)
+def test_bboxes1_is_almost_subregion_of_bboxes2(coords1, coords2, expected):
+    bboxes1 = [Rectangle(*row) for row in coords1]
+    bboxes2 = [Rectangle(*row) for row in coords2]
+    np.testing.assert_array_equal(
+        bboxes1_is_almost_subregion_of_bboxes2(bboxes1, bboxes2), expected
+    )
+
+
+@pytest.mark.parametrize(
+    ("coords", "threshold", "expected"),
+    [
+        (
+            [[0, 0, 10, 10], [2, 2, 12, 12], [10, 10, 20, 20]],
+            0.5,
+            [[True, True, False], [True, True, False], [False, False, True]],
+        ),
+        (
+            [[0, 0, 10, 10], [2, 2, 12, 12], [10, 10, 20, 20]],
+            0.9,
+            [[True, False, False], [False, True, False], [False, False, True]],
+        ),
+        (
+            [[0, 0, 10, 10], [10, 10, 10, 10]],
+            0.5,
+            [[True, False], [False, True]],
+        ),
+    ],
+)
+def test_boxes_self_iou(coords, threshold, expected):
+    bboxes = [Rectangle(*row) for row in coords]
+    np.testing.assert_array_equal(boxes_self_iou(bboxes, threshold), expected)
+
+
+def test_remove_duplicate_elements():
+    sample_elements = TextRegions.from_list(
+        [
+            EmbeddedTextRegion(bbox=Rectangle(0, 0, 10, 10), text="Text 1"),
+            EmbeddedTextRegion(bbox=Rectangle(0, 0, 10, 10), text="Text 2"),
+            EmbeddedTextRegion(bbox=Rectangle(20, 20, 30, 30), text="Text 3"),
+        ]
+    )
+
+    result = remove_duplicate_elements(sample_elements)
+
+    # Check that duplicates were removed and only 2 unique elements remain
+    assert len(result) == 2
+    assert result.texts.tolist() == ["Text 2", "Text 3"]
+    assert result.element_coords.tolist() == [[0, 0, 10, 10], [20, 20, 30, 30]]
+
+
+def test_remove_duplicate_elements_dense_page_is_not_decimated():
+    """Pages with more than ~2000 elements are chunked internally; the dedup mask for each
+    chunk must be offset by the chunk's global index. Otherwise rows in later chunks match
+    themselves and are wrongly dropped, decimating dense pages."""
+    # 2500 unique, non-overlapping boxes on a 50x50 grid (zero IoU between any two)
+    unique = [
+        EmbeddedTextRegion(
+            bbox=Rectangle((i % 50) * 20, (i // 50) * 20, (i % 50) * 20 + 10, (i // 50) * 20 + 10),
+            text=f"Text {i}",
+        )
+        for i in range(2500)
+    ]
+    # one exact duplicate of the first box, appended last so the pair spans two chunks
+    duplicate = EmbeddedTextRegion(bbox=Rectangle(0, 0, 10, 10), text="Text 0 dup")
+    sample_elements = TextRegions.from_list([*unique, duplicate])
+
+    result = remove_duplicate_elements(sample_elements)
+
+    # only the single cross-chunk duplicate pair collapses; every unique box is kept
+    assert len(result) == 2500
+    # the later element of the duplicate pair is the one retained
+    assert "Text 0 dup" in result.texts.tolist()
+    assert "Text 0" not in result.texts.tolist()
+
+
+def test_process_file_with_pdfminer():
+    layout, links = process_file_with_pdfminer(example_doc_path("pdf/layout-parser-paper-fast.pdf"))
+    assert len(layout)
+    assert "LayoutParser: A Unified Toolkit for Deep\n" in layout[0].texts
+    assert links[0][0]["url"] == "https://layout-parser.github.io"
+
+
+def test_process_file_with_pdfminer_is_extracted_array():
+    layout, _ = process_file_with_pdfminer(example_doc_path("pdf/layout-parser-paper-fast.pdf"))
+    # first page contains rotated text that are considered low fidelity, i.e., is_extracted=partial
+    assert layout[0].is_extracted_array[0] is None
+    assert all(is_extracted is IsExtracted.TRUE for is_extracted in layout[1].is_extracted_array)
+
+
+def test_process_file_hidden_ocr_text():
+    """Test processing a PDF that contains hidden OCR text layer."""
+    layout, _ = process_file_with_pdfminer(example_doc_path("pdf/pdf-with-ocr-text.pdf"))
+    assert all(is_extracted is None for is_extracted in layout[0].is_extracted_array[:-1])
+    assert layout[0].is_extracted_array[-1] == IsExtracted.TRUE
+
+
+def test_process_file_recovers_figure_overlay_text():
+    """Text inside a Form XObject (LTFigure overlay) is recovered, not dropped.
+
+    Regression test: such text is real, embedded text held as loose LTChars that pdfminer does not
+    group into LTTextLine objects, so extract_text_objects (LTTextLine only) used to drop it. The
+    fixture has "Printed Name:" in the main content stream and "Jane Doe" inside a form XObject.
+    """
+    layout, _ = process_file_with_pdfminer(example_doc_path("pdf/figure-overlay-text.pdf"))
+    texts = " ".join(str(t) for page in layout for t in page.texts if t)
+    assert "Printed Name:" in texts  # main content stream
+    assert "Jane Doe" in texts  # figure-overlay text (dropped before the fix)
+
+
+# A synthetic AcroForm: filled text fields whose values live only in widget annotations
+# (the page content stream is empty), plus one empty field that must be skipped.
+SYNTHETIC_FORM_FIELDS = [
+    ("name", "Jane Doe", (40, 700, 300, 720)),
+    ("date of birth", "1990-01-01", (40, 650, 300, 670)),
+    ("address", "123 Main Street", (40, 600, 300, 620)),
+    ("phone", "", (40, 550, 300, 570)),  # empty -> should be skipped
+]
+
+
+def _build_synthetic_form_pdf(path: str) -> None:
+    """Write a 1-page PDF whose only text lives in AcroForm text-field (/Tx) widgets.
+
+    The page content stream is empty, so pdfminer's normal text pass yields nothing; the
+    values are reachable only through the widget annotations in ``page.annots``.
+    """
+    from pypdf import PdfWriter
+    from pypdf.generic import (
+        ArrayObject,
+        DictionaryObject,
+        NameObject,
+        NumberObject,
+        TextStringObject,
+    )
+
+    writer = PdfWriter()
+    writer.add_blank_page(width=612, height=792)
+    page = writer.pages[0]
+
+    refs = []
+    for field_name, value, rect in SYNTHETIC_FORM_FIELDS:
+        widget = DictionaryObject()
+        widget[NameObject("/Type")] = NameObject("/Annot")
+        widget[NameObject("/Subtype")] = NameObject("/Widget")
+        widget[NameObject("/FT")] = NameObject("/Tx")
+        widget[NameObject("/T")] = TextStringObject(field_name)
+        widget[NameObject("/V")] = TextStringObject(value)
+        widget[NameObject("/Rect")] = ArrayObject([NumberObject(c) for c in rect])
+        refs.append(writer._add_object(widget))
+
+    page[NameObject("/Annots")] = ArrayObject(refs)
+    acro_form = DictionaryObject()
+    acro_form[NameObject("/Fields")] = ArrayObject(refs)
+    writer._root_object[NameObject("/AcroForm")] = writer._add_object(acro_form)
+
+    with open(path, "wb") as f:
+        writer.write(f)
+
+
+def test_get_widget_text_from_annots_extracts_filled_text_fields():
+    """The widget helper recovers filled /Tx field values and skips empty ones."""
+    from pdfminer.pdfpage import PDFPage
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        pdf_path = os.path.join(tmp_dir, "form.pdf")
+        _build_synthetic_form_pdf(pdf_path)
+        with open(pdf_path, "rb") as fp:
+            page = next(PDFPage.get_pages(fp))
+            widgets = get_widget_text_from_annots(page.annots, height=792)
+
+    texts = [w["text"] for w in widgets]
+    assert texts == ["Jane Doe", "1990-01-01", "123 Main Street"]  # empty "phone" skipped
+    # Every widget carries a valid bounding box.
+    assert all(_validate_bbox(w["bbox"]) for w in widgets)
+
+
+def test_get_widget_text_from_annots_decodes_utf16_text_without_bom():
+    from pdfminer.psparser import PSLiteral
+
+    widgets = get_widget_text_from_annots(
+        [
+            {
+                "Subtype": PSLiteral("Widget"),
+                "FT": PSLiteral("Tx"),
+                "V": b"\xfe\xff\x00J\x00a\x00n\x00e",
+                "Rect": (10, 80, 90, 95),
+            }
+        ],
+        height=100,
+    )
+
+    assert widgets == [{"text": "Jane", "bbox": (10.0, 5.0, 90.0, 20.0)}]
+
+
+def test_get_widget_text_from_annots_decodes_choice_field_value_arrays():
+    from pdfminer.psparser import PSLiteral
+
+    widgets = get_widget_text_from_annots(
+        [
+            {
+                "Subtype": PSLiteral("Widget"),
+                "FT": PSLiteral("Ch"),
+                "V": [PSLiteral("ChoiceA"), b"\xfe\xff\x00C\x00h\x00o\x00i\x00c\x00e\x00B"],
+                "Rect": (10, 70, 90, 95),
+            }
+        ],
+        height=100,
+    )
+
+    assert widgets == [{"text": "ChoiceA\nChoiceB", "bbox": (10.0, 5.0, 90.0, 30.0)}]
+
+
+def test_get_widget_text_from_annots_inherits_field_type_and_value_from_parent():
+    """FT/V absent on the widget are inherited by walking up the /Parent chain.
+
+    The intermediate parent is a direct dict (the case PDFObjRef-only traversal missed) and
+    the root parent carries the inherited /FT, exercising multi-level inheritance.
+    """
+    from pdfminer.psparser import PSLiteral
+
+    root_parent = {"FT": PSLiteral("Tx")}  # field type lives at the top of the hierarchy
+    mid_parent = {"V": b"\xfe\xff\x00J\x00a\x00n\x00e", "Parent": root_parent}  # value mid-chain
+
+    widgets = get_widget_text_from_annots(
+        [
+            {
+                "Subtype": PSLiteral("Widget"),
+                "Parent": mid_parent,  # neither FT nor V on the widget itself
+                "Rect": (10, 80, 90, 95),
+            }
+        ],
+        height=100,
+    )
+
+    assert widgets == [{"text": "Jane", "bbox": (10.0, 5.0, 90.0, 20.0)}]
+
+
+def test_process_file_with_pdfminer_recovers_form_field_text():
+    """The extracted (hi_res) layer includes AcroForm field values as text regions."""
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        pdf_path = os.path.join(tmp_dir, "form.pdf")
+        _build_synthetic_form_pdf(pdf_path)
+        layout, _ = process_file_with_pdfminer(pdf_path)
+
+    texts = [str(t) for t in layout[0].texts if t]
+    assert "Jane Doe" in texts
+    assert "1990-01-01" in texts
+    assert "123 Main Street" in texts
+    # Widget-sourced regions are marked as extracted text.
+    assert IsExtracted.TRUE in list(layout[0].is_extracted_array)
+
+
+def test_partition_pdf_fast_recovers_form_field_text():
+    """End-to-end: the fast strategy emits elements for filled form fields."""
+    from unstructured.partition.pdf import partition_pdf
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        pdf_path = os.path.join(tmp_dir, "form.pdf")
+        _build_synthetic_form_pdf(pdf_path)
+        elements = partition_pdf(filename=pdf_path, strategy="fast")
+
+    blob = "\n".join(el.text for el in elements)
+    assert "Jane Doe" in blob
+    assert "1990-01-01" in blob
+    assert "123 Main Street" in blob
+
+
+@patch("unstructured.partition.pdf_image.pdfminer_utils.LAParams", return_value=LAParams())
+def test_laprams_are_passed_from_partition_to_pdfminer(pdfminer_mock):
+    partition(
+        filename=example_doc_path("pdf/layout-parser-paper-fast.pdf"),
+        pdfminer_line_margin=1.123,
+        pdfminer_char_margin=None,
+        pdfminer_line_overlap=0.0123,
+        pdfminer_word_margin=3.21,
+    )
+    assert pdfminer_mock.call_args.kwargs == {
+        "line_margin": 1.123,
+        "line_overlap": 0.0123,
+        "word_margin": 3.21,
+    }
+
+
+def create_mock_ltchar(text, invisible=False, rotated=False):
+    """Create a mock LTChar object"""
+
+    graphicstate = Mock()
+    matrix = (1, 0.5, 0, 1, 0, 0) if rotated else (1, 0, 0, 1, 0, 0)
+
+    char = LTChar(
+        matrix=matrix,  # transformation matrix
+        font=Mock(),  # you'd need to mock PDFFont
+        fontsize=12,
+        scaling=1,
+        rise=0,
+        text=text,
+        textwidth=10,
+        textdisp=(0, 1),
+        ncs=Mock(),
+        graphicstate=graphicstate,
+    )
+    if invisible:
+        char.rendermode = 3
+    else:
+        char.rendermode = 0
+
+    return char
+
+
+def create_mock_ltcontainer(chars):
+    """Create a mock LTContainer with LTChar objects"""
+    container = LTContainer(bbox=(0, 0, 1, 1))
+
+    # The container should be iterable
+    container.extend(chars)
+
+    return container
+
+
+# Now you can use it in tests
+def test_text_is_embedded():
+    chars = [
+        create_mock_ltchar("H"),
+        create_mock_ltchar("e"),
+        create_mock_ltchar("l"),
+        create_mock_ltchar("l", rotated=True),
+        create_mock_ltchar("o", invisible=True),
+    ]
+
+    container = create_mock_ltcontainer(chars)
+
+    assert text_is_embedded(container, threshold=0.5)
+    assert not text_is_embedded(container, threshold=0.3)
+
+
+# -- Tests for _deduplicate_ltchars (fake bold fix) --
+
+
+def _create_positioned_ltchar(text: str, x0: float, y0: float) -> LTChar:
+    """Create an LTChar with a specific position for deduplication testing."""
+    graphicstate = Mock()
+    # Matrix format: (a, b, c, d, e, f) where e=x, f=y for translation
+    matrix = (1, 0, 0, 1, x0, y0)
+
+    char = LTChar(
+        matrix=matrix,
+        font=Mock(),
+        fontsize=12,
+        scaling=1,
+        rise=0,
+        text=text,
+        textwidth=10,
+        textdisp=(0, 1),
+        ncs=Mock(),
+        graphicstate=graphicstate,
+    )
+    return char
+
+
+class TestDeduplicateLtchars:
+    """Tests for _deduplicate_ltchars function."""
+
+    def test_empty_list_returns_empty(self):
+        """Empty character list should return empty list."""
+        result = _deduplicate_ltchars([], threshold=3.0)
+        assert result == []
+
+    def test_threshold_zero_disables_deduplication(self):
+        """Threshold of 0 should disable deduplication and return original list."""
+        chars = [
+            _create_positioned_ltchar("A", 10.0, 20.0),
+            _create_positioned_ltchar("A", 10.5, 20.0),  # Would be duplicate
+        ]
+        result = _deduplicate_ltchars(chars, threshold=0)
+        assert len(result) == 2
+
+    def test_fake_bold_duplicates_removed(self):
+        """Fake bold (double-rendered) characters should be deduplicated."""
+        # Simulate "AB" rendered as "AABB" with fake bold
+        chars = [
+            _create_positioned_ltchar("A", 10.0, 20.0),
+            _create_positioned_ltchar("A", 10.5, 20.0),  # Duplicate - close position
+            _create_positioned_ltchar("B", 25.0, 20.0),
+            _create_positioned_ltchar("B", 25.5, 20.0),  # Duplicate - close position
+        ]
+        result = _deduplicate_ltchars(chars, threshold=3.0)
+        assert len(result) == 2
+        assert result[0].get_text() == "A"
+        assert result[1].get_text() == "B"
+
+    def test_legitimate_repeated_chars_preserved(self):
+        """Legitimate repeated characters at different positions should be preserved."""
+        # "AA" where both A's are at legitimately different positions
+        chars = [
+            _create_positioned_ltchar("A", 10.0, 20.0),
+            _create_positioned_ltchar("A", 25.0, 20.0),  # Far enough - not duplicate
+        ]
+        result = _deduplicate_ltchars(chars, threshold=3.0)
+        assert len(result) == 2
+
+    def test_single_char_returns_single(self):
+        """Single character should return single character."""
+        chars = [_create_positioned_ltchar("X", 10.0, 20.0)]
+        result = _deduplicate_ltchars(chars, threshold=3.0)
+        assert len(result) == 1
+        assert result[0].get_text() == "X"
+
+    def test_mixed_duplicates_and_normal(self):
+        """Mix of duplicated and normal characters should be handled correctly."""
+        # "HELLO" where only H and L are fake-bold
+        chars = [
+            _create_positioned_ltchar("H", 10.0, 20.0),
+            _create_positioned_ltchar("H", 10.5, 20.0),  # Duplicate
+            _create_positioned_ltchar("E", 20.0, 20.0),  # Normal
+            _create_positioned_ltchar("L", 30.0, 20.0),
+            _create_positioned_ltchar("L", 30.5, 20.0),  # Duplicate
+            _create_positioned_ltchar("L", 40.0, 20.0),  # Second L (normal, different position)
+            _create_positioned_ltchar("O", 50.0, 20.0),  # Normal
+        ]
+        result = _deduplicate_ltchars(chars, threshold=3.0)
+        assert len(result) == 5
+        text = "".join(c.get_text() for c in result)
+        assert text == "HELLO"

test_unstructured/partition/pdf_image/test_pdfminer_utils.py

@@ -0,0 +1,791 @@
+from importlib import reload
+from unittest.mock import MagicMock, patch
+
+from pdfminer.layout import LTChar, LTContainer, LTFigure, LTLayoutContainer, LTTextLine
+from pdfminer.pdftypes import PDFStream
+
+from test_unstructured.unit_utils import example_doc_path
+from unstructured.partition.pdf import partition_pdf
+from unstructured.partition.pdf_image.pdfminer_utils import (
+    CustomPDFPageInterpreter,
+    _is_duplicate_char,
+    deduplicate_chars_in_text_line,
+    extract_text_objects,
+    get_text_with_deduplication,
+)
+from unstructured.partition.utils import config as partition_config
+
+
+def _make_char():
+    return LTChar(
+        matrix=(1, 0, 0, 1, 0, 0),
+        font=MagicMock(),
+        fontsize=12,
+        scaling=1,
+        rise=0,
+        text="x",
+        textwidth=10,
+        textdisp=(0, 1),
+        ncs=MagicMock(),
+        graphicstate=MagicMock(),
+    )
+
+
+def _make_interpreter(cur_item):
+    interp = object.__new__(CustomPDFPageInterpreter)
+    interp.device = MagicMock()
+    interp.device.cur_item = cur_item
+    interp.textstate = MagicMock()
+    interp.graphicstate = MagicMock()
+    return interp
+
+
+def test_patch_render_mode_only_new_chars():
+    """Only chars added after the snapshot index should be patched."""
+    page = LTLayoutContainer(bbox=(0, 0, 100, 100))
+    interp = _make_interpreter(page)
+
+    old_char = _make_char()
+    page.add(old_char)
+
+    interp.textstate.render = 3
+    interp._patch_current_chars_with_render_mode(start=1)
+
+    assert not hasattr(old_char, "rendermode")
+
+
+def test_patch_render_mode_correct_value():
+    """Chars in the patched range should get the current render mode."""
+    page = LTLayoutContainer(bbox=(0, 0, 100, 100))
+    interp = _make_interpreter(page)
+
+    char = _make_char()
+    page.add(char)
+
+    interp.textstate.render = 3
+    interp._patch_current_chars_with_render_mode(start=0)
+
+    assert char.rendermode == 3
+
+
+def test_patch_render_mode_preserved_after_figure_with_text():
+    """When cur_item reverts to the page after a figure that contained text ops,
+    previously patched chars must keep their original render mode."""
+    page = LTLayoutContainer(bbox=(0, 0, 100, 100))
+    interp = _make_interpreter(page)
+
+    # Text op on page with render_mode=0
+    char_a = _make_char()
+    page.add(char_a)
+    interp.textstate.render = 0
+    interp._patch_current_chars_with_render_mode(start=0)
+
+    # begin_figure — cur_item switches to figure
+    figure = LTFigure("test", (0, 0, 50, 50), (1, 0, 0, 1, 0, 0))
+    interp.device.cur_item = figure
+
+    # Text op inside figure
+    fig_char = _make_char()
+    figure.add(fig_char)
+    interp._patch_current_chars_with_render_mode(start=0)
+
+    # end_figure — cur_item reverts to page
+    interp.device.cur_item = page
+
+    # Render mode changes, new text op on page
+    interp.textstate.render = 3
+    char_b = _make_char()
+    page.add(char_b)
+    interp._patch_current_chars_with_render_mode(start=1)
+
+    assert char_a.rendermode == 0
+    assert char_b.rendermode == 3
+
+
+def test_do_TJ_snapshots_before_super():
+    """do_TJ should snapshot len(objs) before super() adds chars,
+    so only the newly added chars are patched."""
+    page = LTLayoutContainer(bbox=(0, 0, 100, 100))
+    interp = _make_interpreter(page)
+
+    old_char = _make_char()
+    page.add(old_char)
+    old_char.rendermode = 0
+
+    new_char = _make_char()
+
+    def fake_super_do_TJ(self, seq):
+        page.add(new_char)
+
+    interp.textstate.render = 3
+    interp.textstate.font = MagicMock()
+    with patch.object(CustomPDFPageInterpreter.__bases__[0], "do_TJ", fake_super_do_TJ):
+        interp.do_TJ([b"test"])
+
+    assert old_char.rendermode == 0
+    assert new_char.rendermode == 3
+
+
+def test_extract_text_objects_nested_containers():
+    """Test extract_text_objects with nested LTContainers."""
+    # Mock LTTextLine objects
+    mock_text_line1 = MagicMock(spec=LTTextLine)
+    mock_text_line2 = MagicMock(spec=LTTextLine)
+
+    # Mock inner container containing one LTTextLine
+    mock_inner_container = MagicMock(spec=LTContainer)
+    mock_inner_container.__iter__.return_value = [mock_text_line2]
+
+    # Mock outer container containing another LTTextLine and the inner container
+    mock_outer_container = MagicMock(spec=LTContainer)
+    mock_outer_container.__iter__.return_value = [mock_text_line1, mock_inner_container]
+
+    # Call the function with the outer container
+    result = extract_text_objects(mock_outer_container)
+
+    # Assert both text line objects are extracted, even from nested containers
+    assert len(result) == 2
+    assert mock_text_line1 in result
+    assert mock_text_line2 in result
+
+
+# -- Tests for character deduplication (fake bold fix) --
+
+
+def _create_mock_ltchar(
+    text: str, x0: float, y0: float, width: float = 6.0, height: float = 2.0
+) -> MagicMock:
+    """Helper to create a mock LTChar with specified text and position.
+
+    Includes x1, y1 so _is_duplicate_char overlap logic works (fake-bold detection
+    uses bounding box overlap). Default width/height give overlap ratio > 0.5 for
+    chars within threshold distance.
+    """
+    mock_char = MagicMock(spec=LTChar)
+    mock_char.get_text.return_value = text
+    mock_char.x0 = x0
+    mock_char.y0 = y0
+    mock_char.x1 = x0 + width
+    mock_char.y1 = y0 + height
+    return mock_char
+
+
+class TestIsDuplicateChar:
+    """Tests for _is_duplicate_char function."""
+
+    def test_same_char_same_position_is_duplicate(self):
+        """Two identical characters at the same position should be duplicates."""
+        char1 = _create_mock_ltchar("A", 10.0, 20.0)
+        char2 = _create_mock_ltchar("A", 10.0, 20.0)
+        assert _is_duplicate_char(char1, char2, threshold=3.0) is True
+
+    def test_same_char_close_position_is_duplicate(self):
+        """Two identical characters at close positions should be duplicates."""
+        char1 = _create_mock_ltchar("B", 10.0, 20.0)
+        char2 = _create_mock_ltchar("B", 11.5, 21.0)  # Within 3.0 threshold
+        assert _is_duplicate_char(char1, char2, threshold=3.0) is True
+
+    def test_same_char_far_position_not_duplicate(self):
+        """Two identical characters at far positions should not be duplicates."""
+        char1 = _create_mock_ltchar("C", 10.0, 20.0)
+        char2 = _create_mock_ltchar("C", 15.0, 20.0)  # 5.0 > 3.0 threshold
+        assert _is_duplicate_char(char1, char2, threshold=3.0) is False
+
+    def test_different_chars_same_position_not_duplicate(self):
+        """Two different characters at the same position should not be duplicates."""
+        char1 = _create_mock_ltchar("A", 10.0, 20.0)
+        char2 = _create_mock_ltchar("B", 10.0, 20.0)
+        assert _is_duplicate_char(char1, char2, threshold=3.0) is False
+
+    def test_threshold_boundary(self):
+        """Test behavior at exact threshold boundary."""
+        char1 = _create_mock_ltchar("X", 10.0, 20.0)
+        char2 = _create_mock_ltchar("X", 13.0, 20.0)  # Exactly at threshold
+        # At threshold means NOT within threshold (uses < not <=)
+        assert _is_duplicate_char(char1, char2, threshold=3.0) is False
+
+        char3 = _create_mock_ltchar("X", 12.9, 20.0)  # Just under threshold
+        assert _is_duplicate_char(char1, char3, threshold=3.0) is True
+
+
+class TestDeduplicateCharsInTextLine:
+    """Tests for deduplicate_chars_in_text_line function."""
+
+    def test_no_duplicates_returns_original(self):
+        """Text line without duplicates should return original text."""
+        chars = [
+            _create_mock_ltchar("H", 10.0, 20.0),
+            _create_mock_ltchar("i", 15.0, 20.0),
+        ]
+        mock_text_line = MagicMock(spec=LTTextLine)
+        mock_text_line.__iter__ = lambda self: iter(chars)
+        mock_text_line.get_text.return_value = "Hi"
+
+        result = deduplicate_chars_in_text_line(mock_text_line, threshold=3.0)
+        assert result == "Hi"
+
+    def test_fake_bold_duplicates_removed(self):
+        """Fake bold text (each char doubled) should be deduplicated."""
+        # Simulates "BOLD" rendered as "BBOOLLDD" with duplicate positions
+        chars = [
+            _create_mock_ltchar("B", 10.0, 20.0),
+            _create_mock_ltchar("B", 10.5, 20.0),  # Duplicate
+            _create_mock_ltchar("O", 20.0, 20.0),
+            _create_mock_ltchar("O", 20.5, 20.0),  # Duplicate
+            _create_mock_ltchar("L", 30.0, 20.0),
+            _create_mock_ltchar("L", 30.5, 20.0),  # Duplicate
+            _create_mock_ltchar("D", 40.0, 20.0),
+            _create_mock_ltchar("D", 40.5, 20.0),  # Duplicate
+        ]
+        mock_text_line = MagicMock(spec=LTTextLine)
+        mock_text_line.__iter__ = lambda self: iter(chars)
+
+        result = deduplicate_chars_in_text_line(mock_text_line, threshold=3.0)
+        assert result == "BOLD"
+
+    def test_threshold_zero_disables_deduplication(self):
+        """Setting threshold to 0 should disable deduplication."""
+        mock_text_line = MagicMock(spec=LTTextLine)
+        mock_text_line.get_text.return_value = "BBOOLLDD"
+
+        result = deduplicate_chars_in_text_line(mock_text_line, threshold=0)
+        assert result == "BBOOLLDD"
+
+    def test_negative_threshold_disables_deduplication(self):
+        """Setting negative threshold should disable deduplication."""
+        mock_text_line = MagicMock(spec=LTTextLine)
+        mock_text_line.get_text.return_value = "BBOOLLDD"
+
+        result = deduplicate_chars_in_text_line(mock_text_line, threshold=-1.0)
+        assert result == "BBOOLLDD"
+
+    def test_empty_text_line(self):
+        """Empty text line should return original text."""
+        mock_text_line = MagicMock(spec=LTTextLine)
+        mock_text_line.__iter__ = lambda self: iter([])
+        mock_text_line.get_text.return_value = ""
+
+        result = deduplicate_chars_in_text_line(mock_text_line, threshold=3.0)
+        assert result == ""
+
+    def test_legitimate_repeated_chars_preserved(self):
+        """Legitimate repeated characters (different positions) should be preserved."""
+        # "AA" where both A's are at different positions
+        chars = [
+            _create_mock_ltchar("A", 10.0, 20.0),
+            _create_mock_ltchar("A", 20.0, 20.0),  # Different position, not duplicate
+        ]
+        mock_text_line = MagicMock(spec=LTTextLine)
+        mock_text_line.__iter__ = lambda self: iter(chars)
+
+        result = deduplicate_chars_in_text_line(mock_text_line, threshold=3.0)
+        assert result == "AA"
+
+
+class TestGetTextWithDeduplication:
+    """Tests for get_text_with_deduplication function."""
+
+    def test_with_text_line(self):
+        """Should properly deduplicate text from LTTextLine."""
+        chars = [
+            _create_mock_ltchar("H", 10.0, 20.0),
+            _create_mock_ltchar("H", 10.5, 20.0),  # Duplicate
+            _create_mock_ltchar("i", 20.0, 20.0),
+        ]
+        mock_text_line = MagicMock(spec=LTTextLine)
+        mock_text_line.__iter__ = lambda self: iter(chars)
+
+        result = get_text_with_deduplication(mock_text_line, threshold=3.0)
+        assert result == "Hi"
+
+    def test_with_container(self):
+        """Should handle LTContainer with nested LTTextLine."""
+        chars = [
+            _create_mock_ltchar("T", 10.0, 20.0),
+            _create_mock_ltchar("T", 10.5, 20.0),  # Duplicate
+        ]
+        mock_text_line = MagicMock(spec=LTTextLine)
+        mock_text_line.__iter__ = lambda self: iter(chars)
+
+        mock_container = MagicMock(spec=LTContainer)
+        mock_container.__iter__ = lambda self: iter([mock_text_line])
+
+        result = get_text_with_deduplication(mock_container, threshold=3.0)
+        assert result == "T"
+
+    def test_with_generic_object(self):
+        """Should fall back to get_text() for non-standard objects."""
+        mock_obj = MagicMock()
+        mock_obj.get_text.return_value = "fallback text"
+
+        result = get_text_with_deduplication(mock_obj, threshold=3.0)
+        assert result == "fallback text"
+
+    def test_without_get_text(self):
+        """Should return empty string for objects without get_text."""
+        mock_obj = MagicMock(spec=[])  # No get_text method
+
+        result = get_text_with_deduplication(mock_obj, threshold=3.0)
+        assert result == ""
+
+
+# -- Integration tests for fake-bold PDF deduplication --
+
+
+class TestFakeBoldPdfIntegration:
+    """Integration tests for fake-bold PDF deduplication using real PDF files.
+
+    The test PDF (fake-bold-sample.pdf) contains text rendered with the "fake bold"
+    technique where each character is drawn twice at slightly offset positions.
+    This causes text extraction to show doubled characters (e.g., "BBOOLLDD" instead
+    of "BOLD") unless deduplication is applied.
+    """
+
+    def test_fake_bold_pdf_without_deduplication_shows_doubled_chars(self, monkeypatch):
+        """Test that extraction WITHOUT deduplication shows doubled characters.
+
+        When PDF_CHAR_DUPLICATE_THRESHOLD is set to 0, deduplication is disabled
+        and the raw text shows the fake-bold doubled characters.
+        """
+        monkeypatch.setenv("PDF_CHAR_DUPLICATE_THRESHOLD", "0")
+        reload(partition_config)
+
+        filename = example_doc_path("pdf/fake-bold-sample.pdf")
+        elements = partition_pdf(filename=filename, strategy="fast")
+        extracted_text = " ".join([el.text for el in elements])
+
+        # Without deduplication, fake-bold text appears with doubled characters
+        assert "BBOOLLDD" in extracted_text, (
+            "Without deduplication, fake-bold text should show doubled characters "
+            "like 'BBOOLLDD' instead of 'BOLD'"
+        )
+
+    def test_fake_bold_pdf_with_deduplication_shows_clean_text(self, monkeypatch):
+        """Test that extraction WITH deduplication shows clean text.
+
+        When PDF_CHAR_DUPLICATE_THRESHOLD is set to default (2.0), deduplication
+        removes the duplicate characters and produces clean, readable text.
+        """
+        monkeypatch.setenv("PDF_CHAR_DUPLICATE_THRESHOLD", "2.0")
+        reload(partition_config)
+
+        filename = example_doc_path("pdf/fake-bold-sample.pdf")
+        elements = partition_pdf(filename=filename, strategy="fast")
+        extracted_text = " ".join([el.text for el in elements])
+
+        # With deduplication, fake-bold text should be clean (no doubled chars)
+        assert "BOLD" in extracted_text, (
+            "With deduplication, text should contain clean 'BOLD' not 'BBOOLLDD'"
+        )
+        # Verify the doubled pattern is NOT present in the deduplicated fake-bold section
+        # Note: The PDF contains 'BBOOLLDD' as explanatory text, so we check for
+        # the specific pattern that would appear if deduplication failed on the
+        # fake-bold rendered text (e.g., "TTEEXXTT" from "TEXT")
+        assert "TTEEXXTT" not in extracted_text, (
+            "With deduplication, fake-bold 'TEXT' should not appear as 'TTEEXXTT'"
+        )
+
+    def test_fake_bold_deduplication_reduces_text_length(self, monkeypatch):
+        """Test that deduplication reduces text length for fake-bold PDFs.
+
+        Compares extraction with and without deduplication to verify that
+        the deduplicated text is shorter due to removal of duplicate characters.
+        """
+        filename = example_doc_path("pdf/fake-bold-sample.pdf")
+
+        # Extract WITHOUT deduplication (threshold=0)
+        monkeypatch.setenv("PDF_CHAR_DUPLICATE_THRESHOLD", "0")
+        reload(partition_config)
+        elements_no_dedup = partition_pdf(filename=filename, strategy="fast")
+        text_no_dedup = " ".join([el.text for el in elements_no_dedup])
+
+        # Extract WITH deduplication (threshold=2.0)
+        monkeypatch.setenv("PDF_CHAR_DUPLICATE_THRESHOLD", "2.0")
+        reload(partition_config)
+        elements_with_dedup = partition_pdf(filename=filename, strategy="fast")
+        text_with_dedup = " ".join([el.text for el in elements_with_dedup])
+
+        # Deduplicated text should be shorter than non-deduplicated text
+        assert len(text_with_dedup) < len(text_no_dedup), (
+            f"Deduplicated text ({len(text_with_dedup)} chars) should be shorter "
+            f"than non-deduplicated text ({len(text_no_dedup)} chars)"
+        )
+
+
+# -- Tests for embedded CMap stream parsing --
+
+
+class TestParseEmbeddedCmapStream:
+    """Unit tests for _parse_embedded_cmap_stream.
+
+    pdfminer.six does not parse embedded Encoding CMap streams for CIDFonts.
+    _parse_embedded_cmap_stream is our workaround that extracts code-to-CID
+    mappings and writing mode from the raw CMap stream bytes.
+    """
+
+    @staticmethod
+    def _parse(data: bytes):
+        from unstructured.partition.pdf_image.pdfminer_utils import _parse_embedded_cmap_stream
+
+        return _parse_embedded_cmap_stream(data)
+
+    def test_vertical_wmode_is_preserved(self):
+        """Embedded CMaps with WMode=1 (vertical) should produce a vertical CMap."""
+        data = b"""
+/WMode 1
+1 begincodespacerange
+<00> <0A>
+endcodespacerange
+1 begincidrange
+<00> <0A> 0
+endcidrange
+"""
+        cmap = self._parse(data)
+        assert cmap.is_vertical() is True
+        assert len(cmap.code2cid) == 11
+
+    def test_horizontal_wmode_is_default(self):
+        """CMaps without WMode or with WMode=0 should produce a horizontal CMap."""
+        data = b"""
+1 begincodespacerange
+<00> <05>
+endcodespacerange
+1 begincidrange
+<00> <05> 0
+endcidrange
+"""
+        cmap = self._parse(data)
+        assert cmap.is_vertical() is False
+
+    def test_oversized_stream_returns_empty_cmap(self):
+        """Streams exceeding the size cap should be rejected before any parsing."""
+        from unstructured.partition.pdf_image.pdfminer_utils import _MAX_CMAP_STREAM_BYTES
+
+        data = b"x" * (_MAX_CMAP_STREAM_BYTES + 1)
+        cmap = self._parse(data)
+        assert not cmap.code2cid
+
+    def test_reversed_range_is_skipped(self):
+        """A begincidrange where end < start should be silently skipped."""
+        data = b"""
+1 begincodespacerange
+<00> <FF>
+endcodespacerange
+1 begincidrange
+<80> <10> 0
+endcidrange
+"""
+        cmap = self._parse(data)
+        assert not cmap.code2cid
+
+    def test_mapping_cap_bounds_total_entries(self):
+        """The total mapping count should be bounded across both cidrange and cidchar."""
+        from unittest.mock import patch
+
+        data = b"""
+1 begincodespacerange
+<0000> <FFFF>
+endcodespacerange
+1 begincidrange
+<0000> <FFFF> 0
+endcidrange
+"""
+        # Use a small cap to verify bounding without allocating huge dicts
+        with patch("unstructured.partition.pdf_image.pdfminer_utils._MAX_CODE2CID_MAPPINGS", 100):
+            cmap = self._parse(data)
+            assert not cmap.code2cid  # 65536 > 100: entire CMap discarded, not partial
+
+    def test_mapping_budget_second_range_discards_entire_cmap(self):
+        """If a later cidrange would exceed the cap, reject the whole map (no holes)."""
+        from unittest.mock import patch
+
+        data = b"""
+1 begincodespacerange
+<00> <19>
+endcodespacerange
+2 begincidrange
+<00> <09> 0
+<0A> <19> 10
+endcidrange
+"""
+        with patch("unstructured.partition.pdf_image.pdfminer_utils._MAX_CODE2CID_MAPPINGS", 15):
+            cmap = self._parse(data)
+            assert not cmap.code2cid
+
+
+class TestBoundedStreamDecode:
+    """Tests for _decode_pdfstream_with_limit.
+
+    Verifies that oversized embedded CMap streams are rejected *before* full
+    materialization, and that the shared PDFStream object is never mutated.
+    """
+
+    def test_oversized_flate_stream_rejected_before_materialization(self):
+        """A small compressed payload that expands past the limit should be rejected
+        without fully materializing the output, and the stream should not be mutated."""
+        import zlib
+
+        from pdfminer.psparser import LIT
+
+        from unstructured.partition.pdf_image.pdfminer_utils import (
+            _decode_pdfstream_with_limit,
+        )
+
+        payload = zlib.compress(b"x" * 200)
+        stream = PDFStream({"Filter": LIT("FlateDecode")}, payload)
+
+        result = _decode_pdfstream_with_limit(stream, max_decoded_bytes=100)
+        assert result is None
+        # Stream object must not be mutated
+        assert stream.get_rawdata() is not None
+        assert stream.data is None
+
+    def test_normal_stream_decodes_within_limit(self):
+        """A stream that fits within the limit should decode successfully."""
+        import zlib
+
+        from pdfminer.psparser import LIT
+
+        from unstructured.partition.pdf_image.pdfminer_utils import (
+            _decode_pdfstream_with_limit,
+        )
+
+        content = b"begincidrange <00> <05> 0 endcidrange"
+        payload = zlib.compress(content)
+        stream = PDFStream({"Filter": LIT("FlateDecode")}, payload)
+
+        result = _decode_pdfstream_with_limit(stream, max_decoded_bytes=1000)
+        assert result == content
+
+    def test_encrypted_stream_without_objid_returns_none(self):
+        """Decipher requires objid/genno; missing values skip decode (no crash)."""
+        from unstructured.partition.pdf_image.pdfminer_utils import (
+            _decode_pdfstream_with_limit,
+        )
+
+        stream = PDFStream({}, b"encrypted-bytes", decipher=lambda *a: b"x")
+        stream.objid = None
+        stream.genno = 0
+        assert _decode_pdfstream_with_limit(stream, max_decoded_bytes=1000) is None
+
+        stream.objid = 1
+        stream.genno = None
+        assert _decode_pdfstream_with_limit(stream, max_decoded_bytes=1000) is None
+
+    def test_uncompressed_stream_returns_raw(self):
+        """A stream with no filters should return the raw data directly."""
+        from unstructured.partition.pdf_image.pdfminer_utils import (
+            _decode_pdfstream_with_limit,
+        )
+
+        content = b"begincidrange <00> <05> 0 endcidrange"
+        stream = PDFStream({}, content)
+
+        result = _decode_pdfstream_with_limit(stream, max_decoded_bytes=1000)
+        assert result == content
+
+
+class TestCustomPDFCIDFont:
+    """Tests for CustomPDFCIDFont constructor-time CMap resolution."""
+
+    def test_vertical_wmode_sets_font_vertical_flag(self):
+        """A vertical embedded CMap (WMode=1) should set font.vertical=True and
+        font.default_disp != 0 at construction time, not just on the CMap."""
+        import zlib
+
+        from pdfminer.pdfinterp import PDFResourceManager
+        from pdfminer.psparser import LIT
+
+        from unstructured.partition.pdf_image.pdfminer_utils import CustomPDFCIDFont
+
+        cmap_data = b"""/CIDInit /ProcSet findresource begin
+12 dict begin
+begincmap
+/CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> def
+/CMapName /Test-Vertical-H def
+/CMapType 1 def
+/WMode 1 def
+1 begincodespacerange
+<00> <0A>
+endcodespacerange
+1 begincidrange
+<00> <0A> 0
+endcidrange
+endcmap
+end
+end"""
+
+        encoding_stream = PDFStream(
+            {
+                "Type": LIT("CMap"),
+                "CMapName": LIT("Test-Vertical-H"),
+                "CIDSystemInfo": {
+                    "Registry": b"Adobe",
+                    "Ordering": b"Identity",
+                    "Supplement": 0,
+                },
+                "Filter": LIT("FlateDecode"),
+            },
+            zlib.compress(cmap_data),
+        )
+
+        tounicode_data = b"""/CIDInit /ProcSet findresource begin
+12 dict begin
+begincmap
+/CIDSystemInfo << /Registry (Adobe) /Ordering (UCS) /Supplement 0 >> def
+/CMapName /Adobe-Identity-UCS def
+/CMapType 2 def
+1 begincodespacerange
+<00> <FF>
+endcodespacerange
+1 beginbfrange
+<00> <0A> <0041>
+endbfrange
+endcmap
+end
+end"""
+        tounicode_stream = PDFStream({}, tounicode_data)
+
+        spec = {
+            "Type": LIT("Font"),
+            "Subtype": LIT("CIDFontType2"),
+            "BaseFont": LIT("TestSans"),
+            "CIDSystemInfo": {
+                "Registry": b"Adobe",
+                "Ordering": b"Identity",
+                "Supplement": 0,
+            },
+            "FontDescriptor": {
+                "Type": LIT("FontDescriptor"),
+                "FontName": LIT("TestSans"),
+                "Flags": 32,
+                "FontBBox": [0, -200, 1000, 800],
+                "ItalicAngle": 0,
+                "Ascent": 800,
+                "Descent": -200,
+                "CapHeight": 700,
+                "StemV": 80,
+            },
+            "DW": 500,
+            "Encoding": encoding_stream,
+            "ToUnicode": tounicode_stream,
+        }
+
+        rsrcmgr = PDFResourceManager()
+        font = CustomPDFCIDFont(rsrcmgr, spec, strict=False)
+
+        assert font.cmap.is_vertical() is True
+        assert font.vertical is True
+        assert font.default_disp != 0
+
+
+class TestCustomPDFResourceManager:
+    """Tests for CustomPDFResourceManager font construction routing."""
+
+    def test_returns_custom_cidfont_for_cid_subtypes(self):
+        """CIDFontType2 fonts should be constructed as CustomPDFCIDFont."""
+        from pdfminer.psparser import LIT
+
+        from unstructured.partition.pdf_image.pdfminer_utils import (
+            CustomPDFCIDFont,
+            CustomPDFResourceManager,
+        )
+
+        cmap_data = b"""begincmap
+1 begincodespacerange <00> <05> endcodespacerange
+1 begincidrange <00> <05> 0 endcidrange
+endcmap"""
+
+        spec = {
+            "Type": LIT("Font"),
+            "Subtype": LIT("CIDFontType2"),
+            "BaseFont": LIT("TestSans"),
+            "CIDSystemInfo": {
+                "Registry": b"Adobe",
+                "Ordering": b"Identity",
+                "Supplement": 0,
+            },
+            "FontDescriptor": {
+                "Type": LIT("FontDescriptor"),
+                "FontName": LIT("TestSans"),
+                "Flags": 32,
+                "FontBBox": [0, -200, 1000, 800],
+                "ItalicAngle": 0,
+                "Ascent": 800,
+                "Descent": -200,
+                "CapHeight": 700,
+                "StemV": 80,
+            },
+            "DW": 500,
+            "Encoding": PDFStream(
+                {
+                    "Type": LIT("CMap"),
+                    "CMapName": LIT("Test-Custom-H"),
+                    "CIDSystemInfo": {
+                        "Registry": b"Adobe",
+                        "Ordering": b"Identity",
+                        "Supplement": 0,
+                    },
+                },
+                cmap_data,
+            ),
+        }
+
+        rsrcmgr = CustomPDFResourceManager()
+        font = rsrcmgr.get_font(1, spec)
+
+        assert isinstance(font, CustomPDFCIDFont)
+
+    def test_caches_font_on_repeated_calls(self):
+        """Repeated get_font calls with the same objid should return the cached instance."""
+        from pdfminer.psparser import LIT
+
+        from unstructured.partition.pdf_image.pdfminer_utils import (
+            CustomPDFResourceManager,
+        )
+
+        cmap_data = b"""begincmap
+1 begincodespacerange <00> <05> endcodespacerange
+1 begincidrange <00> <05> 0 endcidrange
+endcmap"""
+
+        spec = {
+            "Type": LIT("Font"),
+            "Subtype": LIT("CIDFontType2"),
+            "BaseFont": LIT("TestSans"),
+            "CIDSystemInfo": {
+                "Registry": b"Adobe",
+                "Ordering": b"Identity",
+                "Supplement": 0,
+            },
+            "FontDescriptor": {
+                "Type": LIT("FontDescriptor"),
+                "FontName": LIT("TestSans"),
+                "Flags": 32,
+                "FontBBox": [0, -200, 1000, 800],
+                "ItalicAngle": 0,
+                "Ascent": 800,
+                "Descent": -200,
+                "CapHeight": 700,
+                "StemV": 80,
+            },
+            "DW": 500,
+            "Encoding": PDFStream(
+                {
+                    "Type": LIT("CMap"),
+                    "CMapName": LIT("Test-Custom-H"),
+                    "CIDSystemInfo": {
+                        "Registry": b"Adobe",
+                        "Ordering": b"Identity",
+                        "Supplement": 0,
+                    },
+                },
+                cmap_data,
+            ),
+        }
+
+        rsrcmgr = CustomPDFResourceManager()
+        font1 = rsrcmgr.get_font(42, spec)
+        font2 = rsrcmgr.get_font(42, spec)
+
+        assert font1 is font2

test_unstructured/partition/test_api.py

@@ -0,0 +1,660 @@
+import base64
+import contextlib
+import json
+import os
+import pathlib
+from typing import Any
+from unittest.mock import Mock
+
+import pytest
+import requests
+from unstructured_client.general import General
+from unstructured_client.models import shared
+from unstructured_client.models.operations import PartitionRequest
+from unstructured_client.models.shared import PartitionParameters
+from unstructured_client.utils import retries
+
+from unstructured.documents.elements import ElementType, NarrativeText
+from unstructured.partition.api import (
+    DEFAULT_RETRIES_MAX_ELAPSED_TIME_SEC,
+    DEFAULT_RETRIES_MAX_INTERVAL_SEC,
+    get_retries_config,
+    partition_multiple_via_api,
+    partition_via_api,
+)
+
+from ..unit_utils import ANY, FixtureRequest, example_doc_path, method_mock
+
+DIRECTORY = pathlib.Path(__file__).parent.resolve()
+
+# NOTE(yao): point to paid API for now
+API_URL = "https://api.unstructuredapp.io/general/v0/general"
+
+is_in_ci = os.getenv("CI", "").lower() not in {"", "false", "f", "0"}
+skip_not_on_main = os.getenv("GITHUB_REF_NAME", "").lower() != "main"
+
+
+def test_partition_via_api_with_filename_correctly_calls_sdk(
+    request: FixtureRequest, expected_call_: list[Any]
+):
+    partition_mock_ = method_mock(
+        request, General, "partition", return_value=FakeResponse(status_code=200)
+    )
+
+    elements = partition_via_api(filename=example_doc_path("eml/fake-email.eml"))
+
+    partition_mock_.assert_called_once_with(
+        expected_call_[0], request=expected_call_[1], retries=expected_call_[2]
+    )
+    assert isinstance(partition_mock_.call_args_list[0].args[0], General)
+    assert len(elements) == 1
+    assert elements[0] == NarrativeText("This is a test email to use for unit tests.")
+    assert elements[0].metadata.filetype == "message/rfc822"
+
+
+def test_partition_via_api_with_file_correctly_calls_sdk(
+    request: FixtureRequest, expected_call_: list[Any]
+):
+    partition_mock_ = method_mock(
+        request, General, "partition", return_value=FakeResponse(status_code=200)
+    )
+
+    with open(example_doc_path("eml/fake-email.eml"), "rb") as f:
+        elements = partition_via_api(
+            file=f, metadata_filename=example_doc_path("eml/fake-email.eml")
+        )
+
+    # Update the fixture content to match the format passed to partition_via_api
+    modified_expected_call = expected_call_[:]
+    modified_expected_call[1].partition_parameters.files.content = f
+
+    partition_mock_.assert_called_once_with(
+        modified_expected_call[0],
+        request=modified_expected_call[1],
+        retries=modified_expected_call[2],
+    )
+    assert isinstance(partition_mock_.call_args_list[0].args[0], General)
+    assert len(elements) == 1
+    assert elements[0] == NarrativeText("This is a test email to use for unit tests.")
+    assert elements[0].metadata.filetype == "message/rfc822"
+
+
+def test_partition_via_api_warns_with_file_and_filename_and_calls_sdk(
+    request: FixtureRequest, expected_call_: list[Any], caplog: pytest.LogCaptureFixture
+):
+    partition_mock_ = method_mock(
+        request, General, "partition", return_value=FakeResponse(status_code=200)
+    )
+
+    with open(example_doc_path("eml/fake-email.eml"), "rb") as f:
+        partition_via_api(file=f, file_filename=example_doc_path("eml/fake-email.eml"))
+
+    # Update the fixture content to match the format passed to partition_via_api
+    modified_expected_call = expected_call_[:]
+    modified_expected_call[1].partition_parameters.files.content = f
+
+    partition_mock_.assert_called_once_with(
+        modified_expected_call[0],
+        request=modified_expected_call[1],
+        retries=modified_expected_call[2],
+    )
+    assert "WARNING" in caplog.text
+    assert "The file_filename kwarg will be deprecated" in caplog.text
+
+
+def test_partition_via_api_from_file_raises_with_metadata_and_file_and_filename():
+    filename = example_doc_path("eml/fake-email.eml")
+
+    with open(filename, "rb") as f, pytest.raises(ValueError):
+        partition_via_api(file=f, file_filename=filename, metadata_filename=filename)
+
+
+def test_partition_via_api_from_file_raises_without_filename():
+    with open(example_doc_path("eml/fake-email.eml"), "rb") as f, pytest.raises(ValueError):
+        partition_via_api(file=f)
+
+
+def test_partition_via_api_raises_with_bad_response(request: FixtureRequest):
+    partition_mock_ = method_mock(
+        request, General, "partition", return_value=FakeResponse(status_code=500)
+    )
+
+    with pytest.raises(ValueError):
+        partition_via_api(filename=example_doc_path("eml/fake-email.eml"))
+    partition_mock_.assert_called_once()
+
+
+@pytest.mark.skipif(not is_in_ci, reason="Skipping test run outside of CI")
+@pytest.mark.skipif(skip_not_on_main, reason="Skipping test run outside of main branch")
+def test_partition_via_api_with_no_strategy():
+    test_file = example_doc_path("pdf/loremipsum-flat.pdf")
+    elements_no_strategy = partition_via_api(
+        filename=test_file,
+        strategy="auto",
+        api_key=get_api_key(),
+        # The url has changed since the 06/24 API release while the sdk defaults to the old url
+        api_url=API_URL,
+        skip_infer_table_types=["pdf"],
+    )
+    elements_hi_res = partition_via_api(
+        filename=test_file,
+        strategy="hi_res",
+        api_key=get_api_key(),
+        # The url has changed since the 06/24 API release while the sdk defaults to the old url
+        api_url=API_URL,
+        skip_infer_table_types=["pdf"],
+    )
+    elements_fast_res = partition_via_api(
+        filename=test_file,
+        strategy="fast",
+        api_key=get_api_key(),
+        # The url has changed since the 06/24 API release while the sdk defaults to the old url
+        api_url=API_URL,
+        skip_infer_table_types=["pdf"],
+    )
+
+    # confirm that hi_res strategy was not passed as default to partition by comparing outputs
+    # elements_hi_res[3].text =
+    #     'LayoutParser: A Unified Toolkit for Deep Learning Based Document Image Analysis'
+    # while elements_no_strategy[3].text = ']' (as of this writing)
+    assert len(elements_no_strategy) == len(elements_hi_res)
+    assert len(elements_hi_res) != len(elements_fast_res)
+
+    # NOTE(crag): slightly out scope assertion, but avoid extra API call
+    assert elements_hi_res[0].metadata.coordinates is None
+
+
+@pytest.mark.skipif(not is_in_ci, reason="Skipping test run outside of CI")
+@pytest.mark.skipif(skip_not_on_main, reason="Skipping test run outside of main branch")
+def test_partition_via_api_with_image_hi_res_strategy_includes_coordinates():
+    # coordinates not included by default to limit payload size
+    elements = partition_via_api(
+        filename=example_doc_path("pdf/fake-memo.pdf"),
+        strategy="hi_res",
+        coordinates="true",
+        api_key=get_api_key(),
+        api_url=API_URL,
+    )
+
+    assert elements[0].metadata.coordinates is not None
+
+
+@pytest.mark.skipif(not is_in_ci, reason="Skipping test run outside of CI")
+@pytest.mark.skipif(skip_not_on_main, reason="Skipping test run outside of main branch")
+def test_partition_via_api_image_block_extraction():
+    elements = partition_via_api(
+        filename=example_doc_path("pdf/embedded-images-tables.pdf"),
+        strategy="hi_res",
+        extract_image_block_types=["image", "table"],
+        api_key=get_api_key(),
+        # The url has changed since the 06/24 API release while the sdk defaults to the old url
+        api_url=API_URL,
+    )
+    image_elements = [el for el in elements if el.category == ElementType.IMAGE]
+    for el in image_elements:
+        assert el.metadata.image_base64 is not None
+        assert el.metadata.image_mime_type is not None
+        image_data = base64.b64decode(el.metadata.image_base64)
+        assert isinstance(image_data, bytes)
+
+
+@pytest.mark.skipif(not is_in_ci, reason="Skipping test run outside of CI")
+@pytest.mark.skipif(skip_not_on_main, reason="Skipping test run outside of main branch")
+def test_partition_via_api_retries_config():
+    elements = partition_via_api(
+        filename=example_doc_path("pdf/embedded-images-tables.pdf"),
+        strategy="fast",
+        api_key=get_api_key(),
+        # The url has changed since the 06/24 API release while the sdk defaults to the old url
+        api_url=API_URL,
+        retries_initial_interval=5,
+        retries_max_interval=15,
+        retries_max_elapsed_time=100,
+        retries_connection_errors=True,
+        retries_exponent=1.5,
+    )
+
+    assert len(elements) > 0
+
+
+# Note(austin) - This test is way too noisy against the hosted api
+# def test_partition_via_api_invalid_request_data_kwargs():
+#     filename = os.path.join(DIRECTORY, "..", "..", "example-docs", "layout-parser-paper-fast.pdf")
+#     with pytest.raises(SDKError):
+#         partition_via_api(filename=filename, strategy="not_a_strategy")
+
+
+def test_retries_config_with_parameters_set():
+    sdk = Mock()
+    retries_config = get_retries_config(
+        retries_connection_errors=True,
+        retries_exponent=1.75,
+        retries_initial_interval=20,
+        retries_max_elapsed_time=1000,
+        retries_max_interval=100,
+        sdk=sdk,
+    )
+
+    assert retries_config.retry_connection_errors
+    assert retries_config.backoff.exponent == 1.75
+    assert retries_config.backoff.initial_interval == 20
+    assert retries_config.backoff.max_elapsed_time == 1000
+    assert retries_config.backoff.max_interval == 100
+
+
+def test_retries_config_none_parameters_return_empty_config():
+    sdk = Mock()
+    retries_config = get_retries_config(
+        retries_connection_errors=None,
+        retries_exponent=None,
+        retries_initial_interval=None,
+        retries_max_elapsed_time=None,
+        retries_max_interval=None,
+        sdk=sdk,
+    )
+
+    assert retries_config is None
+
+
+def test_retry_config_with_empty_sdk_retry_config_returns_default():
+    sdk = Mock()
+    sdk.sdk_configuration.retry_config = None
+    retries_config = get_retries_config(
+        retries_connection_errors=True,
+        retries_exponent=1.88,
+        retries_initial_interval=3000,
+        retries_max_elapsed_time=None,
+        retries_max_interval=None,
+        sdk=sdk,
+    )
+
+    assert retries_config.retry_connection_errors
+    assert retries_config.backoff.exponent == 1.88
+    assert retries_config.backoff.initial_interval == 3000
+    assert retries_config.backoff.max_elapsed_time == DEFAULT_RETRIES_MAX_ELAPSED_TIME_SEC
+    assert retries_config.backoff.max_interval == DEFAULT_RETRIES_MAX_INTERVAL_SEC
+
+
+def test_retries_config_with_no_parameters_set():
+    retry_config = retries.RetryConfig(
+        "backoff", retries.BackoffStrategy(3000, 720000, 1.88, 1800000), True
+    )
+    sdk = Mock()
+    sdk.sdk_configuration.retry_config = retry_config
+    retries_config = get_retries_config(
+        retries_connection_errors=True,
+        retries_exponent=None,
+        retries_initial_interval=None,
+        retries_max_elapsed_time=None,
+        retries_max_interval=None,
+        sdk=sdk,
+    )
+
+    assert retries_config.retry_connection_errors
+    assert retries_config.backoff.exponent == 1.88
+    assert retries_config.backoff.initial_interval == 3000
+    assert retries_config.backoff.max_elapsed_time == 1800000
+    assert retries_config.backoff.max_interval == 720000
+
+
+def test_retries_config_cascade():
+    # notice max_interval is set to 0 which is incorrect - so the DEFAULT_RETRIES_MAX_INTERVAL_SEC
+    # should be used
+    retry_config = retries.RetryConfig(
+        "backoff", retries.BackoffStrategy(3000, 0, 1.88, None), True
+    )
+    sdk = Mock()
+    sdk.sdk_configuration.retry_config = retry_config
+    retries_config = get_retries_config(
+        retries_connection_errors=False,
+        retries_exponent=1.75,
+        retries_initial_interval=20,
+        retries_max_elapsed_time=None,
+        retries_max_interval=None,
+        sdk=sdk,
+    )
+
+    assert not retries_config.retry_connection_errors
+    assert retries_config.backoff.exponent == 1.75
+    assert retries_config.backoff.initial_interval == 20
+    assert retries_config.backoff.max_elapsed_time == DEFAULT_RETRIES_MAX_ELAPSED_TIME_SEC
+    assert retries_config.backoff.max_interval == DEFAULT_RETRIES_MAX_INTERVAL_SEC
+
+
+def test_partition_multiple_via_api_with_single_filename(request: FixtureRequest):
+    partition_mock_ = method_mock(
+        request, requests, "post", return_value=FakeResponse(status_code=200)
+    )
+    filename = example_doc_path("eml/fake-email.eml")
+
+    elements = partition_multiple_via_api(filenames=[filename])
+
+    partition_mock_.assert_called_once_with(
+        "https://api.unstructured.io/general/v0/general",
+        headers={"ACCEPT": "application/json", "UNSTRUCTURED-API-KEY": ANY},
+        data={},
+        files=[("files", (example_doc_path("eml/fake-email.eml"), ANY, None))],
+    )
+    assert elements[0][0] == NarrativeText("This is a test email to use for unit tests.")
+    assert elements[0][0].metadata.filetype == "message/rfc822"
+
+
+def test_partition_multiple_via_api_from_filenames(request: FixtureRequest):
+    partition_mock_ = method_mock(
+        request, requests, "post", return_value=FakeMultipleResponse(status_code=200)
+    )
+    filenames = [example_doc_path("eml/fake-email.eml"), example_doc_path("fake.docx")]
+
+    elements = partition_multiple_via_api(filenames=filenames)
+
+    partition_mock_.assert_called_once_with(
+        "https://api.unstructured.io/general/v0/general",
+        headers={"ACCEPT": "application/json", "UNSTRUCTURED-API-KEY": ANY},
+        data={},
+        files=[
+            ("files", (example_doc_path("eml/fake-email.eml"), ANY, None)),
+            ("files", (example_doc_path("fake.docx"), ANY, None)),
+        ],
+    )
+    assert len(elements) == 2
+    assert elements[0][0] == NarrativeText("This is a test email to use for unit tests.")
+    assert elements[0][0].metadata.filetype == "message/rfc822"
+
+
+def test_partition_multiple_via_api_from_files(request: FixtureRequest):
+    partition_mock_ = method_mock(
+        request, requests, "post", return_value=FakeMultipleResponse(status_code=200)
+    )
+    filenames = [example_doc_path("eml/fake-email.eml"), example_doc_path("fake.docx")]
+
+    with contextlib.ExitStack() as stack:
+        files = [stack.enter_context(open(filename, "rb")) for filename in filenames]
+        elements = partition_multiple_via_api(
+            files=files,
+            metadata_filenames=filenames,
+        )
+
+    partition_mock_.assert_called_once_with(
+        "https://api.unstructured.io/general/v0/general",
+        headers={"ACCEPT": "application/json", "UNSTRUCTURED-API-KEY": ANY},
+        data={},
+        files=[
+            ("files", (example_doc_path("eml/fake-email.eml"), ANY, None)),
+            ("files", (example_doc_path("fake.docx"), ANY, None)),
+        ],
+    )
+    assert len(elements) == 2
+    assert elements[0][0] == NarrativeText("This is a test email to use for unit tests.")
+    assert elements[0][0].metadata.filetype == "message/rfc822"
+
+
+def test_partition_multiple_via_api_warns_with_file_filename(
+    caplog: pytest.LogCaptureFixture, request: FixtureRequest
+):
+    partition_mock_ = method_mock(
+        request, requests, "post", return_value=FakeMultipleResponse(status_code=200)
+    )
+    filenames = [example_doc_path("eml/fake-email.eml"), example_doc_path("fake.docx")]
+
+    with contextlib.ExitStack() as stack:
+        files = [stack.enter_context(open(filename, "rb")) for filename in filenames]
+        partition_multiple_via_api(
+            files=files,
+            file_filenames=filenames,
+        )
+
+    partition_mock_.assert_called_once_with(
+        "https://api.unstructured.io/general/v0/general",
+        headers={"ACCEPT": "application/json", "UNSTRUCTURED-API-KEY": ANY},
+        data={},
+        files=[
+            ("files", (example_doc_path("eml/fake-email.eml"), ANY, None)),
+            ("files", (example_doc_path("fake.docx"), ANY, None)),
+        ],
+    )
+    assert "WARNING" in caplog.text
+    assert "The file_filenames kwarg will be deprecated" in caplog.text
+
+
+def test_partition_multiple_via_api_raises_with_file_and_metadata_filename():
+    filenames = [example_doc_path("eml/fake-email.eml"), example_doc_path("fake.docx")]
+
+    with contextlib.ExitStack() as stack:
+        files = [stack.enter_context(open(filename, "rb")) for filename in filenames]
+        with pytest.raises(ValueError):
+            partition_multiple_via_api(
+                files=files,
+                metadata_filenames=filenames,
+                file_filenames=filenames,
+            )
+
+
+def test_partition_multiple_via_api_raises_with_bad_response(request: FixtureRequest):
+    partition_mock_ = method_mock(
+        request, requests, "post", return_value=FakeMultipleResponse(status_code=500)
+    )
+    filenames = [example_doc_path("eml/fake-email.eml"), example_doc_path("fake.docx")]
+
+    with pytest.raises(ValueError):
+        partition_multiple_via_api(filenames=filenames)
+    partition_mock_.assert_called_once_with(
+        "https://api.unstructured.io/general/v0/general",
+        headers={"ACCEPT": "application/json", "UNSTRUCTURED-API-KEY": ANY},
+        data={},
+        files=[
+            ("files", (example_doc_path("eml/fake-email.eml"), ANY, None)),
+            ("files", (example_doc_path("fake.docx"), ANY, None)),
+        ],
+    )
+
+
+def test_partition_multiple_via_api_raises_with_content_types_size_mismatch():
+    filenames = [example_doc_path("eml/fake-email.eml"), example_doc_path("fake.docx")]
+
+    with pytest.raises(ValueError):
+        partition_multiple_via_api(
+            filenames=filenames,
+            content_types=["text/plain"],
+        )
+
+
+def test_partition_multiple_via_api_from_files_raises_with_size_mismatch():
+    filenames = [example_doc_path("eml/fake-email.eml"), example_doc_path("fake.docx")]
+
+    with contextlib.ExitStack() as stack:
+        files = [stack.enter_context(open(filename, "rb")) for filename in filenames]
+        with pytest.raises(ValueError):
+            partition_multiple_via_api(
+                files=files,
+                metadata_filenames=filenames,
+                content_types=["text/plain"],
+            )
+
+
+def test_partition_multiple_via_api_from_files_raises_without_filenames():
+    filenames = [example_doc_path("eml/fake-email.eml"), example_doc_path("fake.docx")]
+
+    with contextlib.ExitStack() as stack:
+        files = [stack.enter_context(open(filename, "rb")) for filename in filenames]
+        with pytest.raises(ValueError):
+            partition_multiple_via_api(
+                files=files,
+            )
+
+
+def get_api_key():
+    api_key = os.getenv("UNS_API_KEY")
+    if api_key is None:
+        raise ValueError("UNS_API_KEY environment variable not set")
+    return api_key
+
+
+@pytest.mark.skipif(not is_in_ci, reason="Skipping test run outside of CI")
+@pytest.mark.skipif(skip_not_on_main, reason="Skipping test run outside of main branch")
+def test_partition_multiple_via_api_valid_request_data_kwargs():
+    filenames = [
+        example_doc_path("fake-text.txt"),
+        example_doc_path("fake-email.txt"),
+    ]
+
+    list_of_lists_of_elements = partition_multiple_via_api(
+        filenames=filenames,
+        strategy="fast",
+        api_key=get_api_key(),
+        api_url=API_URL,
+    )
+    # assert there is a list of elements for each file
+    assert len(list_of_lists_of_elements) == 2
+    assert isinstance(list_of_lists_of_elements[0], list)
+    assert isinstance(list_of_lists_of_elements[1], list)
+
+
+@pytest.mark.skipif(not is_in_ci, reason="Skipping test run outside of CI")
+def test_partition_multiple_via_api_invalid_request_data_kwargs():
+    filenames = [
+        example_doc_path("pdf/layout-parser-paper-fast.pdf"),
+        example_doc_path("img/layout-parser-paper-fast.jpg"),
+    ]
+    with pytest.raises(ValueError):
+        partition_multiple_via_api(
+            filenames=filenames,
+            strategy="not_a_strategy",
+            api_key=get_api_key(),
+            # The url has changed since the 06/24 API release while the sdk defaults to the old url
+            api_url=API_URL,
+        )
+
+
+MOCK_TEXT = """[
+    {
+        "element_id": "f49fbd614ddf5b72e06f59e554e6ae2b",
+        "text": "This is a test email to use for unit tests.",
+        "type": "NarrativeText",
+        "metadata": {
+            "sent_from": [
+                "Matthew Robinson <mrobinson@unstructured.io>"
+            ],
+            "sent_to": [
+                "Matthew Robinson <mrobinson@unstructured.io>"
+            ],
+            "subject": "Test Email",
+            "filename": "fake-email.eml",
+            "filetype": "message/rfc822"
+        }
+    }
+]"""
+
+
+class FakeResponse:
+    def __init__(self, status_code: int):
+        self.status_code = status_code
+        # The string representation of partitioned elements is nested in an additional
+        # layer in the new unstructured-client:
+        #     `elements_from_json(text=response.raw_response.text)`
+        self.raw_response = FakeRawResponse()
+        self.headers = {"Content-Type": "application/json"}
+
+    def json(self):
+        return json.loads(self.text)
+
+    @property
+    def text(self):
+        return MOCK_TEXT
+
+
+class FakeRawResponse:
+    def __init__(self):
+        self.text = MOCK_TEXT
+
+
+class FakeMultipleResponse:
+    def __init__(self, status_code: int):
+        self.status_code = status_code
+
+    def json(self):
+        return json.loads(self.text)
+
+    @property
+    def text(self):
+        return """[
+    [
+        {
+            "element_id": "f49fbd614ddf5b72e06f59e554e6ae2b",
+            "text": "This is a test email to use for unit tests.",
+            "type": "NarrativeText",
+            "metadata": {
+                "sent_from": [
+                    "Matthew Robinson <mrobinson@unstructured.io>"
+                ],
+                "sent_to": [
+                    "Matthew Robinson <mrobinson@unstructured.io>"
+                ],
+                "subject": "Test Email",
+                "filename": "fake-email.eml",
+                "filetype": "message/rfc822"
+            }
+        }
+    ],
+    [
+        {
+            "element_id": "f49fbd614ddf5b72e06f59e554e6ae2b",
+            "text": "This is a test email to use for unit tests.",
+            "type": "NarrativeText",
+            "metadata": {
+                "sent_from": [
+                    "Matthew Robinson <mrobinson@unstructured.io>"
+                ],
+                "sent_to": [
+                    "Matthew Robinson <mrobinson@unstructured.io>"
+                ],
+                "subject": "Test Email",
+                "filename": "fake-email.eml",
+                "filetype": "message/rfc822"
+            }
+        }
+    ]
+]"""
+
+
+@pytest.fixture()
+def expected_call_():
+    with open(example_doc_path("eml/fake-email.eml"), "rb") as f:
+        file_bytes = f.read()
+    return [
+        ANY,
+        PartitionRequest(
+            partition_parameters=PartitionParameters(
+                files=shared.Files(
+                    content=file_bytes,
+                    file_name=example_doc_path("eml/fake-email.eml"),
+                ),
+                chunking_strategy=None,
+                combine_under_n_chars=None,
+                coordinates=False,
+                encoding=None,
+                extract_image_block_types=None,
+                gz_uncompressed_content_type=None,
+                hi_res_model_name=None,
+                include_orig_elements=None,
+                include_page_breaks=False,
+                languages=None,
+                max_characters=None,
+                multipage_sections=True,
+                new_after_n_chars=None,
+                ocr_languages=None,
+                output_format=shared.OutputFormat.APPLICATION_JSON,
+                overlap=0,
+                overlap_all=False,
+                pdf_infer_table_structure=True,
+                similarity_threshold=None,
+                skip_infer_table_types=None,
+                split_pdf_concurrency_level=5,
+                split_pdf_page=True,
+                starting_page_number=None,
+                strategy=shared.Strategy.HI_RES,
+                unique_element_ids=False,
+                xml_keep_tags=False,
+            )
+        ),
+        None,  # retries kwarg
+    ]

test_unstructured/partition/test_audio.py

@@ -0,0 +1,416 @@
+# pyright: reportPrivateUsage=false
+
+"""Tests for partition_audio (speech-to-text in multimodal pipeline)."""
+
+from __future__ import annotations
+
+import tempfile
+from contextlib import contextmanager
+from pathlib import Path
+from typing import IO, Generator
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from unstructured.documents.elements import NarrativeText
+from unstructured.file_utils.model import FileType
+from unstructured.partition.audio import partition_audio
+
+# ================================================================================================
+# Shared fixtures
+# ================================================================================================
+
+_ONE_SEGMENT = [{"text": "Hello, world.", "start": 0.0, "end": 1.0}]
+
+
+@contextmanager
+def _tmp_audio(suffix: str = ".wav", content: bytes = b"\x00" * 44) -> Generator[str, None, None]:
+    """Yield the path to a temporary audio file, deleting it on exit."""
+    with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:
+        tmp.write(content)
+        tmp.flush()
+        path = tmp.name
+    try:
+        yield path
+    finally:
+        Path(path).unlink(missing_ok=True)
+
+
+@pytest.fixture
+def mock_stt_agent() -> Generator[MagicMock, None, None]:
+    """Patch SpeechToTextAgent.get_agent and yield the mock agent instance.
+
+    Tests that only need a working agent (and don't care about call details) can use
+    this fixture directly. Tests that need to inspect mock calls should use
+    ``@patch("unstructured.partition.audio.SpeechToTextAgent.get_agent")`` directly so
+    the mock is visible in the function signature.
+    """
+    with patch("unstructured.partition.audio.SpeechToTextAgent.get_agent") as mock_get_agent:
+        agent = mock_get_agent.return_value
+        agent.transcribe_segments.return_value = _ONE_SEGMENT
+        yield agent
+
+
+@pytest.fixture
+def _clear_stt_cache() -> Generator[None, None, None]:
+    """Clear SpeechToTextAgent.get_instance lru_cache before and after the test.
+
+    Use in any test that touches get_instance or get_agent directly so cached state
+    does not leak between tests (mirrors OCRAgent _clear_cache in test_ocr_interface).
+    Tests that patch get_instance (e.g. get_agent tests using mock_get_instance) never
+    call the real get_instance, so the cache is unchanged by them and this fixture is
+    a no-op for those; it matters for tests that call get_instance directly.
+    """
+    from unstructured.partition.utils.speech_to_text.speech_to_text_interface import (
+        SpeechToTextAgent,
+    )
+
+    SpeechToTextAgent.get_instance.cache_clear()
+    yield
+    SpeechToTextAgent.get_instance.cache_clear()
+
+
+# ================================================================================================
+# Input validation
+# ================================================================================================
+
+
+def test_partition_audio_raises_with_neither_filename_nor_file():
+    with pytest.raises(ValueError, match="Exactly one of .* must be specified"):
+        partition_audio()
+
+
+def test_partition_audio_raises_with_both_filename_and_file():
+    with _tmp_audio() as path:
+        with pytest.raises(ValueError, match="Exactly one of .* must be specified"):
+            with open(path, "rb") as f:
+                partition_audio(filename=path, file=f)
+
+
+# ================================================================================================
+# Core element output
+# ================================================================================================
+
+
+@patch("unstructured.partition.audio.SpeechToTextAgent.get_agent")
+def test_partition_audio_from_filename_returns_transcript_elements(mock_get_agent):
+    mock_agent = mock_get_agent.return_value
+    mock_agent.transcribe_segments.return_value = [
+        {"text": "Hello, this is a test transcript.", "start": 0.0, "end": 2.5},
+    ]
+
+    with _tmp_audio() as path:
+        elements = partition_audio(filename=path)
+
+    assert len(elements) == 1
+    assert isinstance(elements[0], NarrativeText)
+    assert elements[0].text == "Hello, this is a test transcript."
+    assert elements[0].metadata.detection_origin == "speech_to_text"
+    assert elements[0].metadata.segment_start_seconds == 0.0
+    assert elements[0].metadata.segment_end_seconds == 2.5
+    mock_get_agent.assert_called_once_with(None)
+    mock_agent.transcribe_segments.assert_called_once_with(path, language=None)
+
+
+def test_partition_audio_empty_transcript_returns_empty_list(mock_stt_agent):
+    mock_stt_agent.transcribe_segments.return_value = []
+    with _tmp_audio() as path:
+        assert partition_audio(filename=path) == []
+
+
+def test_partition_audio_returns_one_element_per_segment(mock_stt_agent):
+    mock_stt_agent.transcribe_segments.return_value = [
+        {"text": "First segment.", "start": 0.0, "end": 1.0},
+        {"text": "Second segment.", "start": 1.0, "end": 2.5},
+        {"text": "Third segment.", "start": 2.5, "end": 4.0},
+    ]
+
+    with _tmp_audio() as path:
+        elements = partition_audio(filename=path)
+
+    assert len(elements) == 3
+    assert elements[0].text == "First segment."
+    assert elements[0].metadata.segment_start_seconds == 0.0
+    assert elements[0].metadata.segment_end_seconds == 1.0
+    assert elements[1].text == "Second segment."
+    assert elements[1].metadata.segment_start_seconds == 1.0
+    assert elements[1].metadata.segment_end_seconds == 2.5
+    assert elements[2].text == "Third segment."
+    assert elements[2].metadata.segment_start_seconds == 2.5
+    assert elements[2].metadata.segment_end_seconds == 4.0
+
+
+def test_partition_audio_filters_whitespace_only_segments(mock_stt_agent):
+    mock_stt_agent.transcribe_segments.return_value = [
+        {"text": "  ", "start": 0.0, "end": 0.5},
+        {"text": "Real content.", "start": 0.5, "end": 2.0},
+        {"text": "\t\n", "start": 2.0, "end": 2.5},
+    ]
+
+    with _tmp_audio() as path:
+        elements = partition_audio(filename=path)
+
+    assert len(elements) == 1
+    assert elements[0].text == "Real content."
+
+
+# ================================================================================================
+# Temp-file lifecycle (file-object input)
+# ================================================================================================
+
+
+@patch("unstructured.partition.audio.SpeechToTextAgent.get_agent")
+def test_partition_audio_from_file_uses_temp_path_and_cleans_up(mock_get_agent):
+    mock_agent = mock_get_agent.return_value
+    mock_agent.transcribe_segments.return_value = [
+        {"text": "From file object.", "start": 0.0, "end": 1.0},
+    ]
+
+    captured_temp_path: list[str] = []
+    real_named_temp = tempfile.NamedTemporaryFile
+
+    def spy_named_temp(*args, **kwargs):
+        ctx = real_named_temp(*args, **kwargs)
+        captured_temp_path.append(ctx.name)
+        return ctx
+
+    with _tmp_audio() as path:
+        with open(path, "rb") as f:
+            with patch("unstructured.partition.audio.tempfile.NamedTemporaryFile", spy_named_temp):
+                elements = partition_audio(file=f, metadata_filename="recording.wav")
+
+    assert len(elements) == 1
+    assert elements[0].text == "From file object."
+    assert elements[0].metadata.filename == "recording.wav"
+    assert len(captured_temp_path) == 1, "expected exactly one temp file to be created"
+    assert not Path(captured_temp_path[0]).exists(), "temp file was not deleted after partitioning"
+
+
+@patch("unstructured.partition.audio.SpeechToTextAgent.get_agent")
+def test_partition_audio_cleans_up_temp_file_when_transcription_raises(mock_get_agent):
+    mock_agent = mock_get_agent.return_value
+    mock_agent.transcribe_segments.side_effect = RuntimeError("transcription failed")
+
+    captured_temp_path: list[str] = []
+    real_named_temp = tempfile.NamedTemporaryFile
+
+    def spy_named_temp(*args, **kwargs):
+        ctx = real_named_temp(*args, **kwargs)
+        captured_temp_path.append(ctx.name)
+        return ctx
+
+    with _tmp_audio() as path:
+        with open(path, "rb") as f:
+            with patch("unstructured.partition.audio.tempfile.NamedTemporaryFile", spy_named_temp):
+                with pytest.raises(RuntimeError, match="transcription failed"):
+                    partition_audio(file=f)
+
+    assert len(captured_temp_path) == 1, "expected exactly one temp file to be created"
+    assert not Path(captured_temp_path[0]).exists(), "temp file was not deleted after exception"
+
+
+# ================================================================================================
+# MIME type / filetype metadata
+# ================================================================================================
+
+
+@patch("unstructured.partition.audio.SpeechToTextAgent.get_agent")
+@patch("unstructured.partition.audio.detect_filetype")
+def test_partition_audio_stamps_correct_mime_type_for_detected_format(mock_detect, mock_get_agent):
+    """metadata.filetype reflects the actual detected audio format, not a hardcoded WAV."""
+    mock_detect.return_value = FileType.MP3
+    mock_get_agent.return_value.transcribe_segments.return_value = [
+        {"text": "MP3 content.", "start": 0.0, "end": 1.0},
+    ]
+
+    with _tmp_audio(suffix=".mp3", content=b"\xff\xfb" + b"\x00" * 42) as path:
+        elements = partition_audio(filename=path)
+
+    assert len(elements) == 1
+    assert elements[0].metadata.filetype == FileType.MP3.mime_type  # "audio/mpeg"
+
+
+@patch("unstructured.partition.audio.SpeechToTextAgent.get_agent")
+@patch("unstructured.partition.audio.detect_filetype")
+def test_partition_audio_falls_back_to_wav_mime_type_when_format_undetectable(
+    mock_detect, mock_get_agent
+):
+    """When filetype detection returns UNK, metadata.filetype falls back to audio/wav."""
+    mock_detect.return_value = FileType.UNK
+    mock_get_agent.return_value.transcribe_segments.return_value = [
+        {"text": "Unknown format.", "start": 0.0, "end": 1.0},
+    ]
+
+    with _tmp_audio() as path:
+        elements = partition_audio(filename=path)
+
+    assert len(elements) == 1
+    assert elements[0].metadata.filetype == FileType.WAV.mime_type  # "audio/wav"
+
+
+# ================================================================================================
+# FileType model
+# ================================================================================================
+
+
+@pytest.mark.parametrize(
+    "file_type",
+    [
+        FileType.FLAC,
+        FileType.M4A,
+        FileType.MP3,
+        FileType.OGG,
+        FileType.OPUS,
+        FileType.WAV,
+        FileType.WEBM,
+    ],
+)
+def test_audio_file_types_are_partitionable(file_type: FileType):
+    assert file_type.is_partitionable
+    assert file_type.partitioner_shortname == "audio"
+    assert file_type.partitioner_function_name == "partition_audio"
+    assert file_type.extra_name == "audio"
+    # STT agent deps are validated at runtime by the chosen agent, not at FileType level
+    assert file_type.importable_package_dependencies == ()
+
+
+# ================================================================================================
+# _audio_suffix helper
+# ================================================================================================
+
+
+@pytest.mark.parametrize(
+    ("metadata_filename", "file_name", "expected"),
+    [
+        ("recording.mp3", None, ".mp3"),
+        ("recording.mp3", "other.wav", ".mp3"),  # metadata_filename wins
+        (None, "audio.flac", ".flac"),
+        (None, None, ".wav"),  # fallback
+        ("noext", None, ".wav"),  # no extension → fallback
+    ],
+)
+def test_audio_suffix(metadata_filename, file_name, expected):
+    from io import BytesIO
+
+    from unstructured.partition.audio import _audio_suffix
+
+    f: IO[bytes] = BytesIO(b"")
+    if file_name is not None:
+        f.name = file_name  # type: ignore[attr-defined]
+    assert _audio_suffix(f, metadata_filename) == expected
+
+
+# ================================================================================================
+# partition_audio parameter forwarding
+# ================================================================================================
+
+
+@patch("unstructured.partition.audio.SpeechToTextAgent.get_agent")
+def test_partition_audio_forwards_custom_stt_agent_to_get_agent(mock_get_agent):
+    mock_get_agent.return_value.transcribe_segments.return_value = _ONE_SEGMENT
+    custom_module = (
+        "unstructured.partition.utils.speech_to_text.whisper_stt.SpeechToTextAgentWhisper"
+    )
+
+    with _tmp_audio() as path:
+        partition_audio(filename=path, stt_agent=custom_module)
+
+    mock_get_agent.assert_called_once_with(custom_module)
+
+
+@patch("unstructured.partition.audio.SpeechToTextAgent.get_agent")
+def test_partition_audio_forwards_language_to_transcribe_segments(mock_get_agent):
+    mock_agent = mock_get_agent.return_value
+    mock_agent.transcribe_segments.return_value = [
+        {"text": "Hola mundo.", "start": 0.0, "end": 1.5},
+    ]
+
+    with _tmp_audio() as path:
+        elements = partition_audio(filename=path, language="es")
+
+    mock_agent.transcribe_segments.assert_called_once_with(path, language="es")
+    assert elements[0].text == "Hola mundo."
+
+
+# ================================================================================================
+# SpeechToTextAgent unit tests
+# ================================================================================================
+
+
+@pytest.mark.usefixtures("_clear_stt_cache")
+class TestSpeechToTextAgentInterface:
+    """Unit tests for the SpeechToTextAgent base class."""
+
+    def test_get_agent_uses_env_config_when_no_module_given(self):
+        import os
+        from unittest.mock import patch as _patch
+
+        from unstructured.partition.utils.speech_to_text.speech_to_text_interface import (
+            SpeechToTextAgent,
+        )
+
+        sentinel = "unstructured.partition.utils.speech_to_text.whisper_stt.SentinelAgent"
+        with (
+            _patch.dict(os.environ, {"STT_AGENT": sentinel}),
+            _patch.object(SpeechToTextAgent, "get_instance") as mock_get_instance,
+        ):
+            SpeechToTextAgent.get_agent(None)
+            mock_get_instance.assert_called_once_with(sentinel)
+
+    def test_get_agent_passes_explicit_module_to_get_instance(self):
+        from unittest.mock import patch as _patch
+
+        from unstructured.partition.utils.speech_to_text.speech_to_text_interface import (
+            SpeechToTextAgent,
+        )
+
+        custom = "unstructured.partition.utils.speech_to_text.whisper_stt.SpeechToTextAgentWhisper"
+        with _patch.object(SpeechToTextAgent, "get_instance") as mock_get_instance:
+            SpeechToTextAgent.get_agent(custom)
+            mock_get_instance.assert_called_once_with(custom)
+
+    def test_get_instance_rejects_non_whitelisted_module(self):
+        from unstructured.partition.utils.speech_to_text.speech_to_text_interface import (
+            SpeechToTextAgent,
+        )
+
+        with pytest.raises(ValueError, match="must be in the whitelist"):
+            SpeechToTextAgent.get_instance("evil.module.EvilAgent")
+
+    def test_get_instance_rejects_whitelisted_module_attribute_that_is_not_a_class(self):
+        """Whitelisted module exposes a non-class (e.g. function or constant) -> TypeError."""
+        from unittest.mock import MagicMock, patch
+
+        from unstructured.partition.utils.speech_to_text.speech_to_text_interface import (
+            SpeechToTextAgent,
+        )
+
+        agent_module = "unstructured.partition.utils.speech_to_text.whisper_stt.some_function"
+        mock_mod = MagicMock()
+        mock_mod.some_function = lambda x: x  # not a class
+
+        with patch(
+            "unstructured.partition.utils.speech_to_text.speech_to_text_interface.importlib.import_module",
+            return_value=mock_mod,
+        ):
+            with pytest.raises(TypeError, match="does not refer to a class"):
+                SpeechToTextAgent.get_instance(agent_module)
+
+    def test_get_instance_rejects_class_that_does_not_subclass_speech_to_text_agent(self):
+        """Whitelisted module exposes a class not subclassing SpeechToTextAgent -> TypeError."""
+        from unittest.mock import MagicMock, patch
+
+        from unstructured.partition.utils.speech_to_text.speech_to_text_interface import (
+            SpeechToTextAgent,
+        )
+
+        agent_module = "unstructured.partition.utils.speech_to_text.whisper_stt.WrongClass"
+        mock_mod = MagicMock()
+        mock_mod.WrongClass = object  # a class, but not a subclass of SpeechToTextAgent
+
+        with patch(
+            "unstructured.partition.utils.speech_to_text.speech_to_text_interface.importlib.import_module",
+            return_value=mock_mod,
+        ):
+            with pytest.raises(TypeError, match="must be a subclass of SpeechToTextAgent"):
+                SpeechToTextAgent.get_instance(agent_module)

test_unstructured/partition/test_constants.py

@@ -0,0 +1,142 @@
+EXPECTED_TABLE = (
+    "<table>"
+    "<tr><td>Stanley Cups</td><td/><td/></tr>"
+    "<tr><td>Team</td><td>Location</td><td>Stanley Cups</td></tr>"
+    "<tr><td>Blues</td><td>STL</td><td>1</td></tr>"
+    "<tr><td>Flyers</td><td>PHI</td><td>2</td></tr>"
+    "<tr><td>Maple Leafs</td><td>TOR</td><td>13</td></tr>"
+    "</table>"
+)
+
+EXPECTED_TABLE_SEMICOLON_DELIMITER = (
+    "<table>"
+    "<tr><td>Year</td><td>Month</td><td>Revenue</td><td>Costs</td><td/></tr>"
+    "<tr><td>2022</td><td>1</td><td>123</td><td>-123</td><td/></tr>"
+    "<tr><td>2023</td><td>2</td><td>143,1</td><td>-814,38</td><td/></tr>"
+    "<tr><td>2024</td><td>3</td><td>215,32</td><td>-11,08</td><td/></tr>"
+    "</table>"
+)
+
+EXPECTED_TABLE_WITH_EMOJI = (
+    "<table>"
+    "<tr><td>Stanley Cups</td><td/><td/></tr>"
+    "<tr><td>Team</td><td>Location</td><td>Stanley Cups</td></tr>"
+    "<tr><td>Blues</td><td>STL</td><td>1</td></tr>"
+    "<tr><td>Flyers</td><td>PHI</td><td>2</td></tr>"
+    "<tr><td>Maple Leafs</td><td>TOR</td><td>13</td></tr>"
+    "<tr><td>👨\\U+1F3FB🔧</td><td>TOR</td><td>15</td></tr>"
+    "</table>"
+)
+
+EXPECTED_TABLE_XLSX = (
+    "<table>"
+    "<tr><td>Team</td><td>Location</td><td>Stanley Cups</td></tr>"
+    "<tr><td>Blues</td><td>STL</td><td>1</td></tr>"
+    "<tr><td>Flyers</td><td>PHI</td><td>2</td></tr>"
+    "<tr><td>Maple Leafs</td><td>TOR</td><td>13</td></tr>"
+    "</table>"
+)
+
+EXPECTED_TABLE_WITH_LINE_DELIMITER = (
+    "<table>"
+    "<tr><td>col1</td><td>col2</td><td>col3</td></tr>"
+    "<tr><td>a</td><td>b</td><td>c</td></tr>"
+    "<tr><td>d</td><td>e</td><td>f</td></tr>"
+    "<tr><td>g</td><td>h</td><td>i</td></tr>"
+    "</table>"
+)
+
+
+EXPECTED_TITLE = "Stanley Cups"
+
+EXPECTED_TEXT = (
+    "Stanley Cups Team Location Stanley Cups Blues STL 1 Flyers PHI 2 Maple Leafs TOR 13"
+)
+
+EXPECTED_TEXT_XLSX = "Team Location Stanley Cups Blues STL 1 Flyers PHI 2 Maple Leafs TOR 13"
+
+EXPECTED_TEXT_WITH_EMOJI = (
+    "Stanley Cups "
+    "Team Location Stanley Cups Blues STL 1 Flyers PHI 2 Maple Leafs TOR 13 👨\\U+1F3FB🔧 TOR 15"
+)
+
+EXPECTED_TEXT_SEMICOLON_DELIMITER = (
+    "Year Month Revenue Costs 2022 1 123 -123 2023 2 143,1 -814,38 2024 3 215,32 -11,08"
+)
+
+EXPECTED_TEXT_WITH_LINE_DELIMITER = "col1 col2 col3 a b c d e f g h i"
+
+EXPECTED_XLS_TABLE = (
+    "<table><tr>"
+    "<td>MC</td>"
+    "<td>What is 2+2?</td>"
+    "<td>4</td>"
+    "<td>correct</td>"
+    "<td>3</td>"
+    "<td>incorrect</td>"
+    "<td/>"
+    "<td/>"
+    "<td/>"
+    "</tr><tr>"  # -----
+    "<td>MA</td>"
+    "<td>What C datatypes are 8 bits? (assume i386)</td>"
+    "<td>int</td>"
+    "<td/>"
+    "<td>float</td>"
+    "<td/>"
+    "<td>double</td>"
+    "<td/>"
+    "<td>char</td>"
+    "</tr><tr>"  # -----
+    "<td>TF</td>"
+    "<td>Bagpipes are awesome.</td>"
+    "<td>true</td>"
+    "<td/>"
+    "<td/>"
+    "<td/>"
+    "<td/>"
+    "<td/>"
+    "<td/>"
+    "</tr><tr>"  # -----
+    "<td>ESS</td>"
+    "<td>How have the original Henry Hornbostel buildings influenced campus architecture and"
+    " design in the last 30 years?</td>"
+    "<td/>"
+    "<td/>"
+    "<td/>"
+    "<td/>"
+    "<td/>"
+    "<td/>"
+    "<td/>"
+    "</tr><tr>"  # -----
+    "<td>ORD</td>"
+    "<td>Rank the following in their order of operation.</td>"
+    "<td>Parentheses</td>"
+    "<td>Exponents</td>"
+    "<td>Division</td>"
+    "<td>Addition</td>"
+    "<td/>"
+    "<td/>"
+    "<td/>"
+    "</tr><tr>"  # -----
+    "<td>FIB</td>"
+    "<td>The student activities fee is</td>"
+    "<td>95</td>"
+    "<td>dollars for students enrolled in</td>"
+    "<td>19</td>"
+    "<td>units or more,</td>"
+    "<td/>"
+    "<td/>"
+    "<td/>"
+    "</tr><tr>"  # -----
+    "<td>MAT</td>"
+    "<td>Match the lower-case greek letter with its capital form.</td>"
+    "<td>λ</td>"
+    "<td>Λ</td>"
+    "<td>α</td>"
+    "<td>γ</td>"
+    "<td>Γ</td>"
+    "<td>φ</td>"
+    "<td>Φ</td>"
+    "</tr></table>"
+)

test_unstructured/partition/test_csv.py

@@ -0,0 +1,329 @@
+# pyright: reportPrivateUsage=false
+
+from __future__ import annotations
+
+import io
+
+import pytest
+from pytest_mock import MockFixture
+
+from test_unstructured.partition.test_constants import (
+    EXPECTED_TABLE,
+    EXPECTED_TABLE_SEMICOLON_DELIMITER,
+    EXPECTED_TABLE_WITH_EMOJI,
+    EXPECTED_TABLE_WITH_LINE_DELIMITER,
+    EXPECTED_TEXT,
+    EXPECTED_TEXT_SEMICOLON_DELIMITER,
+    EXPECTED_TEXT_WITH_EMOJI,
+    EXPECTED_TEXT_WITH_LINE_DELIMITER,
+    EXPECTED_TEXT_XLSX,
+)
+from test_unstructured.unit_utils import (
+    FixtureRequest,
+    Mock,
+    assert_round_trips_through_JSON,
+    example_doc_path,
+    function_mock,
+)
+from unstructured.chunking.title import chunk_by_title
+from unstructured.cleaners.core import clean_extra_whitespace
+from unstructured.documents.elements import Table
+from unstructured.partition.csv import _CsvPartitioningContext, partition_csv
+from unstructured.partition.utils.constants import UNSTRUCTURED_INCLUDE_DEBUG_METADATA
+
+EXPECTED_FILETYPE = "text/csv"
+
+
+@pytest.mark.parametrize(
+    ("filename", "expected_text", "expected_table"),
+    [
+        ("stanley-cups.csv", EXPECTED_TEXT, EXPECTED_TABLE),
+        ("stanley-cups-with-emoji.csv", EXPECTED_TEXT_WITH_EMOJI, EXPECTED_TABLE_WITH_EMOJI),
+        (
+            "table-semicolon-delimiter.csv",
+            EXPECTED_TEXT_SEMICOLON_DELIMITER,
+            EXPECTED_TABLE_SEMICOLON_DELIMITER,
+        ),
+        (
+            "csv-with-line-delimiter.csv",
+            EXPECTED_TEXT_WITH_LINE_DELIMITER,
+            EXPECTED_TABLE_WITH_LINE_DELIMITER,
+        ),
+    ],
+)
+def test_partition_csv_from_filename(filename: str, expected_text: str, expected_table: str):
+    f_path = f"example-docs/{filename}"
+    elements = partition_csv(filename=f_path)
+
+    assert clean_extra_whitespace(elements[0].text) == expected_text
+    assert elements[0].metadata.text_as_html == expected_table
+    assert elements[0].metadata.filetype == EXPECTED_FILETYPE
+    assert elements[0].metadata.filename == filename
+
+
+@pytest.mark.parametrize("infer_table_structure", [True, False])
+def test_partition_csv_from_filename_infer_table_structure(infer_table_structure: bool):
+    f_path = "example-docs/stanley-cups.csv"
+    elements = partition_csv(filename=f_path, infer_table_structure=infer_table_structure)
+
+    table_element_has_text_as_html_field = (
+        hasattr(elements[0].metadata, "text_as_html")
+        and elements[0].metadata.text_as_html is not None
+    )
+    assert table_element_has_text_as_html_field == infer_table_structure
+
+
+def test_partition_csv_from_filename_with_metadata_filename():
+    elements = partition_csv(example_doc_path("stanley-cups.csv"), metadata_filename="test")
+
+    assert clean_extra_whitespace(elements[0].text) == EXPECTED_TEXT
+    assert elements[0].metadata.filename == "test"
+
+
+def test_partition_csv_with_encoding():
+    elements = partition_csv(example_doc_path("stanley-cups-utf-16.csv"), encoding="utf-16")
+
+    assert clean_extra_whitespace(elements[0].text) == EXPECTED_TEXT
+
+
+@pytest.mark.parametrize(
+    ("filename", "expected_text", "expected_table"),
+    [
+        ("stanley-cups.csv", EXPECTED_TEXT, EXPECTED_TABLE),
+        ("stanley-cups-with-emoji.csv", EXPECTED_TEXT_WITH_EMOJI, EXPECTED_TABLE_WITH_EMOJI),
+    ],
+)
+def test_partition_csv_from_file(filename: str, expected_text: str, expected_table: str):
+    f_path = f"example-docs/{filename}"
+    with open(f_path, "rb") as f:
+        elements = partition_csv(file=f)
+    assert clean_extra_whitespace(elements[0].text) == expected_text
+    assert isinstance(elements[0], Table)
+    assert elements[0].metadata.text_as_html == expected_table
+    assert elements[0].metadata.filetype == EXPECTED_FILETYPE
+    assert elements[0].metadata.filename is None
+    if UNSTRUCTURED_INCLUDE_DEBUG_METADATA:
+        assert {element.metadata.detection_origin for element in elements} == {"csv"}
+
+
+def test_partition_csv_from_file_with_metadata_filename():
+    with open(example_doc_path("stanley-cups.csv"), "rb") as f:
+        elements = partition_csv(file=f, metadata_filename="test")
+
+    assert clean_extra_whitespace(elements[0].text) == EXPECTED_TEXT
+    assert elements[0].metadata.filename == "test"
+
+
+# -- .metadata.last_modified ---------------------------------------------------------------------
+
+
+def test_partition_csv_from_file_path_gets_last_modified_from_filesystem(mocker: MockFixture):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.csv.get_last_modified_date",
+        return_value=filesystem_last_modified,
+    )
+
+    elements = partition_csv(example_doc_path("stanley-cups.csv"))
+
+    assert elements[0].metadata.last_modified == filesystem_last_modified
+
+
+def test_partition_csv_from_file_path_prefers_metadata_last_modified(mocker: MockFixture):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    metadata_last_modified = "2020-07-05T09:24:28"
+
+    mocker.patch(
+        "unstructured.partition.csv.get_last_modified_date", return_value=filesystem_last_modified
+    )
+
+    elements = partition_csv(
+        example_doc_path("stanley-cups.csv"), metadata_last_modified=metadata_last_modified
+    )
+
+    assert elements[0].metadata.last_modified == metadata_last_modified
+
+
+def test_partition_csv_from_file_gets_last_modified_None():
+    with open(example_doc_path("stanley-cups.csv"), "rb") as f:
+        elements = partition_csv(file=f)
+
+    assert elements[0].metadata.last_modified is None
+
+
+def test_partition_csv_from_file_prefers_metadata_last_modified():
+    metadata_last_modified = "2020-07-05T09:24:28"
+
+    with open(example_doc_path("stanley-cups.csv"), "rb") as f:
+        elements = partition_csv(file=f, metadata_last_modified=metadata_last_modified)
+
+    assert elements[0].metadata.last_modified == metadata_last_modified
+
+
+# ------------------------------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("filename", ["stanley-cups.csv", "stanley-cups-with-emoji.csv"])
+def test_partition_csv_with_json(filename: str):
+    elements = partition_csv(filename=example_doc_path(filename))
+    assert_round_trips_through_JSON(elements)
+
+
+def test_add_chunking_strategy_to_partition_csv_non_default():
+    filename = "example-docs/stanley-cups.csv"
+
+    elements = partition_csv(filename=filename)
+    chunk_elements = partition_csv(
+        filename,
+        chunking_strategy="by_title",
+        max_characters=9,
+        combine_text_under_n_chars=0,
+        include_header=False,
+    )
+    chunks = chunk_by_title(elements, max_characters=9, combine_text_under_n_chars=0)
+    assert chunk_elements != elements
+    assert chunk_elements == chunks
+
+
+# NOTE (jennings) partition_csv returns a single TableElement per sheet,
+# so leaving off additional tests for multiple languages like the other partitions
+def test_partition_csv_element_metadata_has_languages():
+    filename = "example-docs/stanley-cups.csv"
+    elements = partition_csv(filename=filename, strategy="fast", include_header=False)
+    assert elements[0].metadata.languages == ["eng"]
+
+
+def test_partition_csv_respects_languages_arg():
+    filename = "example-docs/stanley-cups.csv"
+    elements = partition_csv(
+        filename=filename, strategy="fast", languages=["deu"], include_header=False
+    )
+    assert elements[0].metadata.languages == ["deu"]
+
+
+def test_partition_csv_header():
+    elements = partition_csv(
+        example_doc_path("stanley-cups.csv"), strategy="fast", include_header=True
+    )
+
+    table = elements[0]
+    assert table.text == "Stanley Cups Unnamed: 1 Unnamed: 2 " + EXPECTED_TEXT_XLSX
+    assert table.metadata.text_as_html is not None
+
+
+# ================================================================================================
+# UNIT-TESTS
+# ================================================================================================
+
+
+class Describe_CsvPartitioningContext:
+    """Unit-test suite for `unstructured.partition.csv._CsvPartitioningContext`."""
+
+    # -- .load() ------------------------------------------------
+
+    def it_provides_a_validating_alternate_constructor(self):
+        ctx = _CsvPartitioningContext.load(
+            file_path=example_doc_path("stanley-cups.csv"),
+            file=None,
+            encoding=None,
+            include_header=True,
+            infer_table_structure=True,
+        )
+        assert isinstance(ctx, _CsvPartitioningContext)
+
+    def and_the_validating_constructor_raises_on_an_invalid_context(self):
+        with pytest.raises(ValueError, match="either file-path or file-like object must be prov"):
+            _CsvPartitioningContext.load(
+                file_path=None,
+                file=None,
+                encoding=None,
+                include_header=True,
+                infer_table_structure=True,
+            )
+
+    # -- .delimiter ---------------------------------------------
+
+    @pytest.mark.parametrize(
+        "file_name",
+        [
+            "stanley-cups.csv",
+            # -- Issue #2643: previously raised `_csv.Error: Could not determine delimiter` on
+            # -- this file
+            "csv-with-long-lines.csv",
+        ],
+    )
+    def it_auto_detects_the_delimiter_for_a_comma_delimited_CSV_file(self, file_name: str):
+        ctx = _CsvPartitioningContext(example_doc_path(file_name))
+        assert ctx.delimiter == ","
+
+    def and_it_auto_detects_the_delimiter_for_a_semicolon_delimited_CSV_file(self):
+        ctx = _CsvPartitioningContext(example_doc_path("semicolon-delimited.csv"))
+        assert ctx.delimiter == ";"
+
+    def but_it_returns_None_as_the_delimiter_for_a_single_column_CSV_file(self):
+        ctx = _CsvPartitioningContext(example_doc_path("single-column.csv"))
+        assert ctx.delimiter is None
+
+    # -- .header ------------------------------------------------
+
+    @pytest.mark.parametrize(("include_header", "expected_value"), [(False, None), (True, 0)])
+    def it_identifies_the_header_row_based_on_include_header_arg(
+        self, include_header: bool, expected_value: int | None
+    ):
+        assert _CsvPartitioningContext(include_header=include_header).header == expected_value
+
+    # -- .last_modified -----------------------------------------
+
+    def it_gets_last_modified_from_the_filesystem_when_a_path_is_provided(
+        self, get_last_modified_date_: Mock
+    ):
+        filesystem_last_modified = "2024-08-04T02:23:53"
+        get_last_modified_date_.return_value = filesystem_last_modified
+        ctx = _CsvPartitioningContext(file_path="a/b/document.csv")
+
+        last_modified = ctx.last_modified
+
+        get_last_modified_date_.assert_called_once_with("a/b/document.csv")
+        assert last_modified == filesystem_last_modified
+
+    def and_it_falls_back_to_None_for_the_last_modified_date_when_file_path_is_not_provided(self):
+        file = io.BytesIO(b"abcdefg")
+        ctx = _CsvPartitioningContext(file=file)
+
+        last_modified = ctx.last_modified
+
+        assert last_modified is None
+
+    # -- .open() ------------------------------------------------
+
+    def it_provides_transparent_access_to_the_source_file_when_it_is_a_file_like_object(self):
+        with open(example_doc_path("stanley-cups.csv"), "rb") as f:
+            # -- read so file cursor is at end of file --
+            f.read()
+            ctx = _CsvPartitioningContext(file=f)
+            with ctx.open() as file:
+                assert file is f
+                # -- read cursor is reset to 0 on .open() context entry --
+                assert f.tell() == 0
+                assert file.read(14) == b"Stanley Cups,,"
+                assert f.tell() == 14
+
+            # -- and read cursor is reset to 0 on .open() context exit --
+            assert f.tell() == 0
+
+    def it_provides_transparent_access_to_the_source_file_when_it_is_a_file_path(self):
+        ctx = _CsvPartitioningContext(example_doc_path("stanley-cups.csv"))
+        with ctx.open() as file:
+            assert file.read(14) == b"Stanley Cups,,"
+
+    # -- .validate() --------------------------------------------
+
+    def it_raises_when_neither_file_path_nor_file_is_provided(self):
+        with pytest.raises(ValueError, match="either file-path or file-like object must be prov"):
+            _CsvPartitioningContext()._validate()
+
+    # -- fixtures --------------------------------------------------------------------------------
+
+    @pytest.fixture()
+    def get_last_modified_date_(self, request: FixtureRequest) -> Mock:
+        return function_mock(request, "unstructured.partition.csv.get_last_modified_date")

test_unstructured/partition/test_doc.py

@@ -0,0 +1,283 @@
+# pyright: reportPrivateUsage=false
+
+"""Test suite for `unstructured.partition.doc` module."""
+
+from __future__ import annotations
+
+import pathlib
+from typing import Any, Iterator
+
+import pytest
+from pytest_mock import MockFixture
+
+from test_unstructured.unit_utils import (
+    ANY,
+    CaptureFixture,
+    FixtureRequest,
+    assert_round_trips_through_JSON,
+    example_doc_path,
+    method_mock,
+)
+from unstructured.chunking.basic import chunk_elements
+from unstructured.documents.elements import (
+    Address,
+    CompositeElement,
+    Element,
+    ListItem,
+    NarrativeText,
+    Table,
+    TableChunk,
+    Text,
+    Title,
+)
+from unstructured.partition.doc import partition_doc
+from unstructured.partition.docx import partition_docx
+
+
+def test_partition_doc_matches_partition_docx(request: FixtureRequest):
+    doc_file_path = example_doc_path("simple.doc")
+    docx_file_path = example_doc_path("simple.docx")
+
+    assert partition_doc(doc_file_path) == partition_docx(docx_file_path)
+
+
+# -- document-source (file or filename) ----------------------------------------------------------
+
+
+def test_partition_doc_from_filename(expected_elements: list[Element], capsys: CaptureFixture[str]):
+    elements = partition_doc(example_doc_path("simple.doc"))
+
+    assert elements == expected_elements
+    assert all(e.metadata.file_directory == example_doc_path("") for e in elements)
+    assert capsys.readouterr().out == ""
+    assert capsys.readouterr().err == ""
+
+
+def test_partition_doc_from_file_with_libre_office_filter(
+    expected_elements: list[Element], capsys: CaptureFixture[str]
+):
+    with open(example_doc_path("simple.doc"), "rb") as f:
+        elements = partition_doc(file=f, libre_office_filter="MS Word 2007 XML")
+
+    assert elements == expected_elements
+    assert capsys.readouterr().out == ""
+    assert capsys.readouterr().err == ""
+
+
+def test_partition_doc_from_file_with_no_libre_office_filter(
+    expected_elements: list[Element], capsys: CaptureFixture[str]
+):
+    with open(example_doc_path("simple.doc"), "rb") as f:
+        elements = partition_doc(file=f, libre_office_filter=None)
+
+    assert elements == expected_elements
+    assert capsys.readouterr().out == ""
+    assert capsys.readouterr().err == ""
+    assert all(e.metadata.filename is None for e in elements)
+
+
+def test_partition_doc_raises_when_both_a_filename_and_file_are_specified():
+    doc_file_path = example_doc_path("simple.doc")
+
+    with open(doc_file_path, "rb") as f:
+        with pytest.raises(ValueError, match="Exactly one of filename and file must be specified"):
+            partition_doc(filename=doc_file_path, file=f)
+
+
+def test_partition_doc_raises_when_neither_a_file_path_nor_a_file_like_object_are_provided():
+    with pytest.raises(ValueError, match="Exactly one of filename and file must be specified"):
+        partition_doc()
+
+
+def test_partition_raises_with_missing_doc(tmp_path: pathlib.Path):
+    doc_filename = str(tmp_path / "asdf.doc")
+
+    with pytest.raises(ValueError, match="asdf.doc does not exist"):
+        partition_doc(filename=doc_filename)
+
+
+# -- .metadata.filename --------------------------------------------------------------------------
+
+
+def test_partition_doc_from_filename_gets_filename_from_filename_arg():
+    elements = partition_doc(example_doc_path("simple.doc"))
+
+    assert len(elements) > 0
+    assert all(e.metadata.filename == "simple.doc" for e in elements)
+
+
+def test_partition_doc_from_file_gets_filename_None():
+    with open(example_doc_path("simple.doc"), "rb") as f:
+        elements = partition_doc(file=f)
+
+    assert len(elements) > 0
+    assert all(e.metadata.filename is None for e in elements)
+
+
+def test_partition_doc_from_filename_prefers_metadata_filename():
+    elements = partition_doc(example_doc_path("simple.doc"), metadata_filename="test")
+
+    assert len(elements) > 0
+    assert all(element.metadata.filename == "test" for element in elements)
+
+
+def test_partition_doc_from_file_prefers_metadata_filename():
+    with open(example_doc_path("simple.doc"), "rb") as f:
+        elements = partition_doc(file=f, metadata_filename="test")
+
+    assert all(e.metadata.filename == "test" for e in elements)
+
+
+# -- .metadata.filetype --------------------------------------------------------------------------
+
+
+def test_partition_doc_gets_the_DOC_MIME_type_in_metadata_filetype():
+    DOC_MIME_TYPE = "application/msword"
+    elements = partition_doc(example_doc_path("simple.doc"))
+    assert all(e.metadata.filetype == DOC_MIME_TYPE for e in elements), (
+        f"Expected all elements to have '{DOC_MIME_TYPE}' as their filetype, but got:"
+        f" {repr(elements[0].metadata.filetype)}"
+    )
+
+
+# -- .metadata.last_modified ---------------------------------------------------------------------
+
+
+def test_partition_doc_pulls_last_modified_from_filesystem(mocker: MockFixture):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.doc.get_last_modified_date", return_value=filesystem_last_modified
+    )
+
+    elements = partition_doc(example_doc_path("fake.doc"))
+
+    assert all(e.metadata.last_modified == filesystem_last_modified for e in elements)
+
+
+def test_partition_doc_prefers_metadata_last_modified_when_provided(
+    mocker: MockFixture,
+):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    metadata_last_modified = "2020-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.doc.get_last_modified_date", return_value=filesystem_last_modified
+    )
+
+    elements = partition_doc(
+        example_doc_path("simple.doc"), metadata_last_modified=metadata_last_modified
+    )
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+# -- language-recognition metadata ---------------------------------------------------------------
+
+
+def test_partition_doc_adds_languages_metadata():
+    elements = partition_doc(example_doc_path("simple.doc"))
+    assert all(e.metadata.languages == ["eng"] for e in elements)
+
+
+def test_partition_doc_respects_detect_language_per_element_arg():
+    elements = partition_doc(
+        example_doc_path("language-docs/eng_spa_mult.doc"), detect_language_per_element=True
+    )
+    assert [e.metadata.languages for e in elements] == [
+        ["eng"],
+        ["spa", "eng"],
+        ["eng"],
+        ["eng"],
+        ["spa"],
+    ]
+
+
+# -- miscellaneous -------------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("kwargs", "expected_value"),
+    [({}, "hi_res"), ({"strategy": None}, "hi_res"), ({"strategy": "auto"}, "auto")],
+)
+def test_partition_odt_forwards_strategy_arg_to_partition_docx(
+    request: FixtureRequest, kwargs: dict[str, Any], expected_value: str | None
+):
+    from unstructured.partition.docx import _DocxPartitioner
+
+    def fake_iter_document_elements(self: _DocxPartitioner) -> Iterator[Element]:
+        yield Text(f"strategy == {self._opts.strategy}")
+
+    _iter_elements_ = method_mock(
+        request,
+        _DocxPartitioner,
+        "_iter_document_elements",
+        side_effect=fake_iter_document_elements,
+    )
+
+    (element,) = partition_doc(example_doc_path("simple.doc"), **kwargs)
+
+    _iter_elements_.assert_called_once_with(ANY)
+    assert element.text == f"strategy == {expected_value}"
+
+
+def test_partition_doc_grabs_emphasized_texts():
+    expected_emphasized_text_contents = ["bold", "italic", "bold-italic", "bold-italic"]
+    expected_emphasized_text_tags = ["b", "i", "b", "i"]
+
+    elements = partition_doc(example_doc_path("fake-doc-emphasized-text.doc"))
+
+    assert isinstance(elements[0], Table)
+    assert elements[0].metadata.emphasized_text_contents == expected_emphasized_text_contents
+    assert elements[0].metadata.emphasized_text_tags == expected_emphasized_text_tags
+
+    assert elements[1] == NarrativeText("I am a bold italic bold-italic text.")
+    assert elements[1].metadata.emphasized_text_contents == expected_emphasized_text_contents
+    assert elements[1].metadata.emphasized_text_tags == expected_emphasized_text_tags
+
+    assert elements[2] == NarrativeText("I am a normal text.")
+    assert elements[2].metadata.emphasized_text_contents is None
+    assert elements[2].metadata.emphasized_text_tags is None
+
+
+def test_partition_doc_round_trips_through_json():
+    """Elements produced can be serialized then deserialized without loss."""
+    assert_round_trips_through_JSON(partition_doc(example_doc_path("simple.doc")))
+
+
+def test_partition_doc_chunks_elements_when_chunking_strategy_is_specified():
+    document_path = example_doc_path("simple.doc")
+    elements = partition_doc(document_path)
+    chunks = partition_doc(document_path, chunking_strategy="basic")
+
+    # -- all chunks are chunk element-types --
+    assert all(isinstance(c, (CompositeElement, Table, TableChunk)) for c in chunks)
+    # -- chunks from partitioning match those produced by chunking elements in separate step --
+    assert chunks == chunk_elements(elements)
+
+
+def test_partition_doc_assigns_deterministic_and_unique_element_ids():
+    document_path = example_doc_path("duplicate-paragraphs.doc")
+
+    ids = [element.id for element in partition_doc(document_path)]
+    ids_2 = [element.id for element in partition_doc(document_path)]
+
+    # -- ids should match even though partitioned separately --
+    assert ids == ids_2
+    # -- ids should be unique --
+    assert len(ids) == len(set(ids))
+
+
+# == module-level fixtures =======================================================================
+
+
+@pytest.fixture()
+def expected_elements() -> list[Element]:
+    return [
+        Title("These are a few of my favorite things:"),
+        ListItem("Parrots"),
+        ListItem("Hockey"),
+        Text("Analysis"),
+        NarrativeText("This is my first thought. This is my second thought."),
+        NarrativeText("This is my third thought."),
+        Text("2023"),
+        Address("DOYLESTOWN, PA 18901"),
+    ]

test_unstructured/partition/test_email.py

@@ -0,0 +1,636 @@
+"""Test suite for `unstructured.partition.email` module."""
+
+from __future__ import annotations
+
+import io
+import tempfile
+from email.message import EmailMessage
+from typing import Any
+
+import pytest
+
+from test_unstructured.unit_utils import (
+    FixtureRequest,
+    Mock,
+    assert_round_trips_through_JSON,
+    example_doc_path,
+    function_mock,
+)
+from unstructured.chunking.title import chunk_by_title
+from unstructured.documents.elements import (
+    CompositeElement,
+    ListItem,
+    NarrativeText,
+    Table,
+    TableChunk,
+    Text,
+    Title,
+)
+from unstructured.partition.email import EmailPartitioningContext, partition_email
+
+EXPECTED_OUTPUT = [
+    NarrativeText(text="This is a test email to use for unit tests."),
+    Text(text="Important points:"),
+    ListItem(text="Roses are red"),
+    ListItem(text="Violets are blue"),
+]
+
+
+def test_partition_email_from_filename_can_partition_an_RFC_822_email():
+    assert partition_email(example_doc_path("eml/simple-rfc-822.eml")) == [
+        NarrativeText("This is an RFC 822 email message."),
+        NarrativeText(
+            "An RFC 822 message is characterized by its simple, text-based format, which includes"
+            ' a header and a body. The header contains structured fields such as "From", "To",'
+            ' "Date", and "Subject", each followed by a colon and the corresponding information.'
+            " The body follows the header, separated by a blank line, and contains the main"
+            " content of the email."
+        ),
+        NarrativeText(
+            "The structure ensures compatibility and readability across different email systems"
+            " and clients, adhering to the standards set by the Internet Engineering Task Force"
+            " (IETF)."
+        ),
+    ]
+
+
+def test_partition_email_from_file_can_partition_an_email():
+    with open(example_doc_path("eml/fake-email.eml"), "rb") as f:
+        assert partition_email(file=f) == EXPECTED_OUTPUT
+
+
+def test_partition_email_from_spooled_temp_file_can_partition_an_email():
+    with tempfile.SpooledTemporaryFile() as file:
+        with open(example_doc_path("eml/fake-email.eml"), "rb") as f:
+            file.write(f.read())
+        file.seek(0)
+
+        assert partition_email(file=file) == EXPECTED_OUTPUT
+
+
+def test_partition_email_can_partition_an_HTML_only_email_with_Base64_ISO_8859_1_charset():
+    assert partition_email(example_doc_path("eml/mime-html-only.eml")) == [
+        NarrativeText("This is a text/html part."),
+        NarrativeText(
+            "The first emoticon, :) , was proposed by Scott Fahlman in 1982 to indicate just or"
+            " sarcasm in text emails."
+        ),
+        NarrativeText(
+            "Gmail was launched by Google in 2004 with 1 GB of free storage, significantly more"
+            " than what other services offered at the time."
+        ),
+    ]
+
+
+def test_extract_email_from_text_plain_matches_elements_extracted_from_text_html():
+    file_path = example_doc_path("eml/fake-email.eml")
+
+    elements_from_text = partition_email(file_path, content_source="text/plain")
+    elements_from_html = partition_email(file_path, content_source="text/html")
+
+    assert all(e.text == eo.text for e, eo in zip(elements_from_text, EXPECTED_OUTPUT))
+    assert elements_from_html == EXPECTED_OUTPUT
+    assert all(eh.text == et.text for eh, et in zip(elements_from_html, elements_from_text))
+
+
+def test_partition_email_round_trips_via_json():
+    elements = partition_email(example_doc_path("eml/fake-email.eml"))
+    assert_round_trips_through_JSON(elements)
+
+
+# -- transfer-encodings --------------------------------------------------------------------------
+
+
+def test_partition_email_partitions_an_HTML_part_with_Base64_encoded_UTF_8_charset():
+    assert partition_email(example_doc_path("eml/fake-email-b64.eml")) == EXPECTED_OUTPUT
+
+
+def test_partition_email_partitions_a_text_plain_part_with_Base64_encoded_windows_1255_charset():
+    elements = partition_email(
+        example_doc_path("eml/email-no-utf8-2008-07-16.062410.eml"),
+        content_source="text/plain",
+    )
+
+    assert len(elements) == 30
+    assert elements[1].text.startswith("אני חושב שזה לא יהיה מקצועי והוגן שאני אראה לך היכן")
+
+
+def test_partition_email_partitions_an_html_part_with_quoted_printable_encoded_ISO_8859_1_charset():
+    elements = partition_email(
+        example_doc_path("eml/email-no-utf8-2014-03-17.111517.eml"),
+        content_source="text/html",
+        process_attachments=False,
+    )
+
+    assert len(elements) == 1
+    assert isinstance(elements[0], Table)
+    assert elements[0].text.startswith("Slava Gxyzxyz Hi Slava, The password for your Google")
+
+
+# -- edge-cases ----------------------------------------------------------------------------------
+
+
+def test_partition_email_accepts_a_whitespace_only_file():
+    """Should produce no elements but should not raise an exception."""
+    assert partition_email(example_doc_path("eml/empty.eml")) == []
+
+
+def test_partition_email_can_partition_an_empty_email():
+    assert (
+        partition_email(example_doc_path("eml/mime-no-body.eml"), process_attachments=False) == []
+    )
+
+
+def test_partition_email_does_not_break_on_an_encrypted_message():
+    assert (
+        partition_email(example_doc_path("eml/fake-encrypted.eml"), process_attachments=False) == []
+    )
+
+
+def test_partition_email_finds_content_when_it_is_marked_with_content_disposition_inline():
+    elements = partition_email(
+        example_doc_path("eml/email-inline-content-disposition.eml"), process_attachments=False
+    )
+
+    assert len(elements) == 1
+    e = elements[0]
+    assert isinstance(e, Text)
+    assert e.text == "This is a test of inline"
+
+
+def test_partition_email_from_filename_malformed_encoding():
+    elements = partition_email(filename=example_doc_path("eml/fake-email-malformed-encoding.eml"))
+    assert elements == EXPECTED_OUTPUT
+
+
+# -- error behaviors -----------------------------------------------------------------------------
+
+
+def test_partition_email_raises_when_no_message_source_is_specified():
+    with pytest.raises(ValueError, match="no document specified; either a `filename` or `file`"):
+        partition_email()
+
+
+def test_partition_email_raises_with_invalid_content_type():
+    with pytest.raises(ValueError, match="'application/json' is not a valid value for content_s"):
+        partition_email(example_doc_path("eml/fake-email.eml"), content_source="application/json")
+
+
+# -- .metadata -----------------------------------------------------------------------------------
+
+
+def test_partition_email_augments_message_body_elements_with_email_metadata():
+    elements = partition_email(example_doc_path("eml/mime-multi-to-cc-bcc.eml"))
+
+    assert all(
+        e.metadata.bcc_recipient == ["John <john@example.com>", "Mary <mary@example.com>"]
+        for e in elements
+    )
+    assert all(
+        e.metadata.cc_recipient == ["Tom <tom@example.com>", "Alice <alice@example.com>"]
+        for e in elements
+    )
+    assert all(e.metadata.email_message_id == "2143658709@example.com" for e in elements)
+    assert all(e.metadata.sent_from == ["sender@example.com"] for e in elements)
+    assert all(
+        e.metadata.sent_to == ["Bob <bob@example.com>", "Sue <sue@example.com>"] for e in elements
+    )
+    assert all(e.metadata.subject == "Example Plain-Text MIME Message" for e in elements)
+
+
+# -- .metadata.filename --------------------------------------------------------------------------
+
+
+def test_partition_email_from_filename_gets_filename_metadata_from_file_path():
+    elements = partition_email(example_doc_path("eml/fake-email.eml"))
+
+    assert all(e.metadata.filename == "fake-email.eml" for e in elements)
+    assert all(e.metadata.file_directory == example_doc_path("eml") for e in elements)
+
+
+def test_partition_email_from_file_gets_filename_metadata_None():
+    with open(example_doc_path("eml/fake-email.eml"), "rb") as f:
+        elements = partition_email(file=f)
+
+    assert all(e.metadata.filename is None for e in elements)
+    assert all(e.metadata.file_directory is None for e in elements)
+
+
+def test_partition_email_from_filename_prefers_metadata_filename():
+    elements = partition_email(
+        example_doc_path("eml/fake-email.eml"), metadata_filename="a/b/c.eml"
+    )
+
+    assert all(e.metadata.filename == "c.eml" for e in elements)
+    assert all(e.metadata.file_directory == "a/b" for e in elements)
+
+
+def test_partition_email_from_file_prefers_metadata_filename():
+    with open(example_doc_path("eml/fake-email.eml"), "rb") as f:
+        elements = partition_email(file=f, metadata_filename="d/e/f.eml")
+
+    assert all(e.metadata.filename == "f.eml" for e in elements)
+    assert all(e.metadata.file_directory == "d/e" for e in elements)
+
+
+# -- .metadata.filetype --------------------------------------------------------------------------
+
+
+def test_partition_email_gets_the_EML_MIME_type_in_metadata_filetype_for_message_body_elements():
+    EML_MIME_TYPE = "message/rfc822"
+    elements = partition_email(example_doc_path("eml/fake-email.eml"))
+    assert all(e.metadata.filetype == EML_MIME_TYPE for e in elements), (
+        f"Expected all elements to have '{EML_MIME_TYPE}' as their filetype, but got:"
+        f" {repr(elements[0].metadata.filetype)}"
+    )
+
+
+# -- .metadata.languages -------------------------------------------------------------------------
+
+
+def test_partition_email_element_metadata_has_languages():
+    elements = partition_email(example_doc_path("eml/fake-email.eml"))
+    assert all(e.metadata.languages == ["eng"] for e in elements)
+
+
+def test_partition_email_respects_languages_arg():
+    elements = partition_email(example_doc_path("eml/fake-email.eml"), languages=["deu"])
+    assert all(element.metadata.languages == ["deu"] for element in elements)
+
+
+def test_partition_eml_respects_detect_language_per_element():
+    elements = partition_email(
+        example_doc_path("language-docs/eng_spa_mult.eml"),
+        detect_language_per_element=True,
+    )
+    # languages other than English and Spanish are detected by this partitioner,
+    # so this test is slightly different from the other partition tests
+    langs = {e.metadata.languages[0] for e in elements if e.metadata.languages is not None}
+
+    assert "eng" in langs
+    assert "spa" in langs
+
+
+# -- .metadata.last_modified ---------------------------------------------------------------------
+
+
+def test_partition_email_from_file_path_gets_last_modified_from_Date_header():
+    elements = partition_email(example_doc_path("eml/fake-email.eml"))
+    assert all(e.metadata.last_modified == "2022-12-16T22:04:16+00:00" for e in elements)
+
+
+def test_partition_email_from_file_gets_last_modified_from_Date_header():
+    with open(example_doc_path("eml/fake-email.eml"), "rb") as f:
+        elements = partition_email(file=f)
+
+    assert all(e.metadata.last_modified == "2022-12-16T22:04:16+00:00" for e in elements)
+
+
+def test_partition_email_from_file_path_prefers_metadata_last_modified():
+    metadata_last_modified = "2020-07-05T09:24:28"
+
+    elements = partition_email(
+        example_doc_path("eml/fake-email.eml"), metadata_last_modified=metadata_last_modified
+    )
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+def test_partition_email_from_file_prefers_metadata_last_modified():
+    metadata_last_modified = "2020-07-05T09:24:28"
+    with open(example_doc_path("eml/fake-email.eml"), "rb") as f:
+        elements = partition_email(file=f, metadata_last_modified=metadata_last_modified)
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+# -- chunking ------------------------------------------------------------------------------------
+
+
+def test_partition_email_chunks_when_so_instructed():
+    """Note it's actually the delegate partitioners that do the chunking."""
+    elements = partition_email(example_doc_path("eml/fake-email.txt"))
+    chunks = partition_email(example_doc_path("eml/fake-email.txt"), chunking_strategy="by_title")
+    separately_chunked_chunks = chunk_by_title(elements)
+
+    assert all(isinstance(c, (CompositeElement, Table, TableChunk)) for c in chunks)
+    assert chunks != elements
+    assert chunks == separately_chunked_chunks
+
+
+def test_partition_email_chunks_attachments_too():
+    chunks = partition_email(
+        example_doc_path("eml/fake-email-attachment.eml"),
+        chunking_strategy="by_title",
+        process_attachments=True,
+    )
+
+    assert len(chunks) == 2
+    assert all(isinstance(c, CompositeElement) for c in chunks)
+    attachment_chunk = chunks[-1]
+    assert attachment_chunk.text == "Hey this is a fake attachment!"
+    assert attachment_chunk.metadata.filename == "fake-attachment.txt"
+    assert attachment_chunk.metadata.attached_to_filename == "fake-email-attachment.eml"
+    assert all(c.metadata.last_modified == "2022-12-23T18:08:48+00:00" for c in chunks)
+
+
+# -- attachments ---------------------------------------------------------------------------------
+
+
+def test_partition_email_also_partitions_attachments_when_so_instructed():
+    elements = partition_email(
+        example_doc_path("eml/email-equals-attachment-filename.eml"), process_attachments=True
+    )
+
+    assert elements == [
+        NarrativeText("Below is an example of an odd filename"),
+        Title("Odd filename"),
+    ]
+
+
+def test_partition_email_can_process_attachments():
+    elements = partition_email(
+        example_doc_path("eml/fake-email-attachment.eml"), process_attachments=True
+    )
+
+    assert elements == [
+        Text("Hello!"),
+        NarrativeText("Here's the attachments!"),
+        NarrativeText("It includes:"),
+        ListItem("Lots of whitespace"),
+        ListItem("Little to no content"),
+        ListItem("and is a quick read"),
+        Text("Best,"),
+        Text("Mallori"),
+        NarrativeText("Hey this is a fake attachment!"),
+    ]
+    assert all(e.metadata.last_modified == "2022-12-23T18:08:48+00:00" for e in elements)
+    attachment_element = elements[-1]
+    assert attachment_element.text == "Hey this is a fake attachment!"
+    assert attachment_element.metadata.filename == "fake-attachment.txt"
+    assert attachment_element.metadata.attached_to_filename == "fake-email-attachment.eml"
+
+
+def test_partition_email_silently_skips_attachments_it_cannot_partition():
+    """Attachments that raise EXPECTED_ATTACHMENT_ERRORS (e.g. ImportError) are skipped."""
+    from unittest.mock import patch
+
+    with patch("unstructured.partition.auto.partition") as mock_partition:
+        mock_partition.side_effect = ImportError("No module named 'whisper'")
+        elements = partition_email(
+            example_doc_path("eml/mime-attach-mp3.eml"), process_attachments=True
+        )
+
+    # -- No exception; attachment skipped (ImportError is in EXPECTED_ATTACHMENT_ERRORS). --
+    assert elements == [
+        # -- the email body is partitioned --
+        NarrativeText("This is an email with an MP3 attachment."),
+        # -- no elements appear for the attachment --
+    ]
+
+
+# ================================================================================================
+# ISOLATED UNIT TESTS
+# ================================================================================================
+
+
+class DescribeEmailPartitionerOptions:
+    """Unit-test suite for `unstructured.partition.email.EmailPartitioningContext` objects."""
+
+    # -- .load() ---------------------------------
+
+    def it_provides_a_validating_constructor(self, ctx_args: dict[str, Any]):
+        ctx_args["file_path"] = example_doc_path("eml/fake-email.eml")
+
+        ctx = EmailPartitioningContext.load(**ctx_args)
+
+        assert isinstance(ctx, EmailPartitioningContext)
+
+    def but_it_raises_when_no_source_document_was_specified(self, ctx_args: dict[str, Any]):
+        with pytest.raises(ValueError, match="no document specified; either a `filename` or `fi"):
+            EmailPartitioningContext.load(**ctx_args)
+
+    def and_it_raises_when_a_file_open_for_reading_str_is_used(self, ctx_args: dict[str, Any]):
+        ctx_args["file"] = io.StringIO("abcdefg")
+        with pytest.raises(ValueError, match="file object must be opened in binary mode"):
+            EmailPartitioningContext.load(**ctx_args)
+
+    def and_it_raises_when_an_invalid_content_source_is_specified(self, ctx_args: dict[str, Any]):
+        ctx_args["file_path"] = example_doc_path("eml/fake-email.eml")
+        ctx_args["content_source"] = "application/json"
+
+        with pytest.raises(ValueError, match="'application/json' is not a valid value for conte"):
+            EmailPartitioningContext.load(**ctx_args)
+
+    # -- .bcc_addresses --------------------------
+
+    def it_provides_access_to_the_Bcc_addresses_when_present(self):
+        ctx = EmailPartitioningContext(example_doc_path("eml/mime-multi-to-cc-bcc.eml"))
+        assert ctx.bcc_addresses == ["John <john@example.com>", "Mary <mary@example.com>"]
+
+    def but_it_returns_None_when_there_are_no_Bcc_addresses(self):
+        ctx = EmailPartitioningContext(example_doc_path("eml/simple-rfc-822.eml"))
+        assert ctx.bcc_addresses is None
+
+    # -- .body_part ------------------------------
+
+    def it_returns_the_html_body_part_when_there_is_one_by_default(self):
+        ctx = EmailPartitioningContext(example_doc_path("eml/mime-different-plain-html.eml"))
+
+        body_part = ctx.body_part
+
+        assert isinstance(body_part, EmailMessage)
+        content = body_part.get_content()
+        assert isinstance(content, str)
+        assert content.startswith("<!DOCTYPE html>")
+
+    def but_it_returns_the_plain_text_body_part_when_there_is_one_when_so_requested(self):
+        ctx = EmailPartitioningContext(
+            example_doc_path("eml/mime-different-plain-html.eml"), content_source="text/plain"
+        )
+
+        body_part = ctx.body_part
+
+        assert isinstance(body_part, EmailMessage)
+        content = body_part.get_content()
+        assert isinstance(content, str)
+        assert content.startswith("This is the text/plain part.")
+
+    def and_it_returns_None_when_the_email_has_no_body(self):
+        ctx = EmailPartitioningContext(example_doc_path("eml/mime-no-body.eml"))
+        assert ctx.body_part is None
+
+    # -- .cc_addresses ---------------------------
+
+    def it_provides_access_to_the_Cc_addresses_when_present(self):
+        ctx = EmailPartitioningContext(example_doc_path("eml/mime-multi-to-cc-bcc.eml"))
+        assert ctx.cc_addresses == ["Tom <tom@example.com>", "Alice <alice@example.com>"]
+
+    def but_it_returns_None_when_there_are_no_Cc_addresses(self):
+        ctx = EmailPartitioningContext(example_doc_path("eml/simple-rfc-822.eml"))
+        assert ctx.cc_addresses is None
+
+    # -- .content_type_preference ----------------
+
+    @pytest.mark.parametrize(
+        ("content_source", "expected_value"),
+        [
+            ("text/html", ("html", "plain")),
+            ("text/plain", ("plain", "html")),
+        ],
+    )
+    def it_knows_whether_the_caller_prefers_the_HTML_or_plain_text_body(
+        self, content_source: str, expected_value: tuple[str, ...]
+    ):
+        ctx = EmailPartitioningContext(content_source=content_source)
+        assert ctx.content_type_preference == expected_value
+
+    def and_it_defaults_to_preferring_the_HTML_body(self):
+        ctx = EmailPartitioningContext()
+        assert ctx.content_type_preference == ("html", "plain")
+
+    # -- .from -----------------------------------
+
+    def it_knows_the_From_address_of_the_email(self):
+        ctx = EmailPartitioningContext(example_doc_path("eml/mime-simple.eml"))
+        assert ctx.from_address == "sender@example.com"
+
+    # -- .message_id -----------------------------
+
+    def it_provides_access_to_the_Message_ID_when_present(self):
+        ctx = EmailPartitioningContext(example_doc_path("eml/mime-simple.eml"))
+        assert ctx.message_id == "1234567890@example.com"
+
+    def but_it_returns_None_when_there_is_no_Message_ID_header(self):
+        ctx = EmailPartitioningContext(example_doc_path("eml/simple-rfc-822.eml"))
+        assert ctx.message_id is None
+
+    # -- .metadata_file_path ---------------------
+
+    def it_uses_the_metadata_file_path_arg_value_when_one_was_provided(self):
+        ctx = EmailPartitioningContext(metadata_file_path="a/b/c.eml")
+        assert ctx.metadata_file_path == "a/b/c.eml"
+
+    def and_it_uses_the_file_path_arg_value_when_metadata_file_path_was_not_provided(self):
+        ctx = EmailPartitioningContext(file_path="x/y/z.eml")
+        assert ctx.metadata_file_path == "x/y/z.eml"
+
+    def and_it_returns_None_when_neither_file_path_was_provided(self):
+        ctx = EmailPartitioningContext()
+        assert ctx.metadata_file_path is None
+
+    # -- .metadata_last_modified -----------------
+
+    def it_uses_the_metadata_last_modified_arg_value_when_one_was_provided(self):
+        metadata_last_modified = "2023-04-08T12:18:07"
+        ctx = EmailPartitioningContext(metadata_last_modified=metadata_last_modified)
+        assert ctx.metadata_last_modified == metadata_last_modified
+
+    def and_it_uses_the_msg_Date_header_date_when_metadata_last_modified_was_not_provided(
+        self,
+    ):
+        ctx = EmailPartitioningContext(example_doc_path("eml/simple-rfc-822.eml"))
+        assert ctx.metadata_last_modified == "2024-10-01T17:34:56+00:00"
+
+    @pytest.mark.parametrize(
+        ("date_format", "expected_date"),
+        [
+            ("test-iso-8601-date.eml", "2025-07-29T12:42:06+00:00"),
+            ("test-rfc2822-date.eml", "2025-07-29T12:42:06+00:00"),
+        ],
+    )
+    def and_it_correctly_parses_various_date_formats_like_the_ones_that_occur_in_the_wild(
+        self, date_format: str, expected_date: str
+    ):
+        ctx = EmailPartitioningContext(example_doc_path(f"eml/{date_format}"))
+        assert ctx.metadata_last_modified == expected_date
+
+    def and_it_returns_none_when_date_header_is_invalid(self):
+        ctx = EmailPartitioningContext(example_doc_path("eml/test-invalid-date.eml"))
+        assert ctx._sent_date is None
+
+    def and_it_falls_back_to_filesystem_last_modified_when_no_Date_header_is_present(
+        self, get_last_modified_date_: Mock
+    ):
+        """Not an expected case as according to RFC 5322, the Date header is required."""
+        filesystem_last_modified = "2024-07-09T14:08:17"
+        get_last_modified_date_.return_value = filesystem_last_modified
+
+        ctx = EmailPartitioningContext(example_doc_path("eml/rfc822-no-date.eml"))
+
+        assert ctx.metadata_last_modified == filesystem_last_modified
+
+    def and_it_returns_None_when_no_last_modified_is_available(self):
+        with open(example_doc_path("eml/rfc822-no-date.eml"), "rb") as f:
+            ctx = EmailPartitioningContext(file=f)
+            assert ctx.metadata_last_modified is None
+
+    # -- .msg ------------------------------------
+
+    def it_loads_the_email_message_from_the_filesystem_when_a_path_is_provided(self):
+        ctx = EmailPartitioningContext(file_path=example_doc_path("eml/simple-rfc-822.eml"))
+        assert isinstance(ctx.msg, EmailMessage)
+
+    def and_it_loads_the_email_message_from_a_file_like_object_when_one_is_provided(self):
+        with open(example_doc_path("eml/simple-rfc-822.eml"), "rb") as f:
+            ctx = EmailPartitioningContext(file=f)
+            assert isinstance(ctx.msg, EmailMessage)
+
+    # -- .partitioning_kwargs --------------------
+
+    def it_passes_along_the_kwargs_it_received_on_construction(self):
+        kwargs = {"foo": "bar", "baz": "qux"}
+        ctx = EmailPartitioningContext(kwargs=kwargs)
+
+        assert ctx.partitioning_kwargs == kwargs
+
+    # -- .process_attachments --------------------
+
+    @pytest.mark.parametrize("process_attachments", [True, False])
+    def it_knows_whether_the_caller_wants_to_also_partition_attachments(
+        self, process_attachments: bool
+    ):
+        ctx = EmailPartitioningContext(process_attachments=process_attachments)
+        assert ctx.process_attachments == process_attachments
+
+    def but_by_default_it_ignores_attachments(self):
+        ctx = EmailPartitioningContext()
+        assert ctx.process_attachments is False
+
+    # -- .subject --------------------------------
+
+    def it_provides_access_to_the_email_Subject_as_a_string(self):
+        ctx = EmailPartitioningContext(example_doc_path("eml/mime-word-encoded-subject.eml"))
+        assert ctx.subject == "Simple email with ☸☿ Unicode subject"
+
+    def but_it_returns_None_when_there_is_no_Subject_header(self):
+        ctx = EmailPartitioningContext(example_doc_path("eml/mime-no-subject.eml"))
+        assert ctx.subject is None
+
+    # -- .to_addresses ---------------------------
+
+    def it_provides_access_to_the_To_addresses_when_present(self):
+        ctx = EmailPartitioningContext(example_doc_path("eml/mime-multi-to-cc-bcc.eml"))
+        assert ctx.to_addresses == ["Bob <bob@example.com>", "Sue <sue@example.com>"]
+
+    def but_it_returns_None_when_there_are_no_To_addresses(self):
+        ctx = EmailPartitioningContext(example_doc_path("eml/mime-no-to.eml"))
+        assert ctx.to_addresses is None
+
+    # -- fixtures --------------------------------------------------------------------------------
+
+    @pytest.fixture()
+    def ctx_args(self) -> dict[str, Any]:
+        return {
+            "file_path": None,
+            "file": None,
+            "content_source": "text/html",
+            "metadata_file_path": None,
+            "metadata_last_modified": None,
+            "process_attachments": False,
+            "kwargs": {},
+        }
+
+    @pytest.fixture()
+    def get_last_modified_date_(self, request: FixtureRequest) -> Mock:
+        return function_mock(request, "unstructured.partition.email.get_last_modified_date")

test_unstructured/partition/test_epub.py

@@ -0,0 +1,174 @@
+from __future__ import annotations
+
+from pytest_mock import MockFixture
+
+from test_unstructured.unit_utils import assert_round_trips_through_JSON, example_doc_path
+from unstructured.chunking.title import chunk_by_title
+from unstructured.documents.elements import Table, Text
+from unstructured.partition.epub import partition_epub
+from unstructured.partition.utils.constants import UNSTRUCTURED_INCLUDE_DEBUG_METADATA
+
+
+def test_partition_epub_from_filename():
+    elements = partition_epub(example_doc_path("simple.epub"))
+
+    assert len(elements) > 0
+    assert isinstance(elements[0], Text)
+    assert elements[1].text.startswith("a shared culture")
+    if UNSTRUCTURED_INCLUDE_DEBUG_METADATA:
+        assert {element.metadata.detection_origin for element in elements} == {"epub"}
+
+
+def test_partition_epub_from_filename_returns_table_in_elements():
+    elements = partition_epub(example_doc_path("winter-sports.epub"))
+    assert elements[12] == Table(
+        "Contents. List of Illustrations (In certain versions of this etext [in certain\nbrowsers]"
+        " clicking on the image will bring up a larger\nversion.) (etext transcriber's note)"
+    )
+
+
+def test_partition_epub_from_file():
+    with open(example_doc_path("winter-sports.epub"), "rb") as f:
+        elements = partition_epub(file=f)
+
+    assert len(elements) > 0
+    assert elements[2].text.startswith("The Project Gutenberg eBook of Winter Sports")
+
+
+# -- .metadata.filename --------------------------------------------------------------------------
+
+
+def test_partition_epub_from_filename_gets_filename_from_filename_arg():
+    elements = partition_epub(example_doc_path("simple.epub"))
+
+    assert len(elements) > 0
+    assert all(e.metadata.filename == "simple.epub" for e in elements)
+
+
+def test_partition_epub_from_file_gets_filename_None():
+    with open(example_doc_path("simple.epub"), "rb") as f:
+        elements = partition_epub(file=f)
+
+    assert len(elements) > 0
+    assert all(e.metadata.filename is None for e in elements)
+
+
+def test_partition_epub_from_filename_prefers_metadata_filename():
+    elements = partition_epub(example_doc_path("simple.epub"), metadata_filename="orig-name.epub")
+
+    assert len(elements) > 0
+    assert all(element.metadata.filename == "orig-name.epub" for element in elements)
+
+
+def test_partition_epub_from_file_prefers_metadata_filename():
+    with open(example_doc_path("simple.epub"), "rb") as f:
+        elements = partition_epub(file=f, metadata_filename="orig-name.epub")
+
+    assert all(e.metadata.filename == "orig-name.epub" for e in elements)
+
+
+# -- .metadata.filetype --------------------------------------------------------------------------
+
+
+def test_partition_epub_gets_the_EPUB_MIME_type_in_metadata_filetype():
+    EPUB_MIME_TYPE = "application/epub"
+    elements = partition_epub(example_doc_path("simple.epub"))
+    assert all(e.metadata.filetype == EPUB_MIME_TYPE for e in elements), (
+        f"Expected all elements to have '{EPUB_MIME_TYPE}' as their filetype, but got:"
+        f" {repr(elements[0].metadata.filetype)}"
+    )
+
+
+# -- .metadata.last_modified ---------------------------------------------------------------------
+
+
+def test_partition_epub_from_file_path_gets_last_modified_from_filesystem(mocker: MockFixture):
+    filesystem_last_modified = "2024-06-14T16:01:29"
+    mocker.patch(
+        "unstructured.partition.epub.get_last_modified_date", return_value=filesystem_last_modified
+    )
+
+    elements = partition_epub(example_doc_path("winter-sports.epub"))
+
+    assert all(e.metadata.last_modified == filesystem_last_modified for e in elements)
+
+
+def test_partition_epub_from_file_gets_last_modified_None():
+    with open(example_doc_path("simple.epub"), "rb") as f:
+        elements = partition_epub(file=f)
+
+    assert all(e.metadata.last_modified is None for e in elements)
+
+
+def test_partition_epub_from_file_path_prefers_metadata_last_modified(mocker: MockFixture):
+    filesystem_last_modified = "2024-06-14T16:01:29"
+    metadata_last_modified = "2020-03-08T06:10:23"
+    mocker.patch(
+        "unstructured.partition.epub.get_last_modified_date", return_value=filesystem_last_modified
+    )
+
+    elements = partition_epub(
+        example_doc_path("winter-sports.epub"), metadata_last_modified=metadata_last_modified
+    )
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+def test_partition_epub_from_file_prefers_metadata_last_modified():
+    metadata_last_modified = "2020-03-08T06:10:23"
+    with open(example_doc_path("simple.epub"), "rb") as f:
+        elements = partition_epub(file=f, metadata_last_modified=metadata_last_modified)
+
+    assert all(e.metadata.last_modified is metadata_last_modified for e in elements)
+
+
+# ------------------------------------------------------------------------------------------------
+
+
+def test_partition_epub_with_json():
+    filename = "example-docs/winter-sports.epub"
+    elements = partition_epub(filename=filename)
+
+    assert_round_trips_through_JSON(elements)
+
+
+def test_add_chunking_strategy_on_partition_epub():
+    file_path = example_doc_path("winter-sports.epub")
+    elements = partition_epub(file_path)
+    chunk_elements = partition_epub(file_path, chunking_strategy="by_title")
+    chunks = chunk_by_title(elements)
+    assert chunk_elements != elements
+    assert chunk_elements == chunks
+
+
+def test_add_chunking_strategy_on_partition_epub_non_default():
+    file_path = example_doc_path("winter-sports.epub")
+    elements = partition_epub(filename=file_path)
+    chunk_elements = partition_epub(
+        file_path,
+        chunking_strategy="by_title",
+        max_characters=5,
+        new_after_n_chars=5,
+        combine_text_under_n_chars=0,
+    )
+    chunks = chunk_by_title(
+        elements,
+        max_characters=5,
+        new_after_n_chars=5,
+        combine_text_under_n_chars=0,
+    )
+    assert chunk_elements != elements
+    assert chunk_elements == chunks
+
+
+def test_partition_epub_element_metadata_has_languages():
+    filename = example_doc_path("winter-sports.epub")
+    elements = partition_epub(filename=filename)
+    assert elements[0].metadata.languages == ["eng"]
+
+
+def test_partition_epub_respects_detect_language_per_element():
+    filename = "example-docs/language-docs/eng_spa_mult.epub"
+    elements = partition_epub(filename=filename, detect_language_per_element=True)
+    langs = [element.metadata.languages for element in elements]
+    assert langs == [["eng"], ["spa", "eng"], ["eng"], ["eng"], ["spa"]]

test_unstructured/partition/test_json.py

@@ -0,0 +1,313 @@
+"""Test-suite for `unstructured.partition.json` module."""
+
+from __future__ import annotations
+
+import os
+import pathlib
+import tempfile
+
+import pytest
+from pytest_mock import MockFixture
+
+from test_unstructured.unit_utils import example_doc_path
+from unstructured.documents.elements import CompositeElement
+from unstructured.file_utils.model import FileType
+from unstructured.partition.email import partition_email
+from unstructured.partition.html import partition_html
+from unstructured.partition.json import partition_json
+from unstructured.partition.text import partition_text
+from unstructured.partition.xml import partition_xml
+from unstructured.staging.base import elements_to_json
+
+DIRECTORY = pathlib.Path(__file__).parent.resolve()
+
+is_in_docker = os.path.exists("/.dockerenv")
+
+test_files = [
+    "fake-text.txt",
+    "fake-html.html",
+    "eml/fake-email.eml",
+]
+
+is_in_docker = os.path.exists("/.dockerenv")
+
+
+def test_it_chunks_elements_when_a_chunking_strategy_is_specified():
+    chunks = partition_json(
+        "example-docs/spring-weather.html.json", chunking_strategy="basic", max_characters=1500
+    )
+
+    assert len(chunks) == 9
+    assert all(isinstance(ch, CompositeElement) for ch in chunks)
+
+
+@pytest.mark.parametrize("filename", test_files)
+def test_partition_json_from_filename(filename: str):
+    path = example_doc_path(filename)
+    elements = []
+    filetype = FileType.from_extension(os.path.splitext(path)[1])
+    if filetype == FileType.TXT:
+        elements = partition_text(filename=path)
+    if filetype == FileType.HTML:
+        elements = partition_html(filename=path)
+    if filetype == FileType.XML:
+        elements = partition_xml(filename=path)
+    if filetype == FileType.EML:
+        elements = partition_email(filename=path)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        _filename = os.path.basename(filename)
+        test_path = os.path.join(tmpdir, _filename + ".json")
+        elements_to_json(elements, filename=test_path, indent=2)
+        test_elements = partition_json(filename=test_path)
+
+    assert len(elements) > 0
+    assert len(str(elements[0])) > 0
+
+    assert len(elements) == len(test_elements)
+    for i in range(len(elements)):
+        assert elements[i] == test_elements[i]
+        assert elements[i].metadata.filename == filename.split("/")[-1]
+
+
+@pytest.mark.parametrize("filename", test_files)
+def test_partition_json_from_filename_with_metadata_filename(filename: str):
+    path = example_doc_path(filename)
+    elements = []
+    filetype = FileType.from_extension(os.path.splitext(path)[1])
+    if filetype == FileType.TXT:
+        elements = partition_text(filename=path)
+    if filetype == FileType.HTML:
+        elements = partition_html(filename=path)
+    if filetype == FileType.XML:
+        elements = partition_xml(filename=path)
+    if filetype == FileType.EML:
+        elements = partition_email(filename=path)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        _filename = os.path.basename(filename)
+        test_path = os.path.join(tmpdir, _filename + ".json")
+        elements_to_json(elements, filename=test_path, indent=2)
+        test_elements = partition_json(filename=test_path, metadata_filename="test")
+
+    assert len(test_elements) > 0
+    assert len(str(test_elements[0])) > 0
+    assert all(element.metadata.filename == "test" for element in test_elements)
+
+
+@pytest.mark.parametrize("filename", test_files)
+def test_partition_json_from_file(filename: str):
+    path = example_doc_path(filename)
+    elements = []
+    filetype = FileType.from_extension(os.path.splitext(path)[1])
+    if filetype == FileType.TXT:
+        elements = partition_text(filename=path)
+    if filetype == FileType.HTML:
+        elements = partition_html(filename=path)
+    if filetype == FileType.XML:
+        elements = partition_xml(filename=path)
+    if filetype == FileType.EML:
+        elements = partition_email(filename=path)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        _filename = os.path.basename(filename)
+        test_path = os.path.join(tmpdir, _filename + ".json")
+        elements_to_json(elements, filename=test_path, indent=2)
+        with open(test_path, "rb") as f:
+            test_elements = partition_json(file=f)
+
+    assert len(elements) > 0
+    assert len(str(elements[0])) > 0
+    assert len(elements) == len(test_elements)
+    for i in range(len(elements)):
+        assert elements[i] == test_elements[i]
+        assert elements[i].metadata.filename == filename.split("/")[-1]
+
+
+@pytest.mark.parametrize("filename", test_files)
+def test_partition_json_from_file_with_metadata_filename(filename: str):
+    path = example_doc_path(filename)
+    elements = []
+    filetype = FileType.from_extension(os.path.splitext(path)[1])
+    if filetype == FileType.TXT:
+        elements = partition_text(filename=path)
+    if filetype == FileType.HTML:
+        elements = partition_html(filename=path)
+    if filetype == FileType.XML:
+        elements = partition_xml(filename=path)
+    if filetype == FileType.EML:
+        elements = partition_email(filename=path)
+    with tempfile.TemporaryDirectory() as tmpdir:
+        _filename = os.path.basename(filename)
+        test_path = os.path.join(tmpdir, _filename + ".json")
+        elements_to_json(elements, filename=test_path, indent=2)
+        with open(test_path, "rb") as f:
+            test_elements = partition_json(file=f, metadata_filename="test")
+
+    for i in range(len(test_elements)):
+        assert test_elements[i].metadata.filename == "test"
+
+
+@pytest.mark.parametrize("filename", test_files)
+def test_partition_json_from_text(filename: str):
+    path = example_doc_path(filename)
+    elements = []
+    filetype = FileType.from_extension(os.path.splitext(path)[1])
+    if filetype == FileType.TXT:
+        elements = partition_text(filename=path)
+    if filetype == FileType.HTML:
+        elements = partition_html(filename=path)
+    if filetype == FileType.XML:
+        elements = partition_xml(filename=path)
+    if filetype == FileType.EML:
+        elements = partition_email(filename=path)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        _filename = os.path.basename(filename)
+        test_path = os.path.join(tmpdir, _filename + ".json")
+        elements_to_json(elements, filename=test_path, indent=2)
+        with open(test_path) as f:
+            text = f.read()
+        test_elements = partition_json(text=text)
+
+    assert len(elements) > 0
+    assert len(str(elements[0])) > 0
+    assert len(elements) == len(test_elements)
+    for i in range(len(elements)):
+        assert elements[i] == test_elements[i]
+        assert elements[i].metadata.filename == filename.split("/")[-1]
+
+
+def test_partition_json_raises_with_none_specified():
+    with pytest.raises(ValueError):
+        partition_json()
+
+
+def test_partition_json_works_with_empty_string():
+    assert partition_json(text="") == []
+
+
+def test_partition_json_fails_with_empty_item():
+    with pytest.raises(ValueError):
+        partition_json(text="{}")
+
+
+def test_partition_json_works_with_empty_list():
+    assert partition_json(text="[]") == []
+
+
+def test_partition_json_raises_with_too_many_specified():
+    path = example_doc_path("fake-text.txt")
+    elements = []
+    filetype = FileType.from_extension(os.path.splitext(path)[1])
+    if filetype == FileType.TXT:
+        elements = partition_text(filename=path)
+    if filetype == FileType.HTML:
+        elements = partition_html(filename=path)
+    if filetype == FileType.XML:
+        elements = partition_xml(filename=path)
+    if filetype == FileType.EML:
+        elements = partition_email(filename=path)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        test_path = os.path.join(tmpdir, "fake-text.txt.json")
+        elements_to_json(elements, filename=test_path, indent=2)
+        with open(test_path, "rb") as f:
+            text = f.read().decode("utf-8")
+
+    with pytest.raises(ValueError):
+        partition_json(filename=test_path, file=f)
+
+    with pytest.raises(ValueError):
+        partition_json(filename=test_path, text=text)
+
+    with pytest.raises(ValueError):
+        partition_json(file=f, text=text)
+
+    with pytest.raises(ValueError):
+        partition_json(filename=test_path, file=f, text=text)
+
+
+# -- .metadata.last_modified ---------------------------------------------------------------------
+
+
+def test_partition_json_from_file_path_gets_last_modified_from_filesystem(mocker: MockFixture):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.json.get_last_modified_date", return_value=filesystem_last_modified
+    )
+
+    elements = partition_json(example_doc_path("spring-weather.html.json"))
+
+    assert all(e.metadata.last_modified == filesystem_last_modified for e in elements)
+
+
+def test_partition_json_from_file_gets_last_modified_None():
+    with open("example-docs/spring-weather.html.json", "rb") as f:
+        elements = partition_json(file=f)
+
+    assert all(e.metadata.last_modified is None for e in elements)
+
+
+def test_partition_json_from_text_gets_last_modified_None():
+    with open("example-docs/spring-weather.html.json") as f:
+        text = f.read()
+
+    elements = partition_json(text=text)
+
+    assert all(e.metadata.last_modified is None for e in elements)
+
+
+def test_partition_json_from_file_path_prefers_metadata_last_modified(mocker: MockFixture):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    metadata_last_modified = "2020-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.json.get_last_modified_date", return_value=filesystem_last_modified
+    )
+
+    elements = partition_json(
+        "example-docs/spring-weather.html.json", metadata_last_modified=metadata_last_modified
+    )
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+def test_partition_json_from_file_prefers_metadata_last_modified():
+    metadata_last_modified = "2020-07-05T09:24:28"
+    with open(example_doc_path("spring-weather.html.json"), "rb") as f:
+        elements = partition_json(file=f, metadata_last_modified=metadata_last_modified)
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+def test_partition_json_from_text_prefers_metadata_last_modified():
+    metadata_last_modified = "2020-07-05T09:24:28"
+    with open("example-docs/spring-weather.html.json") as f:
+        text = f.read()
+
+    elements = partition_json(text=text, metadata_last_modified=metadata_last_modified)
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+# ------------------------------------------------------------------------------------------------
+
+
+def test_partition_json_raises_with_unprocessable_json_array():
+    text = '[{"invalid": "schema"}]'
+    with pytest.raises(ValueError):
+        partition_json(text=text)
+
+
+def test_partition_json_raises_with_unprocessable_json():
+    # NOTE(robinson) - This is unprocessable because it is not a list of dicts,
+    # per the Unstructured ISD format
+    text = '{"hi": "there"}'
+    with pytest.raises(ValueError):
+        partition_json(text=text)
+
+
+def test_partition_json_raises_with_invalid_json():
+    text = '[{"hi": "there"}]]'
+    with pytest.raises(ValueError):
+        partition_json(text=text)

test_unstructured/partition/test_md.py

@@ -0,0 +1,399 @@
+from __future__ import annotations
+
+from typing import Any
+from unittest.mock import patch
+
+import pytest
+from markdown.extensions.fenced_code import FencedCodeExtension
+from pytest_mock import MockFixture
+
+from test_unstructured.unit_utils import assert_round_trips_through_JSON, example_doc_path
+from unstructured.chunking.title import chunk_by_title
+from unstructured.documents.elements import ElementType, Title
+from unstructured.partition.md import partition_md
+from unstructured.partition.utils.constants import UNSTRUCTURED_INCLUDE_DEBUG_METADATA
+
+
+def test_partition_md_from_filename():
+    filename = example_doc_path("README.md")
+    elements = partition_md(filename=filename)
+
+    assert len(elements) > 0
+    assert "PageBreak" not in [elem.category for elem in elements]
+    assert isinstance(elements[0], Title)
+    if UNSTRUCTURED_INCLUDE_DEBUG_METADATA:
+        assert {element.metadata.detection_origin for element in elements} == {"md"}
+
+
+def test_partition_md_from_file():
+    filename = example_doc_path("README.md")
+    with open(filename, "rb") as f:
+        elements = partition_md(file=f)
+    assert len(elements) > 0
+
+
+def test_partition_md_from_text():
+    with open(example_doc_path("README.md")) as f:
+        text = f.read()
+
+    elements = partition_md(text=text)
+
+    assert len(elements) > 0
+    assert all(e.metadata.filename is None for e in elements)
+
+
+class MockResponse:
+    def __init__(self, text: str, status_code: int, headers: dict[str, Any] = {}):
+        self.text = text
+        self.status_code = status_code
+        self.ok = status_code < 300
+        self.headers = headers
+
+
+def test_partition_md_from_url():
+    filename = example_doc_path("README.md")
+    with open(filename) as f:
+        text = f.read()
+
+    response = MockResponse(
+        text=text,
+        status_code=200,
+        headers={"Content-Type": "text/markdown"},
+    )
+    with patch("unstructured.partition.md.safe_get", return_value=response) as _:
+        elements = partition_md(url="https://fake.url")
+
+    assert len(elements) > 0
+    assert all(e.metadata.filename is None for e in elements)
+
+
+def test_partition_md_from_url_raises_with_bad_status_code():
+    filename = example_doc_path("README.md")
+    with open(filename) as f:
+        text = f.read()
+
+    response = MockResponse(
+        text=text,
+        status_code=500,
+        headers={"Content-Type": "text/html"},
+    )
+    with (
+        patch("unstructured.partition.md.safe_get", return_value=response) as _,
+        pytest.raises(ValueError),
+    ):
+        partition_md(url="https://fake.url")
+
+
+def test_partition_md_from_url_raises_with_bad_content_type():
+    filename = example_doc_path("README.md")
+    with open(filename) as f:
+        text = f.read()
+
+    response = MockResponse(
+        text=text,
+        status_code=200,
+        headers={"Content-Type": "application/json"},
+    )
+    with (
+        patch("unstructured.partition.md.safe_get", return_value=response) as _,
+        pytest.raises(ValueError),
+    ):
+        partition_md(url="https://fake.url")
+
+
+def test_partition_md_raises_with_none_specified():
+    with pytest.raises(ValueError):
+        partition_md()
+
+
+def test_partition_md_raises_with_too_many_specified():
+    filename = example_doc_path("README.md")
+    with open(filename) as f:
+        text = f.read()
+
+    with pytest.raises(ValueError):
+        partition_md(filename=filename, text=text)
+
+
+# -- .metadata.filename --------------------------------------------------------------------------
+
+
+def test_partition_md_from_filename_gets_filename_from_filename_arg():
+    elements = partition_md(example_doc_path("README.md"))
+
+    assert len(elements) > 0
+    assert all(e.metadata.filename == "README.md" for e in elements)
+
+
+def test_partition_md_from_file_gets_filename_None():
+    with open(example_doc_path("README.md"), "rb") as f:
+        elements = partition_md(file=f)
+
+    assert len(elements) > 0
+    assert all(e.metadata.filename is None for e in elements)
+
+
+def test_partition_md_from_filename_prefers_metadata_filename():
+    elements = partition_md(example_doc_path("README.md"), metadata_filename="orig-name.md")
+
+    assert len(elements) > 0
+    assert all(element.metadata.filename == "orig-name.md" for element in elements)
+
+
+def test_partition_md_from_file_prefers_metadata_filename():
+    with open(example_doc_path("README.md"), "rb") as f:
+        elements = partition_md(file=f, metadata_filename="orig-name.md")
+
+    assert all(e.metadata.filename == "orig-name.md" for e in elements)
+
+
+# -- .metadata.filetype --------------------------------------------------------------------------
+
+
+def test_partition_md_gets_the_MD_MIME_type_in_metadata_filetype():
+    MD_MIME_TYPE = "text/markdown"
+    elements = partition_md(example_doc_path("README.md"))
+    assert all(e.metadata.filetype == MD_MIME_TYPE for e in elements), (
+        f"Expected all elements to have '{MD_MIME_TYPE}' as their filetype, but got:"
+        f" {repr(elements[0].metadata.filetype)}"
+    )
+
+
+# -- .metadata.last_modified ---------------------------------------------------------------------
+
+
+def test_partition_md_from_file_path_gets_last_modified_from_filesystem(mocker: MockFixture):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.md.get_last_modified_date", return_value=filesystem_last_modified
+    )
+
+    elements = partition_md(example_doc_path("README.md"))
+
+    assert all(e.metadata.last_modified == filesystem_last_modified for e in elements)
+
+
+def test_partition_md_from_file_gets_last_modified_None():
+    with open(example_doc_path("README.md"), "rb") as f:
+        elements = partition_md(file=f)
+
+    assert all(e.metadata.last_modified is None for e in elements)
+
+
+def test_partition_md_from_text_gets_last_modified_None():
+    with open(example_doc_path("README.md")) as f:
+        text = f.read()
+
+    elements = partition_md(text=text)
+
+    assert all(e.metadata.last_modified is None for e in elements)
+
+
+def test_partition_md_from_file_path_prefers_metadata_last_modified(mocker: MockFixture):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    metadata_last_modified = "2020-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.md.get_last_modified_date", return_value=filesystem_last_modified
+    )
+
+    elements = partition_md(
+        example_doc_path("README.md"), metadata_last_modified=metadata_last_modified
+    )
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+def test_partition_md_from_file_prefers_metadata_last_modified():
+    metadata_last_modified = "2020-07-05T09:24:28"
+    with open(example_doc_path("README.md"), "rb") as f:
+        elements = partition_md(file=f, metadata_last_modified=metadata_last_modified)
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+def test_partition_md_from_text_prefers_metadata_last_modified():
+    metadata_last_modified = "2020-07-05T09:24:28"
+    with open(example_doc_path("README.md")) as f:
+        text = f.read()
+
+    elements = partition_md(text=text, metadata_last_modified=metadata_last_modified)
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+# ------------------------------------------------------------------------------------------------
+
+
+def test_partition_md_with_json():
+    with open(example_doc_path("README.md")) as f:
+        text = f.read()
+    elements = partition_md(text=text)
+    assert_round_trips_through_JSON(elements)
+
+
+def test_add_chunking_strategy_by_title_on_partition_md():
+    filename = example_doc_path("README.md")
+    elements = partition_md(filename)
+    chunk_elements = partition_md(filename, chunking_strategy="by_title")
+    chunks = chunk_by_title(elements)
+
+    assert chunk_elements != elements
+    assert chunk_elements == chunks
+
+
+def test_partition_md_element_metadata_has_languages():
+    filename = "example-docs/README.md"
+    elements = partition_md(filename=filename)
+    assert elements[0].metadata.languages == ["eng"]
+
+
+def test_partition_md_respects_detect_language_per_element():
+    filename = "example-docs/language-docs/eng_spa_mult.md"
+    elements = partition_md(filename=filename, detect_language_per_element=True)
+    langs = [element.metadata.languages for element in elements]
+    assert langs == [["eng"], ["spa", "eng"], ["eng"], ["eng"], ["spa"]]
+
+
+def test_partition_md_languages_empty_disables_detection():
+    """Passing languages=[\"\"] disables language detection (no metadata.languages set)."""
+    filename = "example-docs/README.md"
+    elements = partition_md(filename=filename, languages=[""])
+    # When detection is disabled, metadata.languages should not be set (None)
+    for el in elements:
+        assert el.metadata.languages is None
+
+
+def test_partition_md_parse_table():
+    filename = example_doc_path("simple-table.md")
+    elements = partition_md(filename=filename)
+    assert len(elements) > 0
+    assert elements[0].category == ElementType.TABLE
+
+
+@pytest.mark.parametrize("filename", ["umlauts-utf8.md", "umlauts-non-utf8.md"])
+def test_partition_md_with_umlauts(filename: str):
+    filename = example_doc_path(filename)
+    elements = partition_md(filename=filename)
+    assert len(elements) > 0
+    assert elements[-1].text.endswith("äöüß")
+
+
+def test_partition_md_xml_processing_instruction():
+    xml_content = """```
+<?xml version="1.0"?>
+<sparql xmlns="http://www.w3.org/2005/sparql-results#">
+  <head></head>
+  <boolean>true</boolean>
+</sparql>
+```"""
+
+    elements = partition_md(text=xml_content)
+    assert len(elements) == 1
+
+
+def test_partition_md_xml_processing_instruction_with_indents():
+    xml_content = """```
+  <?xml version="1.0"?>
+<sparql xmlns="http://www.w3.org/2005/sparql-results#">
+  <head></head>
+  <boolean>true</boolean>
+</sparql>
+```"""
+
+    elements = partition_md(text=xml_content)
+    assert len(elements) == 1
+
+
+def test_partition_md_non_xml_processing_instruction():
+    php_content = """```
+    <?php echo "hello"; ?>
+    ```"""
+
+    elements = partition_md(text=php_content)
+    assert len(elements) == 1
+
+
+def test_partition_fenced_code():
+    filename = example_doc_path("codeblock.md")
+    elements = partition_md(filename=filename)
+
+    # Should have 5 elements: 2 titles and 3 code blocks
+    assert len(elements) == 5
+
+    assert elements[0].text == "HTML Example"
+
+    expected_html = """<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>Sample HTML</title>
+</head>
+<body>
+    <h1>Hello, World!</h1>
+    <p>This is a simple HTML example.</p>
+</body>
+</html>"""
+    assert elements[1].text == expected_html
+
+    assert elements[2].text == "XML Example"
+
+    expected_xml = """<note>
+    <to>Tove</to>
+    <from>Jani</from>
+    <heading>Reminder</heading>
+    <body>Don't forget me this weekend!</body>
+</note>"""
+    assert elements[3].text == expected_xml
+
+    assert elements[4].text == expected_xml
+
+
+def test_partition_md_custom_extensions_parameter():
+    """User can override markdown extensions via `extensions` kwarg (fixes #4006)."""
+    text = """```bash
+# create the container
+docker run -dt --name unstructured downloads.unstructured.io/unstructured-io/unstructured:latest
+```"""
+
+    expected_body = (
+        "# create the container\n"
+        "docker run -dt --name unstructured "
+        "downloads.unstructured.io/unstructured-io/unstructured:latest"
+    )
+
+    # Without fenced_code, ``#`` inside the fence is parsed as a heading (undesired).
+    elements_tables_only = partition_md(text=text, extensions=["tables"])
+    assert any(isinstance(el, Title) for el in elements_tables_only)
+
+    # Default and explicit fenced_code keep the block as one element (CodeSnippet from HTML).
+    assert len(partition_md(text=text)) == 1
+    elements_fenced = partition_md(text=text, extensions=["fenced_code"])
+    assert len(elements_fenced) == 1
+    assert elements_fenced[0].category == ElementType.CODE_SNIPPET
+    assert elements_fenced[0].text == expected_body
+
+    # Extension instances (normal Python-Markdown API) match string extension names.
+    elements_instance = partition_md(text=text, extensions=[FencedCodeExtension()])
+    assert elements_instance == elements_fenced
+
+
+def test_partition_md_extensions_not_list_raises():
+    with pytest.raises(ValueError, match="'extensions' must be a list"):
+        partition_md(text="# Hi", extensions=("tables",))  # type: ignore[arg-type]
+
+
+def test_partition_md_extensions_invalid_item_raises():
+    with pytest.raises(ValueError, match="Each entry in 'extensions'"):
+        partition_md(text="# Hi", extensions=[42])  # type: ignore[list-item]
+
+
+def test_partition_md_tables_only_differs_from_default_for_code_fence():
+    """Without ``fenced_code``, ``#`` inside a fence can become a Title (see #4006)."""
+    text = """```bash
+# line
+```"""
+    default_el = partition_md(text=text)[0]
+    tables_only_els = partition_md(text=text, extensions=["tables"])
+    assert default_el.category == ElementType.CODE_SNIPPET
+    assert any(e.category == ElementType.TITLE for e in tables_only_els)

test_unstructured/partition/test_msg.py

@@ -0,0 +1,614 @@
+"""Test suite for `unstructured.partition.msg` module."""
+
+from __future__ import annotations
+
+import io
+from typing import Any
+
+import pytest
+from oxmsg import Message
+
+from test_unstructured.unit_utils import (
+    FixtureRequest,
+    LogCaptureFixture,
+    Mock,
+    assert_round_trips_through_JSON,
+    example_doc_path,
+    function_mock,
+    property_mock,
+)
+from unstructured.chunking.title import chunk_by_title
+from unstructured.documents.elements import (
+    ElementMetadata,
+    ListItem,
+    NarrativeText,
+    Text,
+)
+from unstructured.partition.common import UnsupportedFileFormatError
+from unstructured.partition.msg import MsgPartitionerOptions, partition_msg
+
+EXPECTED_MSG_OUTPUT = [
+    NarrativeText(text="This is a test email to use for unit tests."),
+    Text(text="Important points:"),
+    ListItem(text="Roses are red"),
+    ListItem(text="Violets are blue"),
+]
+
+
+def test_partition_msg_from_filename():
+    filename = example_doc_path("fake-email.msg")
+    elements = partition_msg(filename=filename)
+    parent_id = elements[0].metadata.parent_id
+
+    assert elements == EXPECTED_MSG_OUTPUT
+    assert (
+        elements[0].metadata.to_dict()
+        == ElementMetadata(
+            coordinates=None,
+            filename=filename,
+            last_modified="2023-03-28T17:00:31+00:00",
+            page_number=None,
+            url=None,
+            sent_from=['"Matthew Robinson" <mrobinson@unstructured.io>'],
+            sent_to=["mrobinson@unstructured.io"],
+            subject="Test Email",
+            filetype="application/vnd.ms-outlook",
+            parent_id=parent_id,
+            languages=["eng"],
+        ).to_dict()
+    )
+
+
+def test_partition_msg_from_filename_returns_uns_elements():
+    filename = example_doc_path("fake-email.msg")
+    elements = partition_msg(filename=filename)
+    assert isinstance(elements[0], NarrativeText)
+
+
+def test_partition_msg_from_filename_with_metadata_filename():
+    filename = example_doc_path("fake-email.msg")
+    elements = partition_msg(filename=filename, metadata_filename="test")
+    assert all(element.metadata.filename == "test" for element in elements)
+
+
+def test_partition_msg_from_filename_with_text_content():
+    filename = example_doc_path("fake-email.msg")
+
+    elements = partition_msg(filename=filename)
+
+    assert str(elements[0]) == "This is a test email to use for unit tests."
+    assert elements[0].metadata.filename == "fake-email.msg"
+    assert elements[0].metadata.file_directory == example_doc_path("")
+
+
+def test_partition_msg_raises_with_missing_file():
+    filename = example_doc_path("doesnt-exist.msg")
+    with pytest.raises(FileNotFoundError):
+        partition_msg(filename=filename)
+
+
+def test_partition_msg_from_file():
+    filename = example_doc_path("fake-email.msg")
+    with open(filename, "rb") as f:
+        elements = partition_msg(file=f)
+    assert elements == EXPECTED_MSG_OUTPUT
+    for element in elements:
+        assert element.metadata.filename is None
+
+
+def test_partition_msg_from_file_with_metadata_filename():
+    filename = example_doc_path("fake-email.msg")
+    with open(filename, "rb") as f:
+        elements = partition_msg(file=f, metadata_filename="test")
+    assert elements == EXPECTED_MSG_OUTPUT
+    for element in elements:
+        assert element.metadata.filename == "test"
+
+
+def test_partition_msg_uses_file_path_when_both_are_specified():
+    elements = partition_msg(example_doc_path("fake-email.msg"), file=io.BytesIO(b"abcde"))
+    assert elements == EXPECTED_MSG_OUTPUT
+
+
+def test_partition_msg_raises_with_neither():
+    with pytest.raises(ValueError):
+        partition_msg()
+
+
+# -- attachments ---------------------------------------------------------------------------------
+
+
+def test_partition_msg_can_process_attachments():
+    elements = partition_msg(
+        example_doc_path("fake-email-multiple-attachments.msg"), process_attachments=True
+    )
+
+    assert all(e.metadata.filename == "fake-email-multiple-attachments.msg" for e in elements[:5])
+    assert all(e.metadata.filename == "unstructured_logo.png" for e in elements[5:7])
+    assert all(e.metadata.filename == "dense_doc.pdf" for e in elements[7:343])
+    assert all(e.metadata.filename == "Engineering Onboarding.pptx" for e in elements[343:])
+    assert [e.text for e in elements[:5]] == [
+        "Here are those documents.",
+        "--",
+        "Mallori Harrell",
+        "Unstructured Technologies",
+        "Data Scientist",
+    ]
+    assert [type(e).__name__ for e in elements][:10] == [
+        "NarrativeText",
+        "Text",
+        "Text",
+        "Text",
+        "Text",
+        "Image",
+        "Text",
+        "Text",
+        "Title",
+        "Title",
+    ]
+    assert [type(e).__name__ for e in elements][-10:] == [
+        "Title",
+        "ListItem",
+        "ListItem",
+        "ListItem",
+        "ListItem",
+        "ListItem",
+        "ListItem",
+        "ListItem",
+        "ListItem",
+        "ListItem",
+    ]
+
+
+def test_partition_msg_silently_skips_attachments_it_cannot_partition(request: FixtureRequest):
+    function_mock(
+        request, "unstructured.partition.auto.partition", side_effect=UnsupportedFileFormatError()
+    )
+
+    elements = partition_msg(
+        example_doc_path("fake-email-multiple-attachments.msg"), process_attachments=True
+    )
+
+    # -- no exception is raised --
+    assert elements == [
+        # -- the email body is partitioned --
+        NarrativeText("Here are those documents."),
+        Text("--"),
+        Text("Mallori Harrell"),
+        Text("Unstructured Technologies"),
+        Text("Data Scientist"),
+        # -- no elements appear for the attachment(s) --
+    ]
+
+
+# -- .metadata.filename --------------------------------------------------------------------------
+
+
+def test_partition_msg_from_filename_gets_filename_metadata_from_file_path():
+    elements = partition_msg(example_doc_path("fake-email.msg"))
+
+    assert all(e.metadata.filename == "fake-email.msg" for e in elements)
+    assert all(e.metadata.file_directory == example_doc_path("") for e in elements)
+
+
+def test_partition_msg_from_file_gets_filename_metadata_None():
+    with open(example_doc_path("fake-email.msg"), "rb") as f:
+        elements = partition_msg(file=f)
+
+    assert all(e.metadata.filename is None for e in elements)
+    assert all(e.metadata.file_directory is None for e in elements)
+
+
+def test_partition_msg_from_filename_prefers_metadata_filename():
+    elements = partition_msg(example_doc_path("fake-email.msg"), metadata_filename="a/b/c.msg")
+
+    assert all(e.metadata.filename == "c.msg" for e in elements)
+    assert all(e.metadata.file_directory == "a/b" for e in elements)
+
+
+def test_partition_msg_from_file_prefers_metadata_filename():
+    with open(example_doc_path("fake-email.msg"), "rb") as f:
+        elements = partition_msg(file=f, metadata_filename="d/e/f.msg")
+
+    assert all(e.metadata.filename == "f.msg" for e in elements)
+    assert all(e.metadata.file_directory == "d/e" for e in elements)
+
+
+# -- .metadata.filetype --------------------------------------------------------------------------
+
+
+def test_partition_msg_gets_the_MSG_mime_type_in_metadata_filetype():
+    MSG_MIME_TYPE = "application/vnd.ms-outlook"
+    elements = partition_msg(example_doc_path("fake-email.msg"))
+    assert all(e.metadata.filetype == MSG_MIME_TYPE for e in elements), (
+        f"Expected all elements to have '{MSG_MIME_TYPE}' as their filetype, but got:"
+        f" {repr(elements[0].metadata.filetype)}"
+    )
+
+
+# -- .metadata.last_modified ---------------------------------------------------------------------
+
+
+def test_partition_msg_pulls_last_modified_from_message_sent_date():
+    elements = partition_msg(example_doc_path("fake-email.msg"))
+    assert all(e.metadata.last_modified == "2023-03-28T17:00:31+00:00" for e in elements)
+
+
+def test_partition_msg_from_file_path_prefers_metadata_last_modified():
+    metadata_last_modified = "2020-07-05T09:24:28"
+
+    elements = partition_msg(
+        example_doc_path("fake-email.msg"), metadata_last_modified=metadata_last_modified
+    )
+
+    assert elements[0].metadata.last_modified == metadata_last_modified
+
+
+def test_partition_msg_from_file_prefers_metadata_last_modified():
+    metadata_last_modified = "2020-07-05T09:24:28"
+
+    with open(example_doc_path("fake-email.msg"), "rb") as f:
+        elements = partition_msg(file=f, metadata_last_modified=metadata_last_modified)
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+# ------------------------------------------------------------------------------------------------
+
+
+def test_partition_msg_with_json():
+    elements = partition_msg(example_doc_path("fake-email.msg"))
+    assert_round_trips_through_JSON(elements)
+
+
+def test_partition_msg_with_pgp_encrypted_message(caplog: LogCaptureFixture):
+    elements = partition_msg(example_doc_path("fake-encrypted.msg"))
+
+    assert elements == []
+    assert "WARNING" in caplog.text
+    assert "Encrypted email detected" in caplog.text
+
+
+def test_add_chunking_strategy_by_title_on_partition_msg():
+    filename = example_doc_path("fake-email.msg")
+
+    elements = partition_msg(filename=filename)
+    chunk_elements = partition_msg(filename, chunking_strategy="by_title")
+    chunks = chunk_by_title(elements)
+
+    assert chunk_elements != elements
+    assert chunk_elements == chunks
+
+
+# -- language behaviors --------------------------------------------------------------------------
+
+
+def test_partition_msg_element_metadata_has_languages():
+    filename = "example-docs/fake-email.msg"
+    elements = partition_msg(filename=filename)
+    assert elements[0].metadata.languages == ["eng"]
+
+
+def test_partition_msg_respects_languages_arg():
+    filename = "example-docs/fake-email.msg"
+    elements = partition_msg(filename=filename, languages=["deu"])
+    assert all(element.metadata.languages == ["deu"] for element in elements)
+
+
+def test_partition_msg_raises_TypeError_for_invalid_languages():
+    with pytest.raises(TypeError):
+        filename = "example-docs/fake-email.msg"
+        partition_msg(filename=filename, languages="eng")
+
+
+# ================================================================================================
+# ISOLATED UNIT TESTS
+# ================================================================================================
+# These test components used by `partition_msg()` in isolation such that all edge cases can be
+# exercised.
+# ================================================================================================
+
+
+class DescribeMsgAttachmentFilenameSanitization:
+    """Unit-test suite for filename sanitization in MSG attachments (GHSA-gm8q-m8mv-jj5m)."""
+
+    def it_sanitizes_path_traversal_attempts(self, request: FixtureRequest):
+        from unstructured.partition.msg import _AttachmentPartitioner
+
+        attachment = Mock()
+        attachment.file_name = "../../../etc/passwd"
+        attachment.file_bytes = b"malicious content"
+        attachment.last_modified = None
+
+        opts = Mock()
+        opts.metadata_last_modified = None
+
+        partitioner = _AttachmentPartitioner(attachment, opts)
+
+        assert partitioner._attachment_file_name == "passwd"
+
+    def it_sanitizes_absolute_unix_paths(self, request: FixtureRequest):
+        from unstructured.partition.msg import _AttachmentPartitioner
+
+        attachment = Mock()
+        attachment.file_name = "/etc/passwd"
+        attachment.file_bytes = b"malicious content"
+        attachment.last_modified = None
+
+        opts = Mock()
+        opts.metadata_last_modified = None
+
+        partitioner = _AttachmentPartitioner(attachment, opts)
+
+        assert partitioner._attachment_file_name == "passwd"
+
+    def it_sanitizes_absolute_windows_paths(self, request: FixtureRequest):
+        from unstructured.partition.msg import _AttachmentPartitioner
+
+        attachment = Mock()
+        attachment.file_name = "C:\\Windows\\System32\\config\\sam"
+        attachment.file_bytes = b"malicious content"
+        attachment.last_modified = None
+
+        opts = Mock()
+        opts.metadata_last_modified = None
+
+        partitioner = _AttachmentPartitioner(attachment, opts)
+
+        assert partitioner._attachment_file_name == "sam"
+
+    def it_removes_null_bytes_from_filenames(self, request: FixtureRequest):
+        from unstructured.partition.msg import _AttachmentPartitioner
+
+        attachment = Mock()
+        attachment.file_name = "file\x00.txt"
+        attachment.file_bytes = b"content"
+        attachment.last_modified = None
+
+        opts = Mock()
+        opts.metadata_last_modified = None
+
+        partitioner = _AttachmentPartitioner(attachment, opts)
+
+        assert partitioner._attachment_file_name == "file.txt"
+        assert "\x00" not in partitioner._attachment_file_name
+
+    def it_handles_dot_and_dotdot_filenames(self, request: FixtureRequest):
+        from unstructured.partition.msg import _AttachmentPartitioner
+
+        opts = Mock()
+        opts.metadata_last_modified = None
+
+        # Test single dot
+        attachment1 = Mock()
+        attachment1.file_name = "."
+        attachment1.file_bytes = b"content"
+        attachment1.last_modified = None
+        partitioner1 = _AttachmentPartitioner(attachment1, opts)
+        assert partitioner1._attachment_file_name == "unknown"
+
+        # Test double dot
+        attachment2 = Mock()
+        attachment2.file_name = ".."
+        attachment2.file_bytes = b"content"
+        attachment2.last_modified = None
+        partitioner2 = _AttachmentPartitioner(attachment2, opts)
+        assert partitioner2._attachment_file_name == "unknown"
+
+    def it_handles_missing_filename(self, request: FixtureRequest):
+        from unstructured.partition.msg import _AttachmentPartitioner
+
+        attachment = Mock()
+        attachment.file_name = None
+        attachment.file_bytes = b"content"
+        attachment.last_modified = None
+
+        opts = Mock()
+        opts.metadata_last_modified = None
+
+        partitioner = _AttachmentPartitioner(attachment, opts)
+
+        assert partitioner._attachment_file_name == "unknown"
+
+    def it_allows_valid_filenames_through(self, request: FixtureRequest):
+        from unstructured.partition.msg import _AttachmentPartitioner
+
+        attachment = Mock()
+        attachment.file_name = "document.pdf"
+        attachment.file_bytes = b"content"
+        attachment.last_modified = None
+
+        opts = Mock()
+        opts.metadata_last_modified = None
+
+        partitioner = _AttachmentPartitioner(attachment, opts)
+
+        assert partitioner._attachment_file_name == "document.pdf"
+
+    def it_handles_complex_path_traversal_with_mixed_separators(self, request: FixtureRequest):
+        from unstructured.partition.msg import _AttachmentPartitioner
+
+        attachment = Mock()
+        attachment.file_name = "..\\../\\..\\etc/passwd"
+        attachment.file_bytes = b"malicious content"
+        attachment.last_modified = None
+
+        opts = Mock()
+        opts.metadata_last_modified = None
+
+        partitioner = _AttachmentPartitioner(attachment, opts)
+
+        assert partitioner._attachment_file_name == "passwd"
+
+    def it_handles_empty_string_filename(self, request: FixtureRequest):
+        from unstructured.partition.msg import _AttachmentPartitioner
+
+        attachment = Mock()
+        attachment.file_name = ""
+        attachment.file_bytes = b"content"
+        attachment.last_modified = None
+
+        opts = Mock()
+        opts.metadata_last_modified = None
+
+        partitioner = _AttachmentPartitioner(attachment, opts)
+
+        assert partitioner._attachment_file_name == "unknown"
+
+
+class DescribeMsgPartitionerOptions:
+    """Unit-test suite for `unstructured.partition.msg.MsgPartitionerOptions` objects."""
+
+    # -- .extra_msg_metadata ---------------------
+
+    def it_provides_email_specific_metadata_to_add_to_each_element(self, opts_args: dict[str, Any]):
+        opts_args["file_path"] = example_doc_path("fake-email-with-cc-and-bcc.msg")
+        opts = MsgPartitionerOptions(**opts_args)
+
+        m = opts.extra_msg_metadata
+        assert m.bcc_recipient == ["hello@unstructured.io"]
+        assert m.cc_recipient == ["steve@unstructured.io"]
+        assert m.email_message_id == "14DDEF33-2BA7-4CDD-A4D8-E7C5873B37F2@gmail.com"
+        assert m.sent_from == ['"John" <johnjennings702@gmail.com>']
+        assert m.sent_to == [
+            "john-ctr@unstructured.io",
+            "steve@unstructured.io",
+            "hello@unstructured.io",
+        ]
+        assert m.subject == "Fake email with cc and bcc recipients"
+
+    # -- .is_encrypted ---------------------------
+
+    @pytest.mark.parametrize(
+        ("file_name", "expected_value"), [("fake-encrypted.msg", True), ("fake-email.msg", False)]
+    )
+    def it_knows_when_the_msg_is_encrypted(
+        self, file_name: str, expected_value: bool, opts_args: dict[str, Any]
+    ):
+        opts_args["file_path"] = example_doc_path(file_name)
+        opts = MsgPartitionerOptions(**opts_args)
+
+        assert opts.is_encrypted is expected_value
+
+    # -- .metadata_file_path ---------------------
+
+    def it_uses_the_metadata_file_path_arg_when_provided(self, opts_args: dict[str, Any]):
+        opts_args["file_path"] = "x/y/z.msg"
+        opts_args["metadata_file_path"] = "a/b/c.msg"
+        opts = MsgPartitionerOptions(**opts_args)
+
+        assert opts.metadata_file_path == "a/b/c.msg"
+
+    def and_it_falls_back_to_the_MSG_file_path_arg_when_provided(self, opts_args: dict[str, Any]):
+        file_path = example_doc_path("fake-email.msg")
+        opts_args["file_path"] = file_path
+        opts = MsgPartitionerOptions(**opts_args)
+
+        assert opts.metadata_file_path == file_path
+
+    def but_it_returns_None_when_neither_path_is_available(self, opts_args: dict[str, Any]):
+        opts = MsgPartitionerOptions(**opts_args)
+
+        assert opts.metadata_file_path is None
+
+    # -- .metadata_last_modified -----------------
+
+    def it_uses_metadata_last_modified_when_provided_by_the_caller(self, opts_args: dict[str, Any]):
+        metadata_last_modified = "2024-03-05T17:02:53"
+        opts_args["metadata_last_modified"] = metadata_last_modified
+        opts_args["file_path"] = example_doc_path("fake-email.msg")
+        opts = MsgPartitionerOptions(**opts_args)
+
+        assert opts.metadata_last_modified == metadata_last_modified
+
+    def and_it_uses_the_message_Date_header_when_metadata_last_modified_is_not_provided(
+        self, opts_args: dict[str, Any]
+    ):
+        opts_args["file_path"] = example_doc_path("fake-email.msg")
+        opts = MsgPartitionerOptions(**opts_args)
+
+        assert opts.metadata_last_modified == "2023-03-28T17:00:31+00:00"
+
+    @pytest.mark.parametrize("filesystem_last_modified", ["2024-06-03T20:12:53", None])
+    def and_it_uses_the_last_modified_date_from_the_source_file_when_the_message_has_no_sent_date(
+        self,
+        opts_args: dict[str, Any],
+        filesystem_last_modified: str | None,
+        Message_sent_date_: Mock,
+        _last_modified_prop_: Mock,
+    ):
+        Message_sent_date_.return_value = None
+        _last_modified_prop_.return_value = filesystem_last_modified
+        opts_args["file_path"] = example_doc_path("fake-email.msg")
+        opts = MsgPartitionerOptions(**opts_args)
+
+        assert opts.metadata_last_modified == filesystem_last_modified
+
+    # -- .msg ------------------------------------
+
+    def it_loads_the_msg_document_from_a_file_path_when_provided(self, opts_args: dict[str, Any]):
+        opts_args["file_path"] = example_doc_path("fake-email.msg")
+        opts = MsgPartitionerOptions(**opts_args)
+
+        assert isinstance(opts.msg, Message)
+
+    def and_it_loads_the_msg_document_from_a_file_like_object_when_provided(
+        self, opts_args: dict[str, Any]
+    ):
+        with open(example_doc_path("fake-email.msg"), "rb") as f:
+            opts_args["file"] = io.BytesIO(f.read())
+        opts = MsgPartitionerOptions(**opts_args)
+
+        assert isinstance(opts.msg, Message)
+
+    def but_it_raises_when_neither_is_provided(self, opts_args: dict[str, Any]):
+        with pytest.raises(ValueError, match="one of `file` or `filename` arguments must be prov"):
+            MsgPartitionerOptions(**opts_args).msg
+
+    # -- .partition_attachments ------------------
+
+    @pytest.mark.parametrize("partition_attachments", [True, False])
+    def it_knows_whether_attachments_should_also_be_partitioned(
+        self, partition_attachments: bool, opts_args: dict[str, Any]
+    ):
+        opts_args["file_path"] = example_doc_path("fake-email.msg")
+        opts_args["partition_attachments"] = partition_attachments
+        opts = MsgPartitionerOptions(**opts_args)
+
+        assert opts.partition_attachments is partition_attachments
+
+    # -- .partitioning_kwargs --------------------
+
+    def it_provides_access_to_pass_through_kwargs_collected_by_the_partitioner_function(
+        self, opts_args: dict[str, Any]
+    ):
+        opts_args["kwargs"] = {"foo": 42, "bar": "baz"}
+        opts = MsgPartitionerOptions(**opts_args)
+
+        assert opts.partitioning_kwargs == {"foo": 42, "bar": "baz"}
+
+    # -- fixtures --------------------------------------------------------------------------------
+
+    @pytest.fixture
+    def _last_modified_prop_(self, request: FixtureRequest):
+        return property_mock(request, MsgPartitionerOptions, "_last_modified")
+
+    @pytest.fixture
+    def Message_sent_date_(self, request: FixtureRequest):
+        return property_mock(request, Message, "sent_date")
+
+    @pytest.fixture
+    def opts_args(self) -> dict[str, Any]:
+        """All default arguments for `MsgPartitionerOptions`.
+
+        Individual argument values can be changed to suit each test. Makes construction of opts more
+        compact for testing purposes.
+        """
+        return {
+            "file": None,
+            "file_path": None,
+            "metadata_file_path": None,
+            "metadata_last_modified": None,
+            "partition_attachments": False,
+            "kwargs": {},
+        }

test_unstructured/partition/test_ndjson.py

@@ -0,0 +1,311 @@
+"""Test-suite for `unstructured.partition.ndjson` module."""
+
+from __future__ import annotations
+
+import os
+import pathlib
+import tempfile
+
+import pytest
+from pytest_mock import MockFixture
+
+from test_unstructured.unit_utils import example_doc_path
+from unstructured.documents.elements import CompositeElement
+from unstructured.file_utils.model import FileType
+from unstructured.partition.email import partition_email
+from unstructured.partition.html import partition_html
+from unstructured.partition.ndjson import partition_ndjson
+from unstructured.partition.text import partition_text
+from unstructured.partition.xml import partition_xml
+from unstructured.staging.base import elements_to_ndjson
+
+DIRECTORY = pathlib.Path(__file__).parent.resolve()
+
+is_in_docker = os.path.exists("/.dockerenv")
+
+test_files = [
+    "fake-text.txt",
+    "fake-html.html",
+    "eml/fake-email.eml",
+]
+
+is_in_docker = os.path.exists("/.dockerenv")
+
+
+def test_it_chunks_elements_when_a_chunking_strategy_is_specified():
+    chunks = partition_ndjson(
+        example_doc_path("spring-weather.html.ndjson"),
+        chunking_strategy="basic",
+        max_characters=1500,
+    )
+
+    assert len(chunks) == 9
+    assert all(isinstance(ch, CompositeElement) for ch in chunks)
+
+
+@pytest.mark.parametrize("filename", test_files)
+def test_partition_ndjson_from_filename(filename: str):
+    path = example_doc_path(filename)
+    elements = []
+    filetype = FileType.from_extension(os.path.splitext(path)[1])
+    if filetype == FileType.TXT:
+        elements = partition_text(filename=path)
+    if filetype == FileType.HTML:
+        elements = partition_html(filename=path)
+    if filetype == FileType.XML:
+        elements = partition_xml(filename=path)
+    if filetype == FileType.EML:
+        elements = partition_email(filename=path)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        _filename = os.path.basename(filename)
+        test_path = os.path.join(tmpdir, _filename + ".ndjson")
+        elements_to_ndjson(elements, filename=test_path)
+        test_elements = partition_ndjson(filename=test_path)
+
+    assert len(elements) > 0
+    assert len(str(elements[0])) > 0
+
+    assert len(elements) == len(test_elements)
+    for i in range(len(elements)):
+        assert elements[i] == test_elements[i]
+        assert elements[i].metadata.filename == filename.split("/")[-1]
+
+
+@pytest.mark.parametrize("filename", test_files)
+def test_partition_ndjson_from_filename_with_metadata_filename(filename: str):
+    path = example_doc_path(filename)
+    elements = []
+    filetype = FileType.from_extension(os.path.splitext(path)[1])
+    if filetype == FileType.TXT:
+        elements = partition_text(filename=path)
+    if filetype == FileType.HTML:
+        elements = partition_html(filename=path)
+    if filetype == FileType.XML:
+        elements = partition_xml(filename=path)
+    if filetype == FileType.EML:
+        elements = partition_email(filename=path)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        _filename = os.path.basename(filename)
+        test_path = os.path.join(tmpdir, _filename + ".ndjson")
+        elements_to_ndjson(elements, filename=test_path)
+        test_elements = partition_ndjson(filename=test_path, metadata_filename="test")
+
+    assert len(test_elements) > 0
+    assert len(str(test_elements[0])) > 0
+    assert all(element.metadata.filename == "test" for element in test_elements)
+
+
+@pytest.mark.parametrize("filename", test_files)
+def test_partition_ndjson_from_file(filename: str):
+    path = example_doc_path(filename)
+    elements = []
+    filetype = FileType.from_extension(os.path.splitext(path)[1])
+    if filetype == FileType.TXT:
+        elements = partition_text(filename=path)
+    if filetype == FileType.HTML:
+        elements = partition_html(filename=path)
+    if filetype == FileType.XML:
+        elements = partition_xml(filename=path)
+    if filetype == FileType.EML:
+        elements = partition_email(filename=path)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        _filename = os.path.basename(filename)
+        test_path = os.path.join(tmpdir, _filename + ".ndjson")
+        elements_to_ndjson(elements, filename=test_path)
+        with open(test_path, "rb") as f:
+            test_elements = partition_ndjson(file=f)
+
+    assert len(elements) > 0
+    assert len(str(elements[0])) > 0
+    assert len(elements) == len(test_elements)
+    for i in range(len(elements)):
+        assert elements[i] == test_elements[i]
+        assert elements[i].metadata.filename == filename.split("/")[-1]
+
+
+@pytest.mark.parametrize("filename", test_files)
+def test_partition_ndjson_from_file_with_metadata_filename(filename: str):
+    path = example_doc_path(filename)
+    elements = []
+    filetype = FileType.from_extension(os.path.splitext(path)[1])
+    if filetype == FileType.TXT:
+        elements = partition_text(filename=path)
+    if filetype == FileType.HTML:
+        elements = partition_html(filename=path)
+    if filetype == FileType.XML:
+        elements = partition_xml(filename=path)
+    if filetype == FileType.EML:
+        elements = partition_email(filename=path)
+    with tempfile.TemporaryDirectory() as tmpdir:
+        _filename = os.path.basename(filename)
+        test_path = os.path.join(tmpdir, _filename + ".ndjson")
+        elements_to_ndjson(elements, filename=test_path)
+        with open(test_path, "rb") as f:
+            test_elements = partition_ndjson(file=f, metadata_filename="test")
+
+    for i in range(len(test_elements)):
+        assert test_elements[i].metadata.filename == "test"
+
+
+@pytest.mark.parametrize("filename", test_files)
+def test_partition_ndjson_from_text(filename: str):
+    path = example_doc_path(filename)
+    elements = []
+    filetype = FileType.from_extension(os.path.splitext(path)[1])
+    if filetype == FileType.TXT:
+        elements = partition_text(filename=path)
+    if filetype == FileType.HTML:
+        elements = partition_html(filename=path)
+    if filetype == FileType.XML:
+        elements = partition_xml(filename=path)
+    if filetype == FileType.EML:
+        elements = partition_email(filename=path)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        _filename = os.path.basename(filename)
+        test_path = os.path.join(tmpdir, _filename + ".ndjson")
+        elements_to_ndjson(elements, filename=test_path)
+        with open(test_path) as f:
+            text = f.read()
+        test_elements = partition_ndjson(text=text)
+
+    assert len(elements) > 0
+    assert len(str(elements[0])) > 0
+    assert len(elements) == len(test_elements)
+    for i in range(len(elements)):
+        assert elements[i] == test_elements[i]
+        assert elements[i].metadata.filename == filename.split("/")[-1]
+
+
+def test_partition_json_raises_with_none_specified():
+    with pytest.raises(ValueError):
+        partition_ndjson()
+
+
+def test_partition_ndjson_works_with_empty_string():
+    assert partition_ndjson(text="") == []
+
+
+def test_partition_ndjson_fails_with_empty_item():
+    with pytest.raises(ValueError):
+        partition_ndjson(text="{}")
+
+
+def test_partition_ndjson_fails_with_empty_list():
+    with pytest.raises(ValueError):
+        partition_ndjson(text="[]")
+
+
+def test_partition_ndjson_raises_with_too_many_specified():
+    path = example_doc_path("fake-text.txt")
+    elements = []
+    filetype = FileType.from_extension(os.path.splitext(path)[1])
+    if filetype == FileType.TXT:
+        elements = partition_text(filename=path)
+    if filetype == FileType.HTML:
+        elements = partition_html(filename=path)
+    if filetype == FileType.XML:
+        elements = partition_xml(filename=path)
+    if filetype == FileType.EML:
+        elements = partition_email(filename=path)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        test_path = os.path.join(tmpdir, "fake-text.txt.ndjson")
+        elements_to_ndjson(elements, filename=test_path)
+        with open(test_path, "rb") as f:
+            text = f.read().decode("utf-8")
+
+    with pytest.raises(ValueError):
+        partition_ndjson(filename=test_path, file=f)
+
+    with pytest.raises(ValueError):
+        partition_ndjson(filename=test_path, text=text)
+
+    with pytest.raises(ValueError):
+        partition_ndjson(file=f, text=text)
+
+    with pytest.raises(ValueError):
+        partition_ndjson(filename=test_path, file=f, text=text)
+
+
+# -- .metadata.last_modified ---------------------------------------------------------------------
+
+
+def test_partition_ndjson_from_file_path_gets_last_modified_from_filesystem(mocker: MockFixture):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.ndjson.get_last_modified_date",
+        return_value=filesystem_last_modified,
+    )
+
+    elements = partition_ndjson(example_doc_path("spring-weather.html.ndjson"))
+
+    assert all(e.metadata.last_modified == filesystem_last_modified for e in elements)
+
+
+def test_partition_ndjson_from_file_gets_last_modified_None():
+    with open(example_doc_path("spring-weather.html.ndjson"), "rb") as f:
+        elements = partition_ndjson(file=f)
+
+    assert all(e.metadata.last_modified is None for e in elements)
+
+
+def test_partition_ndjson_from_text_gets_last_modified_None():
+    with open(example_doc_path("spring-weather.html.ndjson")) as f:
+        text = f.read()
+
+    elements = partition_ndjson(text=text)
+
+    assert all(e.metadata.last_modified is None for e in elements)
+
+
+def test_partition_ndjson_from_file_path_prefers_metadata_last_modified(mocker: MockFixture):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    metadata_last_modified = "2020-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.ndjson.get_last_modified_date",
+        return_value=filesystem_last_modified,
+    )
+
+    elements = partition_ndjson(
+        example_doc_path("spring-weather.html.ndjson"),
+        metadata_last_modified=metadata_last_modified,
+    )
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+def test_partition_ndjson_from_file_prefers_metadata_last_modified():
+    metadata_last_modified = "2020-07-05T09:24:28"
+    with open(example_doc_path("spring-weather.html.ndjson"), "rb") as f:
+        elements = partition_ndjson(file=f, metadata_last_modified=metadata_last_modified)
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+def test_partition_ndjson_from_text_prefers_metadata_last_modified():
+    metadata_last_modified = "2020-07-05T09:24:28"
+    with open(example_doc_path("spring-weather.html.ndjson")) as f:
+        text = f.read()
+
+    elements = partition_ndjson(text=text, metadata_last_modified=metadata_last_modified)
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+# ------------------------------------------------------------------------------------------------
+
+
+def test_partition_json_raises_with_unprocessable_json():
+    text = '{"invalid": "schema"}'
+    with pytest.raises(ValueError):
+        partition_ndjson(text=text)
+
+
+def test_partition_json_raises_with_invalid_json():
+    text = '[{"hi": "there"}]]'
+    with pytest.raises(ValueError):
+        partition_ndjson(text=text)

test_unstructured/partition/test_odt.py

@@ -0,0 +1,219 @@
+# pyright: reportPrivateUsage=false
+
+"""Test suite for `unstructured.partition.odt` module."""
+
+from __future__ import annotations
+
+from typing import Any, Iterator
+
+import pytest
+from pytest_mock import MockFixture
+
+from test_unstructured.unit_utils import (
+    ANY,
+    FixtureRequest,
+    assert_round_trips_through_JSON,
+    example_doc_path,
+    method_mock,
+)
+from unstructured.chunking.basic import chunk_elements
+from unstructured.documents.elements import (
+    CompositeElement,
+    Element,
+    NarrativeText,
+    Table,
+    TableChunk,
+    Text,
+)
+from unstructured.partition.docx import partition_docx
+from unstructured.partition.odt import partition_odt
+from unstructured.partition.utils.constants import UNSTRUCTURED_INCLUDE_DEBUG_METADATA
+
+
+def test_partition_odt_matches_partition_docx():
+    odt_file_path = example_doc_path("simple.odt")
+    docx_file_path = example_doc_path("simple.docx")
+
+    assert partition_odt(odt_file_path) == partition_docx(docx_file_path)
+
+
+# -- document-source (file or filename) ----------------------------------------------------------
+
+
+def test_partition_odt_from_filename():
+    elements = partition_odt(example_doc_path("fake.odt"))
+
+    assert elements == [
+        NarrativeText("Lorem ipsum dolor sit amet."),
+        Table(
+            "Header row Mon Wed Fri Color Blue Red Green Time 1pm 2pm 3pm Leader Sarah Mark Ryan"
+        ),
+    ]
+    assert all(e.metadata.filename == "fake.odt" for e in elements)
+    if UNSTRUCTURED_INCLUDE_DEBUG_METADATA:
+        # -- document is ultimately partitioned by partition_docx() --
+        assert {e.metadata.detection_origin for e in elements} == {"docx"}
+
+
+def test_partition_odt_from_file():
+    with open(example_doc_path("fake.odt"), "rb") as f:
+        elements = partition_odt(file=f)
+
+    assert elements == [
+        NarrativeText("Lorem ipsum dolor sit amet."),
+        Table(
+            "Header row Mon Wed Fri Color Blue Red Green Time 1pm 2pm 3pm Leader Sarah Mark Ryan"
+        ),
+    ]
+
+
+# -- .metadata.filename --------------------------------------------------------------------------
+
+
+def test_partition_odt_from_filename_gets_the_ODT_filename_in_metadata_not_the_DOCX_filename():
+    elements = partition_odt(example_doc_path("simple.odt"))
+    assert all(e.metadata.filename == "simple.odt" for e in elements), (
+        f"Expected all elements to have 'simple.odt' as their filename, but got:"
+        f" {repr(elements[0].metadata.filename)}"
+    )
+
+
+def test_partition_odt_from_filename_with_metadata_filename():
+    elements = partition_odt(example_doc_path("fake.odt"), metadata_filename="test")
+    assert all(e.metadata.filename == "test" for e in elements)
+
+
+def test_partition_odt_from_file_with_metadata_filename():
+    with open(example_doc_path("fake.odt"), "rb") as f:
+        elements = partition_odt(file=f, metadata_filename="test")
+    assert all(e.metadata.filename == "test" for e in elements)
+
+
+# -- .metadata.filetype --------------------------------------------------------------------------
+
+
+def test_partition_odt_gets_the_ODT_MIME_type_in_metadata_filetype():
+    ODT_MIME_TYPE = "application/vnd.oasis.opendocument.text"
+    elements = partition_odt(example_doc_path("simple.odt"))
+    assert all(e.metadata.filetype == ODT_MIME_TYPE for e in elements), (
+        f"Expected all elements to have '{ODT_MIME_TYPE}' as their filetype, but got:"
+        f" {repr(elements[0].metadata.filetype)}"
+    )
+
+
+# -- .metadata.text_as_html ----------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("kwargs", [{}, {"infer_table_structure": True}])
+def test_partition_odt_adds_text_as_html_when_infer_table_structure_is_omitted_or_True(
+    kwargs: dict[str, Any],
+):
+    with open(example_doc_path("fake.odt"), "rb") as f:
+        elements = partition_odt(file=f, **kwargs)
+
+    table = elements[1]
+    assert isinstance(table, Table)
+    assert table.metadata.text_as_html is not None
+    assert table.metadata.text_as_html.startswith("<table>")
+
+
+def test_partition_odt_suppresses_text_as_html_when_infer_table_structure_is_False():
+    with open(example_doc_path("fake.odt"), "rb") as f:
+        elements = partition_odt(file=f, infer_table_structure=False)
+
+    table = elements[1]
+    assert isinstance(table, Table)
+    assert table.metadata.text_as_html is None
+
+
+# -- .metadata.last_modified ---------------------------------------------------------------------
+
+
+def test_partition_odt_pulls_last_modified_from_filesystem(mocker: MockFixture):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.odt.get_last_modified_date", return_value=filesystem_last_modified
+    )
+
+    elements = partition_odt(example_doc_path("fake.odt"))
+
+    assert all(e.metadata.last_modified == filesystem_last_modified for e in elements)
+
+
+def test_partition_odt_prefers_metadata_last_modified_when_provided(mocker: MockFixture):
+    filesystem_last_modified = "2029-07-05T09:24:28"
+    metadata_last_modified = "2020-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.odt.get_last_modified_date", return_value=filesystem_last_modified
+    )
+
+    elements = partition_odt(
+        example_doc_path("simple.odt"), metadata_last_modified=metadata_last_modified
+    )
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+# -- .metadata.languages -------------------------------------------------------------------------
+
+
+def test_partition_odt_adds_languages_metadata():
+    elements = partition_odt(example_doc_path("simple.odt"))
+    assert all(e.metadata.languages == ["eng"] for e in elements)
+
+
+def test_partition_odt_respects_detect_language_per_element_arg():
+    elements = partition_odt(
+        example_doc_path("language-docs/eng_spa_mult.odt"), detect_language_per_element=True
+    )
+    assert [e.metadata.languages for e in elements] == [
+        ["eng"],
+        ["spa", "eng"],
+        ["eng"],
+        ["eng"],
+        ["spa"],
+    ]
+
+
+# -- miscellaneous -------------------------------------------------------------------------------
+
+
+@pytest.mark.parametrize(
+    ("kwargs", "expected_value"),
+    [({}, "hi_res"), ({"strategy": None}, "hi_res"), ({"strategy": "auto"}, "auto")],
+)
+def test_partition_odt_forwards_strategy_arg_to_partition_docx(
+    request: FixtureRequest, kwargs: dict[str, Any], expected_value: str | None
+):
+    from unstructured.partition.docx import _DocxPartitioner
+
+    def fake_iter_document_elements(self: _DocxPartitioner) -> Iterator[Element]:
+        yield Text(f"strategy == {self._opts.strategy}")
+
+    _iter_elements_ = method_mock(
+        request,
+        _DocxPartitioner,
+        "_iter_document_elements",
+        side_effect=fake_iter_document_elements,
+    )
+
+    (element,) = partition_odt(example_doc_path("simple.odt"), **kwargs)
+
+    _iter_elements_.assert_called_once_with(ANY)
+    assert element.text == f"strategy == {expected_value}"
+
+
+def test_partition_odt_round_trips_through_json():
+    """Elements produced can be serialized then deserialized without loss."""
+    assert_round_trips_through_JSON(partition_odt(example_doc_path("simple.odt")))
+
+
+def test_partition_odt_chunks_elements_when_chunking_strategy_is_specified():
+    document_path = example_doc_path("simple.odt")
+    elements = partition_odt(document_path)
+    chunks = partition_odt(document_path, chunking_strategy="basic")
+
+    # -- all chunks are chunk element-types --
+    assert all(isinstance(c, (CompositeElement, Table, TableChunk)) for c in chunks)
+    # -- chunks from partitioning match those produced by chunking elements in separate step --
+    assert chunks == chunk_elements(elements)

test_unstructured/partition/test_org.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from pytest_mock import MockFixture
+
+from test_unstructured.unit_utils import (
+    assert_round_trips_through_JSON,
+    example_doc_path,
+    find_text_in_elements,
+)
+from unstructured.chunking.title import chunk_by_title
+from unstructured.documents.elements import Title
+from unstructured.partition.org import partition_org
+
+
+def test_partition_org_from_filename():
+    elements = partition_org(example_doc_path("README.org"))
+
+    assert elements[0] == Title("Example Docs")
+    assert elements[0].metadata.filetype == "text/org"
+
+
+def test_partition_org_from_file():
+    with open(example_doc_path("README.org"), "rb") as f:
+        elements = partition_org(file=f)
+
+    assert elements[0] == Title("Example Docs")
+
+
+# -- .metadata.filename --------------------------------------------------------------------------
+
+
+def test_partition_org_from_filename_gets_filename_from_filename_arg():
+    elements = partition_org(example_doc_path("README.org"))
+
+    assert len(elements) > 0
+    assert all(e.metadata.filename == "README.org" for e in elements)
+
+
+def test_partition_org_from_file_gets_filename_None():
+    with open(example_doc_path("README.org"), "rb") as f:
+        elements = partition_org(file=f)
+
+    assert len(elements) > 0
+    assert all(e.metadata.filename is None for e in elements)
+
+
+def test_partition_org_from_filename_prefers_metadata_filename():
+    elements = partition_org(example_doc_path("README.org"), metadata_filename="orig-name.org")
+
+    assert len(elements) > 0
+    assert all(element.metadata.filename == "orig-name.org" for element in elements)
+
+
+def test_partition_org_from_file_prefers_metadata_filename():
+    with open(example_doc_path("README.org"), "rb") as f:
+        elements = partition_org(file=f, metadata_filename="orig-name.org")
+
+    assert all(e.metadata.filename == "orig-name.org" for e in elements)
+
+
+# -- .metadata.filetype --------------------------------------------------------------------------
+
+
+def test_partition_org_gets_the_ORG_MIME_type_in_metadata_filetype():
+    ORG_MIME_TYPE = "text/org"
+    elements = partition_org(example_doc_path("README.org"))
+    assert all(e.metadata.filetype == ORG_MIME_TYPE for e in elements), (
+        f"Expected all elements to have '{ORG_MIME_TYPE}' as their filetype, but got:"
+        f" {repr(elements[0].metadata.filetype)}"
+    )
+
+
+# -- .metadata.last_modified ---------------------------------------------------------------------
+
+
+def test_partition_org_from_filename_gets_last_modified_from_filesystem(mocker: MockFixture):
+    filesystem_last_modified = "2024-06-14T16:01:29"
+    mocker.patch(
+        "unstructured.partition.org.get_last_modified_date", return_value=filesystem_last_modified
+    )
+
+    elements = partition_org(example_doc_path("README.org"))
+
+    assert all(e.metadata.last_modified == filesystem_last_modified for e in elements)
+
+
+def test_partition_org_from_file_gets_last_modified_None():
+    with open(example_doc_path("README.org"), "rb") as f:
+        elements = partition_org(file=f)
+
+    assert all(e.metadata.last_modified is None for e in elements)
+
+
+def test_partition_org_from_filename_prefers_metadata_last_modified(mocker: MockFixture):
+    filesystem_last_modified = "2020-08-04T06:11:47"
+    metadata_last_modified = "2024-06-14T16:01:29"
+    mocker.patch(
+        "unstructured.partition.org.get_last_modified_date", return_value=filesystem_last_modified
+    )
+
+    elements = partition_org(
+        example_doc_path("README.org"), metadata_last_modified=metadata_last_modified
+    )
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+def test_partition_org_from_file_prefers_metadata_last_modified():
+    metadata_last_modified = "2020-07-05T09:24:28"
+    with open(example_doc_path("README.org"), "rb") as f:
+        elements = partition_org(file=f, metadata_last_modified=metadata_last_modified)
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+# ------------------------------------------------------------------------------------------------
+
+
+def test_partition_org_with_json():
+    elements = partition_org(example_doc_path("README.org"))
+    assert_round_trips_through_JSON(elements)
+
+
+def test_add_chunking_strategy_by_title_on_partition_org():
+    file_path = example_doc_path("README.org")
+    elements = partition_org(file_path)
+    chunk_elements = partition_org(file_path, chunking_strategy="by_title")
+    chunks = chunk_by_title(elements)
+
+    assert chunk_elements != elements
+    assert chunk_elements == chunks
+
+
+def test_partition_org_element_metadata_has_languages():
+    elements = partition_org(example_doc_path("README.org"))
+    assert elements[0].metadata.languages == ["eng"]
+
+
+def test_partition_org_respects_detect_language_per_element():
+    elements = partition_org(
+        example_doc_path("language-docs/eng_spa_mult.org"), detect_language_per_element=True
+    )
+    langs = [element.metadata.languages for element in elements]
+    assert langs == [["eng"], ["spa", "eng"], ["eng"], ["eng"], ["spa"]]
+
+
+def test_org_wont_include_external_files():
+    # Make sure our import file is in place (otherwise the import fails silently and test passes)
+    assert Path(example_doc_path("file_we_dont_want_imported")).exists()
+    elements = partition_org(example_doc_path("README-w-include.org"))
+    # The partition should contain some elements
+    assert elements
+    # We find something we expect to find from file we partitioned directly
+    assert find_text_in_elements("instructions", elements)
+    # But we don't find something from the file included within the file we partitioned directly
+    assert not find_text_in_elements("wombat", elements)

test_unstructured/partition/test_ppt.py

@@ -0,0 +1,180 @@
+"""Test-suite for `unstructured.partition.ppt` module."""
+
+from __future__ import annotations
+
+import pytest
+from pytest_mock import MockFixture
+
+from test_unstructured.unit_utils import assert_round_trips_through_JSON, example_doc_path
+from unstructured.chunking.title import chunk_by_title
+from unstructured.documents.elements import ListItem, NarrativeText, PageBreak, Title
+from unstructured.partition.ppt import partition_ppt
+from unstructured.partition.utils.constants import UNSTRUCTURED_INCLUDE_DEBUG_METADATA
+
+EXPECTED_PPT_OUTPUT = [
+    Title(text="Adding a Bullet Slide"),
+    ListItem(text="Find the bullet slide layout"),
+    ListItem(text="Use _TextFrame.text for first bullet"),
+    ListItem(text="Use _TextFrame.add_paragraph() for subsequent bullets"),
+    NarrativeText(text="Here is a lot of text!"),
+    NarrativeText(text="Here is some text in a text box!"),
+]
+
+
+def test_partition_ppt_from_filename():
+    elements = partition_ppt(example_doc_path("fake-power-point.ppt"))
+    assert elements == EXPECTED_PPT_OUTPUT
+    for element in elements:
+        assert element.metadata.filename == "fake-power-point.ppt"
+    if UNSTRUCTURED_INCLUDE_DEBUG_METADATA:
+        assert {element.metadata.detection_origin for element in elements} == {"pptx"}
+
+
+def test_partition_ppt_raises_with_missing_file():
+    with pytest.raises(ValueError):
+        partition_ppt(example_doc_path("doesnt-exist.ppt"))
+
+
+def test_partition_ppt_from_file():
+    with open(example_doc_path("fake-power-point.ppt"), "rb") as f:
+        elements = partition_ppt(file=f)
+    assert elements == EXPECTED_PPT_OUTPUT
+    for element in elements:
+        assert element.metadata.filename is None
+
+
+def test_partition_ppt_from_file_with_metadata_filename():
+    with open(example_doc_path("fake-power-point.ppt"), "rb") as f:
+        elements = partition_ppt(file=f, metadata_filename="test")
+    assert elements == EXPECTED_PPT_OUTPUT
+    for element in elements:
+        assert element.metadata.filename == "test"
+
+
+def test_partition_ppt_raises_with_both_specified():
+    filename = example_doc_path("fake-power-point.ppt")
+    with open(filename, "rb") as f, pytest.raises(ValueError):
+        partition_ppt(filename=filename, file=f)
+
+
+def test_partition_ppt_raises_when_neither_file_path_or_file_is_provided():
+    with pytest.raises(ValueError):
+        partition_ppt()
+
+
+# -- .metadata.filename --------------------------------------------------------------------------
+
+
+def test_partition_ppt_from_filename_gets_filename_from_filename_arg():
+    elements = partition_ppt(example_doc_path("fake-power-point.ppt"))
+
+    assert len(elements) > 0
+    assert all(e.metadata.filename == "fake-power-point.ppt" for e in elements)
+
+
+def test_partition_ppt_from_file_gets_filename_None():
+    with open(example_doc_path("fake-power-point.ppt"), "rb") as f:
+        elements = partition_ppt(file=f)
+
+    assert len(elements) > 0
+    assert all(e.metadata.filename is None for e in elements)
+
+
+def test_partition_ppt_from_filename_prefers_metadata_filename():
+    elements = partition_ppt(example_doc_path("fake-power-point.ppt"), metadata_filename="test")
+
+    assert len(elements) > 0
+    assert all(element.metadata.filename == "test" for element in elements)
+
+
+def test_partition_ppt_from_file_prefers_metadata_filename():
+    with open(example_doc_path("fake-power-point.ppt"), "rb") as f:
+        elements = partition_ppt(file=f, metadata_filename="test")
+
+    assert all(e.metadata.filename == "test" for e in elements)
+
+
+# -- .metadata.last_modified ---------------------------------------------------------------------
+
+
+def test_partition_ppt_from_file_path_gets_last_modified_from_filesystem(mocker: MockFixture):
+    filesystem_last_modified = "2024-05-01T15:37:28"
+    mocker.patch(
+        "unstructured.partition.ppt.get_last_modified_date", return_value=filesystem_last_modified
+    )
+
+    elements = partition_ppt(example_doc_path("fake-power-point.ppt"))
+
+    assert all(e.metadata.last_modified == filesystem_last_modified for e in elements)
+
+
+def test_partition_ppt_from_file_gets_last_modified_None():
+    with open(example_doc_path("fake-power-point.ppt"), "rb") as f:
+        elements = partition_ppt(file=f)
+
+    assert all(e.metadata.last_modified is None for e in elements)
+
+
+def test_partition_ppt_from_file_path_prefers_metadata_last_modified(mocker: MockFixture):
+    filesystem_last_modified = "2024-05-01T15:37:28"
+    metadata_last_modified = "2020-07-05T09:24:28"
+    mocker.patch(
+        "unstructured.partition.ppt.get_last_modified_date", return_value=filesystem_last_modified
+    )
+
+    elements = partition_ppt(
+        example_doc_path("fake-power-point.ppt"), metadata_last_modified=metadata_last_modified
+    )
+
+    assert all(e.metadata.last_modified == metadata_last_modified for e in elements)
+
+
+def test_partition_ppt_from_file_prefers_metadata_last_modified():
+    metadata_last_modified = "2020-07-05T09:24:28"
+
+    with open(example_doc_path("fake-power-point.ppt"), "rb") as f:
+        elements = partition_ppt(file=f, metadata_last_modified=metadata_last_modified)
+
+    assert elements[0].metadata.last_modified == metadata_last_modified
+
+
+# ------------------------------------------------------------------------------------------------
+
+
+def test_partition_ppt_with_json():
+    elements = partition_ppt(example_doc_path("fake-power-point.ppt"))
+    assert_round_trips_through_JSON(elements)
+
+
+def test_add_chunking_strategy_by_title_on_partition_ppt():
+    file_path = example_doc_path("fake-power-point.ppt")
+    elements = partition_ppt(file_path)
+    chunk_elements = partition_ppt(file_path, chunking_strategy="by_title")
+    chunks = chunk_by_title(elements)
+    assert chunk_elements != elements
+    assert chunk_elements == chunks
+
+
+def test_partition_ppt_params():
+    """Integration test of params: languages, include_page_break, and include_slide_notes."""
+    elements = partition_ppt(
+        example_doc_path("language-docs/eng_spa_mult.ppt"),
+        include_page_breaks=True,
+        include_slide_notes=True,
+    )
+    assert elements[0].metadata.languages == ["eng"]
+    assert any(isinstance(element, PageBreak) for element in elements)
+    # The example doc contains a slide note with the text "This is a slide note."
+    assert any(element.text == "This is a slide note." for element in elements)
+
+
+def test_partition_ppt_respects_detect_language_per_element():
+    elements = partition_ppt(
+        example_doc_path("language-docs/eng_spa_mult.ppt"), detect_language_per_element=True
+    )
+    langs = [element.metadata.languages for element in elements]
+    # languages other than English and Spanish are detected by this partitioner,
+    # so this test is slightly different from the other partition tests
+    langs = {element.metadata.languages[0] for element in elements if element.metadata.languages}
+    assert "eng" in langs
+    assert "spa" in langs