chore: import upstream snapshot with attribution

test_unstructured/chunking/test_basic.py

@@ -0,0 +1,308 @@
+"""Test suite for the `unstructured.chunking.basic` module.
+
+That module implements the baseline chunking strategy. The baseline strategy has all behaviors
+shared by all chunking strategies and no extra rules like perserve section or page boundaries.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from test_unstructured.unit_utils import FixtureRequest, Mock, function_mock
+from unstructured.chunking.basic import chunk_elements
+from unstructured.documents.elements import (
+    CompositeElement,
+    ElementMetadata,
+    Table,
+    TableChunk,
+    Text,
+    Title,
+)
+from unstructured.partition.docx import partition_docx
+
+
+def test_it_chunks_a_document_when_basic_chunking_strategy_is_specified_on_partition_function():
+    """Basic chunking can be combined with partitioning, exercising the decorator."""
+    filename = "example-docs/handbook-1p.docx"
+
+    chunks = partition_docx(filename, chunking_strategy="basic")
+
+    assert chunks == [
+        CompositeElement(
+            "US Trustee Handbook\n\nCHAPTER 1\n\nINTRODUCTION\n\nCHAPTER 1 – INTRODUCTION"
+            "\n\nA. PURPOSE"
+        ),
+        CompositeElement(
+            "The United States Trustee appoints and supervises standing trustees and monitors and"
+            " supervises cases under chapter 13 of title 11 of the United States Code. 28 U.S.C."
+            " § 586(b). The Handbook, issued as part of our duties under 28 U.S.C. § 586,"
+            " establishes or clarifies the position of the United States Trustee Program (Program)"
+            " on the duties owed by a standing trustee to the debtors, creditors, other parties in"
+            " interest, and the United States Trustee. The Handbook does not present a full and"
+        ),
+        CompositeElement(
+            "complete statement of the law; it should not be used as a substitute for legal"
+            " research and analysis. The standing trustee must be familiar with relevant"
+            " provisions of the Bankruptcy Code, Federal Rules of Bankruptcy Procedure (Rules),"
+            " any local bankruptcy rules, and case law. 11 U.S.C. § 321, 28 U.S.C. § 586,"
+            " 28 C.F.R. § 58.6(a)(3). Standing trustees are encouraged to follow Practice Tips"
+            " identified in this Handbook but these are not considered mandatory."
+        ),
+        CompositeElement(
+            "Nothing in this Handbook should be construed to excuse the standing trustee from"
+            " complying with all duties imposed by the Bankruptcy Code and Rules, local rules, and"
+            " orders of the court. The standing trustee should notify the United States Trustee"
+            " whenever the provision of the Handbook conflicts with the local rules or orders of"
+            " the court. The standing trustee is accountable for all duties set forth in this"
+            " Handbook, but need not personally perform any duty unless otherwise indicated. All"
+        ),
+        CompositeElement(
+            "statutory references in this Handbook refer to the Bankruptcy Code, 11 U.S.C. § 101"
+            " et seq., unless otherwise indicated."
+        ),
+        CompositeElement(
+            "This Handbook does not create additional rights against the standing trustee or"
+            " United States Trustee in favor of other parties.\n\nB. ROLE OF THE UNITED STATES"
+            " TRUSTEE"
+        ),
+        CompositeElement(
+            "The Bankruptcy Reform Act of 1978 removed the bankruptcy judge from the"
+            " responsibilities for daytoday administration of cases. Debtors, creditors, and"
+            " third parties with adverse interests to the trustee were concerned that the court,"
+            " which previously appointed and supervised the trustee, would not impartially"
+            " adjudicate their rights as adversaries of that trustee. To address these concerns,"
+            " judicial and administrative functions within the bankruptcy system were bifurcated."
+        ),
+        CompositeElement(
+            "Many administrative functions formerly performed by the court were placed within the"
+            " Department of Justice through the creation of the Program. Among the administrative"
+            " functions assigned to the United States Trustee were the appointment and supervision"
+            " of chapter 13 trustees./ This Handbook is issued under the authority of the"
+            " Program’s enabling statutes.\n\nC. STATUTORY DUTIES OF A STANDING TRUSTEE"
+        ),
+        CompositeElement(
+            "The standing trustee has a fiduciary responsibility to the bankruptcy estate. The"
+            " standing trustee is more than a mere disbursing agent. The standing trustee must"
+            " be personally involved in the trustee operation. If the standing trustee is or"
+            " becomes unable to perform the duties and responsibilities of a standing trustee,"
+            " the standing trustee must immediately advise the United States Trustee."
+            " 28 U.S.C. § 586(b), 28 C.F.R. § 58.4(b) referencing 28 C.F.R. § 58.3(b)."
+        ),
+        CompositeElement(
+            "Although this Handbook is not intended to be a complete statutory reference, the"
+            " standing trustee’s primary statutory duties are set forth in 11 U.S.C. § 1302, which"
+            " incorporates by reference some of the duties of chapter 7 trustees found in"
+            " 11 U.S.C. § 704. These duties include, but are not limited to, the"
+            " following:\n\nCopyright"
+        ),
+    ]
+
+
+def test_it_chunks_elements_when_the_user_already_has_them():
+    elements = [
+        Title("Introduction"),
+        Text(
+            # --------------------------------------------------------- 64 -v
+            "Lorem ipsum dolor sit amet consectetur adipiscing elit. In rhoncus ipsum sed lectus"
+            " porta volutpat.",
+        ),
+    ]
+
+    chunks = chunk_elements(elements, max_characters=64)
+
+    assert chunks == [
+        CompositeElement("Introduction"),
+        # -- splits on even word boundary, not mid-"rhoncus" --
+        CompositeElement("Lorem ipsum dolor sit amet consectetur adipiscing elit. In"),
+        CompositeElement("rhoncus ipsum sed lectus porta volutpat."),
+    ]
+
+
+def test_it_includes_original_elements_as_metadata_when_requested():
+    element = Title("Introduction")
+    element_2 = Text("Lorem ipsum dolor sit amet consectetur adipiscing elit.")
+    element_3 = Text("In rhoncus ipsum sed lectus porta volutpat.")
+
+    chunks = chunk_elements(
+        [element, element_2, element_3], max_characters=70, include_orig_elements=True
+    )
+
+    assert len(chunks) == 2
+    chunk = chunks[0]
+    assert chunk == CompositeElement(
+        "Introduction\n\nLorem ipsum dolor sit amet consectetur adipiscing elit."
+    )
+    assert chunk.metadata.orig_elements == [element, element_2]
+    # --
+    chunk = chunks[1]
+    assert chunk == CompositeElement("In rhoncus ipsum sed lectus porta volutpat.")
+    assert chunk.metadata.orig_elements == [element_3]
+
+
+def test_it_repeats_table_headers_by_default_but_can_opt_out():
+    table_html = (
+        "<table>"
+        "<thead>"
+        "<tr><th>Header A</th><th>Header B</th></tr>"
+        "<tr><th>Subhead A</th><th>Subhead B</th></tr>"
+        "</thead>"
+        "<tbody>"
+        "<tr><td>Body 1</td><td>Alpha</td></tr>"
+        "<tr><td>Body 2</td><td>Bravo</td></tr>"
+        "<tr><td>Body 3</td><td>Charlie</td></tr>"
+        "<tr><td>Body 4</td><td>Delta</td></tr>"
+        "</tbody>"
+        "</table>"
+    )
+    table_text = (
+        "Header A Header B\n"
+        "Subhead A Subhead B\n"
+        "Body 1 Alpha\n"
+        "Body 2 Bravo\n"
+        "Body 3 Charlie\n"
+        "Body 4 Delta"
+    )
+    table = Table(table_text, metadata=ElementMetadata(text_as_html=table_html))
+
+    repeated_header_chunks = chunk_elements([table], max_characters=55)
+    opt_out_chunks = chunk_elements([table], max_characters=55, repeat_table_headers=False)
+
+    assert len(repeated_header_chunks) == 4
+    assert all(isinstance(chunk, TableChunk) for chunk in repeated_header_chunks)
+    assert [chunk.text for chunk in repeated_header_chunks] == [
+        "Header A Header B Subhead A Subhead B Body 1 Alpha",
+        "Header A Header B Subhead A Subhead B Body 2 Bravo",
+        "Header A Header B Subhead A Subhead B Body 3 Charlie",
+        "Header A Header B Subhead A Subhead B Body 4 Delta",
+    ]
+    assert [chunk.text for chunk in opt_out_chunks] == [
+        "Header A Header B Subhead A Subhead B Body 1 Alpha",
+        "Body 2 Bravo Body 3 Charlie Body 4 Delta",
+    ]
+
+
+def test_skip_table_chunking_passes_oversized_table_through_unchanged():
+    table_text = "cell " * 200  # 1000 chars, well above max_characters=100
+    table = Table(table_text.strip())
+    text_before = Text("Hello world")
+    text_after = Text("Goodbye world")
+
+    chunks = chunk_elements(
+        [text_before, table, text_after],
+        max_characters=100,
+        skip_table_chunking=True,
+    )
+
+    assert len(chunks) == 3
+    assert isinstance(chunks[0], CompositeElement)
+    assert isinstance(chunks[1], Table)
+    assert isinstance(chunks[2], CompositeElement)
+    # -- table text is unchanged --
+    assert chunks[1].text == table_text.strip()
+
+
+def test_skip_table_chunking_does_not_affect_text_element_chunking():
+    long_text = Text("word " * 200)
+    table = Table("small table")
+
+    chunks = chunk_elements(
+        [long_text, table],
+        max_characters=100,
+        skip_table_chunking=True,
+    )
+
+    # -- long text element is still split, table is still isolated --
+    text_chunks = [c for c in chunks if isinstance(c, CompositeElement)]
+    table_chunks = [c for c in chunks if isinstance(c, Table)]
+    assert len(text_chunks) > 1
+    assert len(table_chunks) == 1
+    assert table_chunks[0].text == "small table"
+
+
+# ------------------------------------------------------------------------------------------------
+# UNIT TESTS
+# ------------------------------------------------------------------------------------------------
+
+
+class Describe_chunk_elements:
+    """Unit-test suite for `unstructured.chunking.basic.chunk_elements()` function."""
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"include_orig_elements": True}, True),
+            ({"include_orig_elements": False}, False),
+            ({"include_orig_elements": None}, True),
+            ({}, True),
+        ],
+    )
+    def it_supports_the_include_orig_elements_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_elements_: Mock
+    ):
+        # -- this line would raise if "include_orig_elements" was not an available parameter on
+        # -- `chunk_elements()`.
+        chunk_elements([], **kwargs)
+
+        _, opts = _chunk_elements_.call_args.args
+        assert opts.include_orig_elements is expected_value
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"repeat_table_headers": True}, True),
+            ({"repeat_table_headers": False}, False),
+            ({"repeat_table_headers": None}, True),
+            ({}, True),
+        ],
+    )
+    def it_supports_the_repeat_table_headers_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_elements_: Mock
+    ):
+        # -- this line would raise if "repeat_table_headers" was not an available parameter on
+        # -- `chunk_elements()`.
+        chunk_elements([], **kwargs)
+
+        _, opts = _chunk_elements_.call_args.args
+        assert opts.repeat_table_headers is expected_value
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"skip_table_chunking": True}, True),
+            ({"skip_table_chunking": False}, False),
+            ({"skip_table_chunking": None}, False),
+            ({}, False),
+        ],
+    )
+    def it_supports_the_skip_table_chunking_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_elements_: Mock
+    ):
+        chunk_elements([], **kwargs)
+
+        _, opts = _chunk_elements_.call_args.args
+        assert opts.skip_table_chunking is expected_value
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"isolate_table": True}, True),
+            ({"isolate_table": False}, False),
+            ({"isolate_table": None}, True),
+            ({}, True),
+        ],
+    )
+    def it_supports_the_isolate_table_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_elements_: Mock
+    ):
+        chunk_elements([], **kwargs)
+
+        _, opts = _chunk_elements_.call_args.args
+        assert opts.isolate_table is expected_value
+
+    # -- fixtures --------------------------------------------------------------------------------
+
+    @pytest.fixture()
+    def _chunk_elements_(self, request: FixtureRequest):
+        return function_mock(request, "unstructured.chunking.basic._chunk_elements")

test_unstructured/chunking/test_dispatch.py

@@ -0,0 +1,92 @@
+# pyright: reportPrivateUsage=false
+
+"""Unit-test suite for the `unstructured.chunking.dispatch` module."""
+
+from __future__ import annotations
+
+from typing import Any, Iterable, Optional
+
+import pytest
+
+from unstructured.chunking import add_chunking_strategy, register_chunking_strategy
+from unstructured.chunking.dispatch import _ChunkerSpec, chunk
+from unstructured.documents.elements import CompositeElement, Element, Text
+
+
+class Describe_add_chunking_strategy:
+    """Unit-test suite for `unstructured.chunking.add_chunking_strategy()` decorator."""
+
+    def it_dispatches_the_partitioned_elements_to_the_indicated_chunker(self):
+        decorated_partitioner = add_chunking_strategy(partition_this)
+
+        chunks = decorated_partitioner(chunking_strategy="basic")
+
+        assert chunks == [CompositeElement("Lorem ipsum.\n\nSit amet.")]
+
+    def but_it_skips_dispatch_when_no_chunking_strategy_is_specified(self):
+        decorated_partitioner = add_chunking_strategy(partition_this)
+
+        elements = decorated_partitioner()
+
+        assert elements == [Text("Lorem ipsum."), Text("Sit amet.")]
+
+
+class Describe_chunk:
+    """Unit-test suite for `unstructured.chunking.dispatch.chunk()` function."""
+
+    def it_dispatches_to_the_chunker_registered_for_the_chunking_strategy(self):
+        register_chunking_strategy("by_something_else", chunk_by_something_else)
+        kwargs = {
+            "max_characters": 750,
+            # -- unused kwargs shouldn't cause a problem; in general `kwargs` will contain all
+            # -- keyword arguments used in the partitioning call.
+            "foo": "bar",
+        }
+
+        chunks = chunk([Text("Lorem"), Text("Ipsum")], "by_something_else", **kwargs)
+
+        assert chunks == [
+            CompositeElement("chunked 2 elements with `(max_characters=750, whizbang=None)`")
+        ]
+
+    def it_raises_when_the_requested_chunking_strategy_is_not_registered(self):
+        with pytest.raises(
+            ValueError,
+            match="unrecognized chunking strategy 'foobar'",
+        ):
+            chunk(elements=[], chunking_strategy="foobar")
+
+
+class Describe_ChunkerSpec:
+    """Unit-test suite for `unstructured.chunking.dispatch._ChunkerSpec` objects."""
+
+    def it_provides_access_to_the_chunking_function(self):
+        spec = _ChunkerSpec(chunk_by_something_else)
+        assert spec.chunker is chunk_by_something_else
+
+    def it_knows_which_keyword_args_the_chunking_function_can_accept(self):
+        spec = _ChunkerSpec(chunk_by_something_else)
+        assert spec.kw_arg_names == ("max_characters", "whizbang")
+
+
+# -- MODULE-LEVEL FIXTURES -----------------------------------------------------------------------
+
+
+def chunk_by_something_else(
+    elements: Iterable[Element],
+    max_characters: Optional[int] = None,
+    whizbang: Optional[float] = None,
+) -> list[Element]:
+    """A "fake" minimal chunker suitable for use in tests."""
+    els = list(elements)
+    return [
+        CompositeElement(
+            f"chunked {len(els)} elements with"
+            f" `(max_characters={max_characters}, whizbang={whizbang})`"
+        )
+    ]
+
+
+def partition_this(**kwargs: Any) -> list[Element]:
+    """A fake partitioner."""
+    return [Text("Lorem ipsum."), Text("Sit amet.")]

test_unstructured/chunking/test_html_output.py

@@ -0,0 +1,90 @@
+from functools import partial
+
+import pytest
+
+from unstructured.chunking.basic import chunk_elements
+from unstructured.chunking.title import chunk_by_title
+from unstructured.documents.elements import ElementMetadata, NarrativeText, Text, Title
+
+
+@pytest.fixture(params=[chunk_elements, partial(chunk_by_title, combine_text_under_n_chars=0)])
+def chunking_fn(request):
+    return request.param
+
+
+def test_combining_html_metadata_when_multiple_elements_in_composite_element(chunking_fn):
+    metadata_1 = '<h1 class="Title" id="1">Header </h1>'
+    metadata_2 = '<time class="CalendarDate" id="2">Date: October 30, 2023 </time>'
+    metadata_3 = (
+        '<form class="Form" id="3"> '
+        '<label class="FormField" for="company-name" id="4">Form field name </label>'
+        '<input class="FormFieldValue" id="5" value="Example value" />'
+        "</form>"
+    )
+    combined_metadata = " ".join([metadata_1, metadata_2, metadata_3])
+
+    elements = [
+        Title(text="Header", metadata=ElementMetadata(text_as_html=metadata_1)),
+        Text(text="Date: October 30, 2023", metadata=ElementMetadata(text_as_html=metadata_2)),
+        Text(
+            text="Form field name Example value", metadata=ElementMetadata(text_as_html=metadata_3)
+        ),
+    ]
+    chunks = chunking_fn(elements)
+    assert len(chunks) == 1
+    assert chunks[0].metadata.text_as_html == combined_metadata
+
+
+def test_combining_html_metadata_with_nested_relationship_between_elements(chunking_fn):
+    """
+    Ground truth
+    <Document>
+        <Page>
+            <Section>
+                <p>First</p>
+                <p>Second</p>
+            </Section>
+        </Page>
+    </Document>
+    Elements: Document, Page, Section, Paragraph, Paragraph
+    Chunk 1: Document, Page, Section, Paragraph
+
+    Chunk 2:
+        Paragraph
+    """
+
+    metadata_1 = '<div class="Section" id="1" />'
+    metadata_2 = '<p class="Paragraph" id="2">First </p>'
+    metadata_3 = '<p class="Paragraph" id="3">Second </p>'
+
+    elements = [
+        Text(text="", metadata=ElementMetadata(text_as_html=metadata_1)),
+        NarrativeText(
+            text="First", metadata=ElementMetadata(text_as_html=metadata_2, parent_id="1")
+        ),
+        NarrativeText(
+            text="Second", metadata=ElementMetadata(text_as_html=metadata_3, parent_id="1")
+        ),
+    ]
+    chunks = chunking_fn(elements, max_characters=6)
+    assert len(chunks) == 2
+    assert chunks[0].text == "First"
+    assert chunks[1].text == "Second"
+
+    assert chunks[0].metadata.text_as_html == metadata_1 + " " + metadata_2
+    assert chunks[1].metadata.text_as_html == metadata_3
+
+
+def test_html_metadata_exist_in_both_element_when_text_is_split(chunking_fn):
+    """Mimic behaviour of elements with non-html metadata"""
+    metadata_1 = '<h1 class="Title" id="1">Header </h1>'
+    elements = [
+        Title(text="Header", metadata=ElementMetadata(text_as_html=metadata_1)),
+    ]
+    chunks = chunking_fn(elements, max_characters=3)
+    assert len(chunks) == 2
+
+    assert chunks[0].text == "Hea"
+    assert chunks[1].text == "der"
+    assert chunks[0].metadata.text_as_html == '<h1 class="Title" id="1">Header </h1>'
+    assert chunks[1].metadata.text_as_html == '<h1 class="Title" id="1">Header </h1>'

test_unstructured/chunking/test_table_isolation.py

@@ -0,0 +1,357 @@
+# pyright: reportPrivateUsage=false
+
+"""Regression and characterization tests for table isolation during chunking (issue #3921).
+
+`Table` (and `TableChunk`, which subclasses `Table`) must never share a pre-chunk with unrelated
+text elements, so downstream logic can emit standalone table chunks instead of `CompositeElement`
+wrapping mixed content.
+"""
+
+from __future__ import annotations
+
+from typing import Callable
+
+import pytest
+
+from unstructured.chunking.base import (
+    ChunkingOptions,
+    PreChunk,
+    PreChunkBuilder,
+    PreChunkCombiner,
+    PreChunker,
+)
+from unstructured.chunking.basic import chunk_elements
+from unstructured.chunking.title import chunk_by_title
+from unstructured.documents.elements import (
+    CompositeElement,
+    ElementMetadata,
+    Table,
+    TableChunk,
+    Text,
+    Title,
+)
+
+
+class DescribeTableIsolationPreChunkBuilder:
+    """`PreChunkBuilder` must keep every table-family element in its own pre-chunk."""
+
+    @pytest.mark.parametrize(
+        ("preamble", "make_table"),
+        [
+            (Text("Short preamble."), lambda: Table("H\nC")),
+            (Text("Short preamble."), lambda: TableChunk(text="x", metadata=ElementMetadata())),
+        ],
+    )
+    def it_refuses_to_append_a_table_after_any_other_element(
+        self, preamble: Text, make_table: Callable[[], Table]
+    ):
+        builder = PreChunkBuilder(opts=ChunkingOptions(max_characters=500))
+        builder.add_element(preamble)
+
+        assert not builder.will_fit(make_table())
+
+    @pytest.mark.parametrize(
+        "trailing",
+        [
+            Text("Follow-up paragraph."),
+            Title("Next section"),
+        ],
+    )
+    def it_refuses_to_append_a_non_table_after_a_table(self, trailing: Text | Title):
+        builder = PreChunkBuilder(opts=ChunkingOptions(max_characters=500))
+        builder.add_element(Table("Heading\nCell"))
+
+        assert not builder.will_fit(trailing)
+
+    def it_allows_only_a_table_when_the_builder_is_empty(self):
+        builder = PreChunkBuilder(opts=ChunkingOptions())
+        assert builder.will_fit(Table("Heading\nCell text"))
+
+    def it_allows_text_after_flush_even_if_previous_pre_chunk_was_a_table(self):
+        opts = ChunkingOptions(max_characters=200)
+        builder = PreChunkBuilder(opts=opts)
+        builder.add_element(Table("Heading\nCell text"))
+        list(builder.flush())  # clears builder state
+
+        assert builder.will_fit(Text("Fresh start after table pre-chunk."))
+
+
+class DescribeTableIsolationPreChunkStream:
+    """End-to-end pre-chunk segmentation from `PreChunker.iter_pre_chunks`."""
+
+    def it_emits_a_table_only_pre_chunk_between_text_blocks(self):
+        elements = [
+            Title("Section A"),
+            Text("Narrative before the table."),
+            Table("Col1\nCell A"),
+            Text("Narrative after the table."),
+        ]
+        # -- `new_after_n_chars=0` forces one element per pre-chunk (deterministic layout) --
+        opts = ChunkingOptions(max_characters=500, new_after_n_chars=0)
+
+        pre_chunks = list(PreChunker.iter_pre_chunks(elements, opts=opts))
+
+        assert len(pre_chunks) == 4
+        assert pre_chunks[0]._elements == [Title("Section A")]
+        assert pre_chunks[1]._elements == [Text("Narrative before the table.")]
+        assert pre_chunks[2]._elements == [Table("Col1\nCell A")]
+        assert pre_chunks[3]._elements == [Text("Narrative after the table.")]
+
+    def it_emits_one_pre_chunk_per_table_when_multiple_tables_are_adjacent(self):
+        elements = [
+            Table("T1\nA"),
+            Table("T2\nB"),
+            Text("Closing text."),
+        ]
+        opts = ChunkingOptions(max_characters=500)
+
+        pre_chunks = list(PreChunker.iter_pre_chunks(elements, opts=opts))
+
+        assert len(pre_chunks) == 3
+        assert pre_chunks[0]._elements == [Table("T1\nA")]
+        assert pre_chunks[1]._elements == [Table("T2\nB")]
+        assert pre_chunks[2]._elements == [Text("Closing text.")]
+
+
+class DescribeTableIsolationPreChunkCombiner:
+    """`PreChunkCombiner` must not stitch table pre-chunks onto text neighbors."""
+
+    def it_keeps_a_table_pre_chunk_separate_when_combining_is_enabled(self):
+        opts = ChunkingOptions(max_characters=500, combine_text_under_n_chars=500)
+        stream = [
+            PreChunk([Text("Hello world.")], overlap_prefix="", opts=opts),
+            PreChunk([Table("H\nC")], overlap_prefix="", opts=opts),
+            PreChunk([Text("Goodbye world.")], overlap_prefix="", opts=opts),
+        ]
+
+        combined = list(PreChunkCombiner(stream, opts=opts).iter_combined_pre_chunks())
+
+        assert len(combined) == 3
+        assert combined[0]._elements == [Text("Hello world.")]
+        assert combined[1]._elements == [Table("H\nC")]
+        assert combined[2]._elements == [Text("Goodbye world.")]
+
+
+class DescribeTableIsolationOrderingGuarantees:
+    """Invariants that should hold for any future refactor of table isolation."""
+
+    def it_preserves_global_element_order_in_pre_chunks(self):
+        elements = [
+            Text("alpha"),
+            Table("T\n1"),
+            Text("beta"),
+            Table("T\n2"),
+            Title("gamma"),
+        ]
+        opts = ChunkingOptions(max_characters=500, new_after_n_chars=0)
+        flat = [e for pc in PreChunker.iter_pre_chunks(elements, opts=opts) for e in pc._elements]
+
+        assert flat == elements
+
+    def it_preserves_global_element_order_in_chunk_elements_output(self):
+        elements = [
+            Title("Intro"),
+            Text("Body before."),
+            Table("K\nV"),
+            Text("Body after."),
+        ]
+        chunks = chunk_elements(elements, max_characters=500, new_after_n_chars=0)
+        # -- flatten chunk element categories in stream order (table chunks are atomic) --
+        categories = [c.category for c in chunks]
+
+        assert categories == [
+            "CompositeElement",
+            "CompositeElement",
+            "Table",
+            "CompositeElement",
+        ]
+
+
+class DescribeTableIsolationChunkElements:
+    """`chunk_elements` (basic strategy) should emit `Table`/`TableChunk`, not mixed composites."""
+
+    def it_does_not_wrap_a_table_and_surrounding_text_in_one_composite_element(self):
+        elements = [
+            Title("Report"),
+            Text("Short intro."),
+            Table("Key\nValue"),
+            Text("Short outro."),
+        ]
+        chunks = chunk_elements(
+            elements,
+            max_characters=500,
+            new_after_n_chars=0,
+        )
+
+        assert len(chunks) == 4
+        assert isinstance(chunks[0], CompositeElement)
+        assert isinstance(chunks[1], CompositeElement)
+        assert isinstance(chunks[2], Table)
+        assert isinstance(chunks[3], CompositeElement)
+        assert "Key" in chunks[2].text or "Value" in chunks[2].text
+
+    def it_yields_distinct_chunks_for_two_tables_in_a_row(self):
+        elements = [
+            Table("T1\nA"),
+            Table("T2\nB"),
+        ]
+        chunks = chunk_elements(elements, max_characters=500)
+
+        assert len(chunks) == 2
+        assert all(isinstance(c, Table) for c in chunks)
+
+    def it_still_isolates_a_table_even_when_the_window_is_very_large(self):
+        """Regression: isolation is a semantic rule, not a size heuristic."""
+        elements = [
+            Text("x"),
+            Table("tiny"),
+            Text("y"),
+        ]
+        chunks = chunk_elements(elements, max_characters=50_000, new_after_n_chars=10_000)
+
+        table_chunks = [c for c in chunks if isinstance(c, Table)]
+        composite_chunks = [c for c in chunks if isinstance(c, CompositeElement)]
+
+        assert len(table_chunks) == 1
+        assert len(composite_chunks) == 2
+        assert "tiny" in table_chunks[0].text
+
+    def it_never_produces_a_composite_element_that_lists_a_table_in_orig_elements(
+        self,
+    ):
+        """`CompositeElement` chunks come from `_Chunker`, not `_TableChunker`."""
+        elements = [
+            Text("preamble"),
+            Table("H\nC"),
+            Text("post"),
+        ]
+        chunks = chunk_elements(
+            elements,
+            max_characters=400,
+            new_after_n_chars=0,
+            include_orig_elements=True,
+        )
+
+        composites = [c for c in chunks if isinstance(c, CompositeElement)]
+        for comp in composites:
+            orig = comp.metadata.orig_elements or []
+            assert not any(isinstance(e, Table) for e in orig)
+
+
+class DescribeTableIsolationDisabled:
+    """When `isolate_table=False`, the pre-#4307 behavior is restored."""
+
+    def it_lets_a_table_share_a_pre_chunk_with_adjacent_text_when_disabled(self):
+        opts = ChunkingOptions(max_characters=500, isolate_table=False)
+        builder = PreChunkBuilder(opts=opts)
+        builder.add_element(Text("Short preamble."))
+
+        assert builder.will_fit(Table("Heading\nCell text"))
+
+    def it_lets_text_follow_a_table_in_the_same_pre_chunk_when_disabled(self):
+        opts = ChunkingOptions(max_characters=500, isolate_table=False)
+        builder = PreChunkBuilder(opts=opts)
+        builder.add_element(Table("Heading\nCell text"))
+
+        assert builder.will_fit(Text("Follow-up paragraph."))
+
+    def it_combines_a_table_pre_chunk_with_text_neighbors_when_disabled(self):
+        opts = ChunkingOptions(
+            max_characters=500, combine_text_under_n_chars=500, isolate_table=False
+        )
+        stream = [
+            PreChunk([Text("Hello world.")], overlap_prefix="", opts=opts),
+            PreChunk([Table("H\nC")], overlap_prefix="", opts=opts),
+            PreChunk([Text("Goodbye world.")], overlap_prefix="", opts=opts),
+        ]
+
+        combined = list(PreChunkCombiner(stream, opts=opts).iter_combined_pre_chunks())
+
+        assert len(combined) == 1
+        assert combined[0]._elements == [
+            Text("Hello world."),
+            Table("H\nC"),
+            Text("Goodbye world."),
+        ]
+
+    def it_wraps_a_table_into_a_composite_element_with_neighbors_when_disabled(self):
+        """End-to-end: opting out collapses tiny table + text into one CompositeElement."""
+        elements = [
+            Text("preamble"),
+            Table("H\nC"),
+            Text("post"),
+        ]
+        chunks = chunk_elements(
+            elements,
+            max_characters=500,
+            isolate_table=False,
+        )
+
+        assert len(chunks) == 1
+        assert isinstance(chunks[0], CompositeElement)
+        text = chunks[0].text
+        assert "preamble" in text
+        assert "post" in text
+
+
+class DescribeTableIsolationOverlapAll:
+    """With overlap_all=True, overlap must not cross table / narrative boundaries."""
+
+    def it_does_not_prefix_table_chunk_with_prior_text_overlap(self):
+        """Regression: pre-chunk overlap_tail must not become table's overlap_prefix."""
+        elements = [Text("Alpha beta gamma delta."), Table("H\nC")]
+        chunks = chunk_elements(
+            elements,
+            max_characters=500,
+            new_after_n_chars=0,
+            overlap=5,
+            overlap_all=True,
+        )
+
+        table_chunks = [c for c in chunks if isinstance(c, Table)]
+        assert len(table_chunks) == 1
+        t = table_chunks[0].text or ""
+        assert "Alpha" not in t
+        assert "elta" not in t  # tail of "delta" leaked in buggy overlap
+
+    def it_does_not_prefix_text_after_table_with_table_overlap(self):
+        elements = [Table("H\nC"), Text("Omega sigma tau upsilon.")]
+        chunks = chunk_elements(
+            elements,
+            max_characters=500,
+            new_after_n_chars=0,
+            overlap=5,
+            overlap_all=True,
+        )
+
+        composites = [c for c in chunks if isinstance(c, CompositeElement)]
+        assert len(composites) == 1
+        assert composites[0].text.startswith("Omega")
+        assert "H" not in composites[0].text[:20]
+
+    def it_chunk_by_title_respects_same_overlap_boundaries(self):
+        elements = [
+            Title("Section"),
+            Text("Alpha beta gamma delta."),
+            Table("H\nC"),
+            Text("Omega sigma tau upsilon."),
+        ]
+        chunks = chunk_by_title(
+            elements,
+            max_characters=500,
+            new_after_n_chars=0,
+            overlap=5,
+            overlap_all=True,
+            combine_text_under_n_chars=0,
+        )
+
+        table_chunks = [c for c in chunks if isinstance(c, Table)]
+        assert len(table_chunks) == 1
+        assert "Alpha" not in (table_chunks[0].text or "")
+        assert "elta" not in (table_chunks[0].text or "")
+
+        omega_composites = [
+            c for c in chunks if isinstance(c, CompositeElement) and "Omega" in (c.text or "")
+        ]
+        assert len(omega_composites) == 1
+        assert omega_composites[0].text.startswith("Omega")

test_unstructured/chunking/test_title.py

@@ -0,0 +1,890 @@
+# pyright: reportPrivateUsage=false
+
+"""Test suite for the `unstructured.chunking.title` module."""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+import pytest
+
+from test_unstructured.unit_utils import FixtureRequest, Mock, function_mock, input_path
+from unstructured.chunking.base import CHUNK_MULTI_PAGE_DEFAULT
+from unstructured.chunking.title import _ByTitleChunkingOptions, chunk_by_title
+from unstructured.documents.coordinates import CoordinateSystem
+from unstructured.documents.elements import (
+    CheckBox,
+    CompositeElement,
+    CoordinatesMetadata,
+    Element,
+    ElementMetadata,
+    ListItem,
+    Table,
+    TableChunk,
+    Text,
+    Title,
+)
+from unstructured.partition.html import partition_html
+from unstructured.staging.base import elements_from_json
+
+# ================================================================================================
+# INTEGRATION-TESTS
+# ================================================================================================
+# These test `chunk_by_title()` as an integrated whole, calling `chunk_by_title()` and inspecting
+# the outputs.
+# ================================================================================================
+
+
+def test_it_chunks_text_followed_by_table_together_when_both_fit():
+    elements = elements_from_json(input_path("chunking/title_table_200.json"))
+
+    chunks = chunk_by_title(elements, combine_text_under_n_chars=0)
+
+    assert len(chunks) == 2
+    assert isinstance(chunks[0], CompositeElement)
+    assert isinstance(chunks[1], Table)
+
+
+def test_it_chunks_table_followed_by_text_together_when_both_fit():
+    elements = elements_from_json(input_path("chunking/table_text_200.json"))
+
+    # -- disable chunk combining so we test pre-chunking behavior, not chunk-combining --
+    chunks = chunk_by_title(elements, combine_text_under_n_chars=0)
+
+    assert len(chunks) == 2
+    assert isinstance(chunks[0], Table)
+    assert isinstance(chunks[1], CompositeElement)
+
+
+def test_it_splits_oversized_table():
+    elements = elements_from_json(input_path("chunking/table_2000.json"))
+
+    chunks = chunk_by_title(elements)
+
+    assert len(chunks) == 5
+    assert all(isinstance(chunk, TableChunk) for chunk in chunks)
+
+
+def test_skip_table_chunking_passes_oversized_table_through_unchanged():
+    elements = elements_from_json(input_path("chunking/table_2000.json"))
+
+    chunks = chunk_by_title(elements, skip_table_chunking=True)
+
+    assert len(chunks) == 1
+    assert isinstance(chunks[0], Table)
+
+
+def test_skip_table_chunking_does_not_combine_table_with_adjacent_text():
+    table_text = "cell " * 200
+    table = Table(table_text.strip())
+    text_before = Text("Hello world")
+    text_after = Text("Goodbye world")
+
+    chunks = chunk_by_title(
+        [text_before, table, text_after],
+        max_characters=5000,
+        combine_text_under_n_chars=5000,
+        skip_table_chunking=True,
+    )
+
+    assert isinstance(chunks[0], CompositeElement)
+    assert isinstance(chunks[1], Table)
+    assert isinstance(chunks[2], CompositeElement)
+    assert chunks[1].text == table_text.strip()
+
+
+def test_it_repeats_table_headers_by_default_but_can_opt_out():
+    table_html = (
+        "<table>"
+        "<thead>"
+        "<tr><th>Header A</th><th>Header B</th></tr>"
+        "<tr><th>Subhead A</th><th>Subhead B</th></tr>"
+        "</thead>"
+        "<tbody>"
+        "<tr><td>Body 1</td><td>Alpha</td></tr>"
+        "<tr><td>Body 2</td><td>Bravo</td></tr>"
+        "<tr><td>Body 3</td><td>Charlie</td></tr>"
+        "<tr><td>Body 4</td><td>Delta</td></tr>"
+        "</tbody>"
+        "</table>"
+    )
+    table_text = (
+        "Header A Header B\n"
+        "Subhead A Subhead B\n"
+        "Body 1 Alpha\n"
+        "Body 2 Bravo\n"
+        "Body 3 Charlie\n"
+        "Body 4 Delta"
+    )
+    table = Table(table_text, metadata=ElementMetadata(text_as_html=table_html))
+
+    repeated_header_chunks = chunk_by_title(
+        [table], combine_text_under_n_chars=0, max_characters=55
+    )
+    opt_out_chunks = chunk_by_title(
+        [table],
+        combine_text_under_n_chars=0,
+        max_characters=55,
+        repeat_table_headers=False,
+    )
+
+    assert len(repeated_header_chunks) == 4
+    assert all(isinstance(chunk, TableChunk) for chunk in repeated_header_chunks)
+    assert [chunk.text for chunk in repeated_header_chunks] == [
+        "Header A Header B Subhead A Subhead B Body 1 Alpha",
+        "Header A Header B Subhead A Subhead B Body 2 Bravo",
+        "Header A Header B Subhead A Subhead B Body 3 Charlie",
+        "Header A Header B Subhead A Subhead B Body 4 Delta",
+    ]
+    assert [chunk.text for chunk in opt_out_chunks] == [
+        "Header A Header B Subhead A Subhead B Body 1 Alpha",
+        "Body 2 Bravo Body 3 Charlie Body 4 Delta",
+    ]
+
+
+def test_it_starts_new_chunk_for_table_after_full_text_chunk():
+    elements = elements_from_json(input_path("chunking/long_text_table_200.json"))
+
+    chunks = chunk_by_title(elements, max_characters=250)
+
+    assert len(chunks) == 2
+    assert [type(chunk) for chunk in chunks] == [CompositeElement, Table]
+
+
+def test_it_starts_new_chunk_for_text_after_full_table_chunk():
+    elements = elements_from_json(input_path("chunking/full_table_long_text_250.json"))
+
+    chunks = chunk_by_title(elements, max_characters=250)
+
+    assert len(chunks) == 2
+    assert [type(chunk) for chunk in chunks] == [Table, CompositeElement]
+
+
+def test_it_splits_a_large_text_element_into_multiple_chunks():
+    elements: list[Element] = [
+        Title("Introduction"),
+        Text(
+            "Lorem ipsum dolor sit amet consectetur adipiscing elit. In rhoncus ipsum sed lectus"
+            " porta volutpat.",
+        ),
+    ]
+
+    chunks = chunk_by_title(elements, max_characters=50)
+
+    assert chunks == [
+        CompositeElement("Introduction"),
+        CompositeElement("Lorem ipsum dolor sit amet consectetur adipiscing"),
+        CompositeElement("elit. In rhoncus ipsum sed lectus porta volutpat."),
+    ]
+
+
+def test_it_splits_elements_by_title_and_table():
+    elements: list[Element] = [
+        Title("A Great Day"),
+        Text("Today is a great day."),
+        Text("It is sunny outside."),
+        Table("Heading\nCell text"),
+        Title("An Okay Day"),
+        Text("Today is an okay day."),
+        Text("It is rainy outside."),
+        Title("A Bad Day"),
+        Text("Today is a bad day."),
+        Text("It is storming outside."),
+        CheckBox(),
+    ]
+
+    chunks = chunk_by_title(elements, combine_text_under_n_chars=0, include_orig_elements=True)
+
+    assert len(chunks) == 4
+    # --
+    chunk = chunks[0]
+    assert isinstance(chunk, CompositeElement)
+    assert chunk.metadata.orig_elements == [
+        Title("A Great Day"),
+        Text("Today is a great day."),
+        Text("It is sunny outside."),
+    ]
+    # --
+    chunk = chunks[1]
+    assert isinstance(chunk, Table)
+    assert len(chunk.metadata.orig_elements) == 1
+    assert isinstance(chunk.metadata.orig_elements[0], Table)
+    assert chunk.metadata.orig_elements[0].text == "Heading\nCell text"
+    # --
+    chunk = chunks[2]
+    assert isinstance(chunk, CompositeElement)
+    assert chunk.metadata.orig_elements == [
+        Title("An Okay Day"),
+        Text("Today is an okay day."),
+        Text("It is rainy outside."),
+    ]
+    # --
+    chunk = chunks[3]
+    assert isinstance(chunk, CompositeElement)
+    assert chunk.metadata.orig_elements == [
+        Title("A Bad Day"),
+        Text("Today is a bad day."),
+        Text("It is storming outside."),
+        CheckBox(),
+    ]
+
+
+def test_chunk_by_title():
+    elements: list[Element] = [
+        Title("A Great Day", metadata=ElementMetadata(emphasized_text_contents=["Day"])),
+        Text("Today is a great day.", metadata=ElementMetadata(emphasized_text_contents=["day"])),
+        Text("It is sunny outside."),
+        Table("Heading\nCell text"),
+        Title("An Okay Day"),
+        Text("Today is an okay day."),
+        Text("It is rainy outside."),
+        Title("A Bad Day"),
+        Text("Today is a bad day."),
+        Text("It is storming outside."),
+        CheckBox(),
+    ]
+
+    chunks = chunk_by_title(elements, combine_text_under_n_chars=0, include_orig_elements=False)
+
+    assert len(chunks) == 4
+    assert chunks[0] == CompositeElement(
+        "A Great Day\n\nToday is a great day.\n\nIt is sunny outside."
+    )
+    assert isinstance(chunks[1], Table)
+    assert chunks[1].text == "Heading\nCell text"
+    assert chunks[2] == CompositeElement(
+        "An Okay Day\n\nToday is an okay day.\n\nIt is rainy outside."
+    )
+    assert chunks[3] == CompositeElement(
+        "A Bad Day\n\nToday is a bad day.\n\nIt is storming outside."
+    )
+    assert chunks[0].metadata == ElementMetadata(emphasized_text_contents=["Day", "day"])
+
+
+def test_chunk_by_title_separates_by_page_number():
+    elements: list[Element] = [
+        Title("A Great Day", metadata=ElementMetadata(page_number=1)),
+        Text("Today is a great day.", metadata=ElementMetadata(page_number=2)),
+        Text("It is sunny outside.", metadata=ElementMetadata(page_number=2)),
+        Table("Heading\nCell text"),
+        Title("An Okay Day"),
+        Text("Today is an okay day."),
+        Text("It is rainy outside."),
+        Title("A Bad Day"),
+        Text("Today is a bad day."),
+        Text("It is storming outside."),
+        CheckBox(),
+    ]
+    chunks = chunk_by_title(elements, multipage_sections=False, combine_text_under_n_chars=0)
+
+    assert len(chunks) == 5
+    assert chunks[0] == CompositeElement("A Great Day")
+    assert chunks[1] == CompositeElement("Today is a great day.\n\nIt is sunny outside.")
+    assert isinstance(chunks[2], Table)
+    assert chunks[2].text == "Heading\nCell text"
+    assert chunks[3] == CompositeElement(
+        "An Okay Day\n\nToday is an okay day.\n\nIt is rainy outside."
+    )
+    assert chunks[4] == CompositeElement(
+        "A Bad Day\n\nToday is a bad day.\n\nIt is storming outside."
+    )
+
+
+def test_chuck_by_title_respects_multipage():
+    elements: list[Element] = [
+        Title("A Great Day", metadata=ElementMetadata(page_number=1)),
+        Text("Today is a great day.", metadata=ElementMetadata(page_number=2)),
+        Text("It is sunny outside.", metadata=ElementMetadata(page_number=2)),
+        Table("Heading\nCell text"),
+        Title("An Okay Day"),
+        Text("Today is an okay day."),
+        Text("It is rainy outside."),
+        Title("A Bad Day"),
+        Text("Today is a bad day."),
+        Text("It is storming outside."),
+        CheckBox(),
+    ]
+    chunks = chunk_by_title(elements, multipage_sections=True, combine_text_under_n_chars=0)
+    assert len(chunks) == 4
+    assert chunks[0] == CompositeElement(
+        "A Great Day\n\nToday is a great day.\n\nIt is sunny outside."
+    )
+    assert isinstance(chunks[1], Table)
+    assert chunks[1].text == "Heading\nCell text"
+    assert chunks[2] == CompositeElement(
+        "An Okay Day\n\nToday is an okay day.\n\nIt is rainy outside."
+    )
+    assert chunks[3] == CompositeElement(
+        "A Bad Day\n\nToday is a bad day.\n\nIt is storming outside."
+    )
+
+
+def test_chunk_by_title_groups_across_pages():
+    elements: list[Element] = [
+        Title("A Great Day", metadata=ElementMetadata(page_number=1)),
+        Text("Today is a great day.", metadata=ElementMetadata(page_number=2)),
+        Text("It is sunny outside.", metadata=ElementMetadata(page_number=2)),
+        Table("Heading\nCell text"),
+        Title("An Okay Day"),
+        Text("Today is an okay day."),
+        Text("It is rainy outside."),
+        Title("A Bad Day"),
+        Text("Today is a bad day."),
+        Text("It is storming outside."),
+        CheckBox(),
+    ]
+    chunks = chunk_by_title(elements, multipage_sections=True, combine_text_under_n_chars=0)
+
+    assert len(chunks) == 4
+    assert chunks[0] == CompositeElement(
+        "A Great Day\n\nToday is a great day.\n\nIt is sunny outside."
+    )
+    assert isinstance(chunks[1], Table)
+    assert chunks[1].text == "Heading\nCell text"
+    assert chunks[2] == CompositeElement(
+        "An Okay Day\n\nToday is an okay day.\n\nIt is rainy outside."
+    )
+    assert chunks[3] == CompositeElement(
+        "A Bad Day\n\nToday is a bad day.\n\nIt is storming outside."
+    )
+
+
+def test_add_chunking_strategy_on_partition_html():
+    filename = "example-docs/example-10k-1p.html"
+    chunk_elements = partition_html(filename, chunking_strategy="by_title")
+    elements = partition_html(filename)
+    chunks = chunk_by_title(elements)
+    assert chunk_elements != elements
+    assert chunk_elements == chunks
+
+
+def test_add_chunking_strategy_respects_max_characters():
+    filename = "example-docs/example-10k-1p.html"
+    chunk_elements = partition_html(
+        filename,
+        chunking_strategy="by_title",
+        combine_text_under_n_chars=0,
+        new_after_n_chars=50,
+        max_characters=100,
+    )
+    elements = partition_html(filename)
+    chunks = chunk_by_title(
+        elements,
+        combine_text_under_n_chars=0,
+        new_after_n_chars=50,
+        max_characters=100,
+    )
+
+    for chunk in chunks:
+        assert isinstance(chunk, Text)
+        assert len(chunk.text) <= 100
+    for chunk_element in chunk_elements:
+        assert isinstance(chunk_element, Text)
+        assert len(chunk_element.text) <= 100
+    assert chunk_elements != elements
+    assert chunk_elements == chunks
+
+
+def test_add_chunking_strategy_forwards_repeat_table_headers():
+    filename = "example-docs/example-10k-1p.html"
+    chunk_elements = partition_html(
+        filename,
+        chunking_strategy="by_title",
+        repeat_table_headers=False,
+    )
+    elements = partition_html(filename)
+    chunks = chunk_by_title(elements, repeat_table_headers=False)
+
+    assert chunk_elements != elements
+    assert chunk_elements == chunks
+
+
+def test_chunk_by_title_drops_detection_class_prob():
+    elements: list[Element] = [
+        Title(
+            "A Great Day",
+            metadata=ElementMetadata(
+                detection_class_prob=0.5,
+            ),
+        ),
+        Text(
+            "Today is a great day.",
+            metadata=ElementMetadata(
+                detection_class_prob=0.62,
+            ),
+        ),
+        Text(
+            "It is sunny outside.",
+            metadata=ElementMetadata(
+                detection_class_prob=0.73,
+            ),
+        ),
+        Title(
+            "An Okay Day",
+            metadata=ElementMetadata(
+                detection_class_prob=0.84,
+            ),
+        ),
+        Text(
+            "Today is an okay day.",
+            metadata=ElementMetadata(
+                detection_class_prob=0.95,
+            ),
+        ),
+    ]
+    chunks = chunk_by_title(elements, combine_text_under_n_chars=0)
+    assert str(chunks[0]) == str(
+        CompositeElement("A Great Day\n\nToday is a great day.\n\nIt is sunny outside."),
+    )
+    assert str(chunks[1]) == str(CompositeElement("An Okay Day\n\nToday is an okay day."))
+
+
+def test_chunk_by_title_drops_extra_metadata():
+    elements: list[Element] = [
+        Title(
+            "A Great Day",
+            metadata=ElementMetadata(
+                coordinates=CoordinatesMetadata(
+                    points=(
+                        (0.1, 0.1),
+                        (0.2, 0.1),
+                        (0.1, 0.2),
+                        (0.2, 0.2),
+                    ),
+                    system=CoordinateSystem(width=0.1, height=0.1),
+                ),
+            ),
+        ),
+        Text(
+            "Today is a great day.",
+            metadata=ElementMetadata(
+                coordinates=CoordinatesMetadata(
+                    points=(
+                        (0.2, 0.2),
+                        (0.3, 0.2),
+                        (0.2, 0.3),
+                        (0.3, 0.3),
+                    ),
+                    system=CoordinateSystem(width=0.2, height=0.2),
+                ),
+            ),
+        ),
+        Text(
+            "It is sunny outside.",
+            metadata=ElementMetadata(
+                coordinates=CoordinatesMetadata(
+                    points=(
+                        (0.3, 0.3),
+                        (0.4, 0.3),
+                        (0.3, 0.4),
+                        (0.4, 0.4),
+                    ),
+                    system=CoordinateSystem(width=0.3, height=0.3),
+                ),
+            ),
+        ),
+        Title(
+            "An Okay Day",
+            metadata=ElementMetadata(
+                coordinates=CoordinatesMetadata(
+                    points=(
+                        (0.3, 0.3),
+                        (0.4, 0.3),
+                        (0.3, 0.4),
+                        (0.4, 0.4),
+                    ),
+                    system=CoordinateSystem(width=0.3, height=0.3),
+                ),
+            ),
+        ),
+        Text(
+            "Today is an okay day.",
+            metadata=ElementMetadata(
+                coordinates=CoordinatesMetadata(
+                    points=(
+                        (0.4, 0.4),
+                        (0.5, 0.4),
+                        (0.4, 0.5),
+                        (0.5, 0.5),
+                    ),
+                    system=CoordinateSystem(width=0.4, height=0.4),
+                ),
+            ),
+        ),
+    ]
+
+    chunks = chunk_by_title(elements, combine_text_under_n_chars=0)
+
+    assert str(chunks[0]) == str(
+        CompositeElement("A Great Day\n\nToday is a great day.\n\nIt is sunny outside."),
+    )
+
+    assert str(chunks[1]) == str(CompositeElement("An Okay Day\n\nToday is an okay day."))
+
+
+def test_it_considers_separator_length_when_pre_chunking():
+    """PreChunker includes length of separators when computing remaining space."""
+    elements: list[Element] = [
+        Title("Chunking Priorities"),  # 19 chars
+        ListItem("Divide text into manageable chunks"),  # 34 chars
+        ListItem("Preserve semantic boundaries"),  # 28 chars
+        ListItem("Minimize mid-text chunk-splitting"),  # 33 chars
+    ]  # 114 chars total but 120 chars with separators
+
+    chunks = chunk_by_title(elements, max_characters=115)
+
+    assert chunks == [
+        CompositeElement(
+            "Chunking Priorities"
+            "\n\nDivide text into manageable chunks"
+            "\n\nPreserve semantic boundaries",
+        ),
+        CompositeElement("Minimize mid-text chunk-splitting"),
+    ]
+
+
+# ================================================================================================
+# UNIT-TESTS
+# ================================================================================================
+# These test individual components in isolation so can exercise all edge cases while still
+# performing well.
+# ================================================================================================
+
+
+class Describe_chunk_by_title:
+    """Unit-test suite for `unstructured.chunking.title.chunk_by_title()` function."""
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"include_orig_elements": True}, True),
+            ({"include_orig_elements": False}, False),
+            ({"include_orig_elements": None}, True),
+            ({}, True),
+        ],
+    )
+    def it_supports_the_include_orig_elements_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_by_title_: Mock
+    ):
+        # -- this line would raise if "include_orig_elements" was not an available parameter on
+        # -- `chunk_by_title()`.
+        chunk_by_title([], **kwargs)
+
+        _, opts = _chunk_by_title_.call_args.args
+        assert opts.include_orig_elements is expected_value
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"repeat_table_headers": True}, True),
+            ({"repeat_table_headers": False}, False),
+            ({"repeat_table_headers": None}, True),
+            ({}, True),
+        ],
+    )
+    def it_supports_the_repeat_table_headers_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_by_title_: Mock
+    ):
+        # -- this line would raise if "repeat_table_headers" was not an available parameter on
+        # -- `chunk_by_title()`.
+        chunk_by_title([], **kwargs)
+
+        _, opts = _chunk_by_title_.call_args.args
+        assert opts.repeat_table_headers is expected_value
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"skip_table_chunking": True}, True),
+            ({"skip_table_chunking": False}, False),
+            ({"skip_table_chunking": None}, False),
+            ({}, False),
+        ],
+    )
+    def it_supports_the_skip_table_chunking_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_by_title_: Mock
+    ):
+        chunk_by_title([], **kwargs)
+
+        _, opts = _chunk_by_title_.call_args.args
+        assert opts.skip_table_chunking is expected_value
+
+    @pytest.mark.parametrize(
+        ("kwargs", "expected_value"),
+        [
+            ({"isolate_table": True}, True),
+            ({"isolate_table": False}, False),
+            ({"isolate_table": None}, True),
+            ({}, True),
+        ],
+    )
+    def it_supports_the_isolate_table_option(
+        self, kwargs: dict[str, Any], expected_value: bool, _chunk_by_title_: Mock
+    ):
+        chunk_by_title([], **kwargs)
+
+        _, opts = _chunk_by_title_.call_args.args
+        assert opts.isolate_table is expected_value
+
+    # -- fixtures --------------------------------------------------------------------------------
+
+    @pytest.fixture()
+    def _chunk_by_title_(self, request: FixtureRequest):
+        return function_mock(request, "unstructured.chunking.title._chunk_by_title")
+
+
+class Describe_ByTitleChunkingOptions:
+    """Unit-test suite for `unstructured.chunking.title._ByTitleChunkingOptions` objects."""
+
+    @pytest.mark.parametrize("n_chars", [-1, -42])
+    def it_rejects_combine_text_under_n_chars_for_n_less_than_zero(self, n_chars: int):
+        with pytest.raises(
+            ValueError,
+            match=f"'combine_text_under_n_chars' argument must be >= 0, got {n_chars}",
+        ):
+            _ByTitleChunkingOptions.new(combine_text_under_n_chars=n_chars)
+
+    def it_accepts_0_for_combine_text_under_n_chars_to_disable_chunk_combining(self):
+        """Specifying `combine_text_under_n_chars=0` is how a caller disables chunk-combining."""
+        opts = _ByTitleChunkingOptions(combine_text_under_n_chars=0)
+        assert opts.combine_text_under_n_chars == 0
+
+    def it_does_not_complain_when_specifying_combine_text_under_n_chars_by_itself(self):
+        """Caller can specify `combine_text_under_n_chars` arg without specifying other options."""
+        try:
+            opts = _ByTitleChunkingOptions(combine_text_under_n_chars=50)
+        except ValueError:
+            pytest.fail("did not accept `combine_text_under_n_chars` as option by itself")
+
+        assert opts.combine_text_under_n_chars == 50
+
+    @pytest.mark.parametrize(
+        ("combine_text_under_n_chars", "max_characters", "expected_hard_max"),
+        [(600, None, 500), (600, 450, 450)],
+    )
+    def it_rejects_combine_text_under_n_chars_greater_than_maxchars(
+        self, combine_text_under_n_chars: int, max_characters: Optional[int], expected_hard_max: int
+    ):
+        """`combine_text_under_n_chars` > `max_characters` can produce behavior confusing to users.
+
+        The behavior is no different from `combine_text_under_n_chars == max_characters`, but if
+        `max_characters` is left to default (500) and `combine_text_under_n_chars` is set to a
+        larger number like 1500 then it can look like chunk-combining isn't working.
+        """
+        with pytest.raises(
+            ValueError,
+            match=(
+                "'combine_text_under_n_chars' argument must not exceed `max_characters` value,"
+                f" got {combine_text_under_n_chars} > {expected_hard_max}"
+            ),
+        ):
+            _ByTitleChunkingOptions.new(
+                max_characters=max_characters, combine_text_under_n_chars=combine_text_under_n_chars
+            )
+
+    def it_does_not_complain_when_specifying_new_after_n_chars_by_itself(self):
+        """Caller can specify `new_after_n_chars` arg without specifying any other options."""
+        try:
+            opts = _ByTitleChunkingOptions.new(new_after_n_chars=200)
+        except ValueError:
+            pytest.fail("did not accept `new_after_n_chars` as option by itself")
+
+        assert opts.soft_max == 200
+
+    @pytest.mark.parametrize(
+        ("multipage_sections", "expected_value"),
+        [(True, True), (False, False), (None, CHUNK_MULTI_PAGE_DEFAULT)],
+    )
+    def it_knows_whether_to_break_chunks_on_page_boundaries(
+        self, multipage_sections: bool, expected_value: bool
+    ):
+        opts = _ByTitleChunkingOptions(multipage_sections=multipage_sections)
+        assert opts.multipage_sections is expected_value
+
+
+# ================================================================================================
+# TOKEN-BASED CHUNKING INTEGRATION TESTS
+# ================================================================================================
+
+
+class DescribeTokenBasedChunking:
+    """Integration tests for token-based chunking with chunk_by_title()."""
+
+    @pytest.fixture
+    def _tiktoken_installed(self):
+        """Skip test if tiktoken is not installed."""
+        pytest.importorskip("tiktoken")
+
+    def it_chunks_elements_by_token_count(self, _tiktoken_installed: None):
+        """Test that chunk_by_title works with max_tokens parameter and respects token limits."""
+        import tiktoken
+
+        enc = tiktoken.get_encoding("cl100k_base")
+        elements: list[Element] = [
+            Title("Introduction"),
+            Text("This is a test document with some text content."),
+            Text("Here is more content that should be chunked appropriately."),
+        ]
+
+        chunks = chunk_by_title(
+            elements,
+            max_tokens=20,
+            tokenizer="cl100k_base",
+            combine_text_under_n_chars=0,
+        )
+
+        # -- verify chunks were created --
+        assert len(chunks) >= 1
+        assert all(isinstance(chunk, (CompositeElement, Table, TableChunk)) for chunk in chunks)
+
+        # -- verify each chunk respects the token limit --
+        for chunk in chunks:
+            token_count = len(enc.encode(chunk.text))
+            assert token_count <= 20, f"Chunk exceeded token limit: {token_count} tokens"
+
+        # -- verify the first chunk contains expected content --
+        assert "Introduction" in chunks[0].text
+
+    def it_respects_new_after_n_tokens_soft_limit(self, _tiktoken_installed: None):
+        """Test that new_after_n_tokens creates smaller chunks with correct content."""
+        import tiktoken
+
+        enc = tiktoken.get_encoding("cl100k_base")
+        elements: list[Element] = [
+            Title("Section 1"),
+            Text("Some text for section one."),
+            Text("More text for section one."),
+            Title("Section 2"),
+            Text("Text for section two."),
+        ]
+
+        # -- with a low soft limit, each section should become its own chunk --
+        chunks = chunk_by_title(
+            elements,
+            max_tokens=100,
+            new_after_n_tokens=10,
+            tokenizer="cl100k_base",
+            combine_text_under_n_chars=0,
+        )
+
+        # -- verify we get multiple chunks --
+        assert len(chunks) >= 2
+
+        # -- verify each chunk respects the hard limit --
+        for chunk in chunks:
+            token_count = len(enc.encode(chunk.text))
+            assert token_count <= 100, f"Chunk exceeded token limit: {token_count} tokens"
+
+        # -- verify content is distributed across chunks --
+        all_text = " ".join(chunk.text for chunk in chunks)
+        assert "Section 1" in all_text
+        assert "Section 2" in all_text
+
+    def it_rejects_max_tokens_and_max_characters_together(self, _tiktoken_installed: None):
+        """Test that specifying both max_tokens and max_characters raises ValueError."""
+        elements: list[Element] = [Text("Some text")]
+
+        with pytest.raises(ValueError, match="mutually exclusive"):
+            chunk_by_title(
+                elements,
+                max_tokens=100,
+                max_characters=500,
+                tokenizer="cl100k_base",
+            )
+
+    def it_rejects_max_tokens_without_tokenizer(self, _tiktoken_installed: None):
+        """Test that max_tokens requires tokenizer to be specified."""
+        elements: list[Element] = [Text("Some text")]
+
+        with pytest.raises(ValueError, match="'tokenizer' is required"):
+            chunk_by_title(elements, max_tokens=100)
+
+    def it_accepts_model_name_as_tokenizer(self, _tiktoken_installed: None):
+        """Test that model names like 'gpt-4' work as tokenizer."""
+        import tiktoken
+
+        # -- gpt-4 uses cl100k_base encoding --
+        enc = tiktoken.encoding_for_model("gpt-4")
+        elements: list[Element] = [
+            Title("Test"),
+            Text("Some test content."),
+        ]
+
+        chunks = chunk_by_title(
+            elements,
+            max_tokens=50,
+            tokenizer="gpt-4",
+            combine_text_under_n_chars=0,
+        )
+
+        assert len(chunks) >= 1
+
+        # -- verify token counts are within limit --
+        for chunk in chunks:
+            token_count = len(enc.encode(chunk.text))
+            assert token_count <= 50, f"Chunk exceeded token limit: {token_count} tokens"
+
+        # -- verify content --
+        assert chunks[0].text == "Test\n\nSome test content."
+
+    def it_splits_oversized_element_respecting_token_limit(self, _tiktoken_installed: None):
+        """Test that oversized elements are split correctly by token count."""
+        import tiktoken
+
+        enc = tiktoken.get_encoding("cl100k_base")
+        # -- create a text that will definitely need to be split (>10 tokens) --
+        long_text = "The quick brown fox jumps over the lazy dog. " * 10
+        elements: list[Element] = [Text(long_text)]
+
+        chunks = chunk_by_title(
+            elements,
+            max_tokens=15,
+            tokenizer="cl100k_base",
+            combine_text_under_n_chars=0,
+        )
+
+        # -- should produce multiple chunks --
+        assert len(chunks) > 1
+
+        # -- each chunk should respect the token limit --
+        for chunk in chunks:
+            token_count = len(enc.encode(chunk.text))
+            assert token_count <= 15, f"Chunk exceeded token limit: {token_count} tokens"
+
+    def it_applies_token_based_overlap_in_split_chunks(self, _tiktoken_installed: None):
+        """Test that overlap in token mode is measured in tokens, not characters."""
+        import tiktoken
+
+        enc = tiktoken.get_encoding("cl100k_base")
+        # -- create text that will be split into multiple chunks --
+        long_text = "Apple banana cherry date elderberry fig grape honeydew. " * 5
+        elements: list[Element] = [Text(long_text)]
+
+        chunks = chunk_by_title(
+            elements,
+            max_tokens=12,
+            tokenizer="cl100k_base",
+            overlap=3,  # 3 tokens of overlap
+            combine_text_under_n_chars=0,
+        )
+
+        # -- should produce multiple chunks --
+        assert len(chunks) >= 2
+
+        # -- verify overlap: second chunk should start with tokens from end of first --
+        if len(chunks) >= 2:
+            # -- the overlap tail from chunk 0 should appear at start of chunk 1 --
+            first_chunk_text = chunks[0].text
+            second_chunk_text = chunks[1].text
+
+            # -- the overlap should be approximately 3 tokens worth of text --
+            # -- verify second chunk starts with content from end of first chunk --
+            # -- (the overlap mechanism prepends tail to remainder) --
+            assert len(second_chunk_text) > 0
+            # -- verify some content from first chunk appears at start of second --
+            first_chunk_words = first_chunk_text.split()
+            assert any(word in second_chunk_text for word in first_chunk_words[-3:])
+            # -- verify each chunk respects limits --
+            for chunk in chunks:
+                token_count = len(enc.encode(chunk.text))
+                assert token_count <= 12, f"Chunk exceeded token limit: {token_count} tokens"