chore: import upstream snapshot with attribution

test_unstructured/partition/html/test_convert.py

@@ -0,0 +1,511 @@
+# pyright: reportPrivateUsage=false, reportUnknownMemberType=false, reportOptionalMemberAccess=false
+# pyright: reportAttributeAccessIssue=false, reportUnknownLambdaType=false
+from collections import defaultdict
+from typing import Any, Optional
+
+import pytest
+from bs4 import BeautifulSoup, Tag
+from pytest_mock import MockerFixture
+
+from unstructured.documents.elements import Element, ElementMetadata, ElementType
+from unstructured.partition.html.convert import (
+    HTML_PARSER,
+    ElementHtml,
+    ImageElementHtml,
+    LinkElementHtml,
+    ListItemElementHtml,
+    TableElementHtml,
+    TextElementHtml,
+    TitleElementHtml,
+    UnorderedListElementHtml,
+    _elements_to_html_tags,
+    _elements_to_html_tags_by_page,
+    _elements_to_html_tags_by_parent,
+    _group_element_children,
+    elements_to_html,
+    group_elements_by_page,
+)
+
+
+class MockElement(Element):
+    def __init__(
+        self,
+        text: str = "",
+        metadata: Optional[ElementMetadata] = None,
+        category: str = "",
+        id: str = "",
+    ) -> None:
+        self.text = text
+        self.metadata = metadata or ElementMetadata()
+        self.category = category
+        self._element_id = id
+
+
+class MockElementMetadata(ElementMetadata):
+    def __init__(
+        self,
+        text_as_html: Optional[str] = None,
+        category_depth: Optional[int] = None,
+        image_base64: Optional[str] = None,
+        image_mime_type: Optional[str] = None,
+        url: Optional[str] = None,
+        parent_id: Optional[str] = None,
+        page_number: Optional[int] = None,
+    ) -> None:
+        self.text_as_html = text_as_html
+        self.category_depth = category_depth
+        self.image_base64 = image_base64
+        self.image_mime_type = image_mime_type
+        self.url = url
+        self.parent_id = parent_id
+        self.page_number = page_number
+
+
+@pytest.fixture
+def mock_element() -> MockElement:
+    metadata = MockElementMetadata(text_as_html="<p>Test Text</p>")
+    return MockElement(text="Test Text", metadata=metadata, category="test-category", id="test-id")
+
+
+@pytest.fixture
+def element_html(mock_element: MockElement) -> ElementHtml:
+    return ElementHtml(mock_element)
+
+
+@pytest.fixture
+def title_element_html(mock_element: MockElement) -> TitleElementHtml:
+    metadata = MockElementMetadata(text_as_html="<p>Test HTML</p>")
+    MockElement(text="Test Text", metadata=metadata, category="test-category", id="test-id")
+    return TitleElementHtml(mock_element)
+
+
+@pytest.fixture
+def image_element_html(mock_element: MockElement) -> ImageElementHtml:
+    return ImageElementHtml(mock_element)
+
+
+@pytest.fixture
+def table_element_html(mock_element: MockElement) -> TableElementHtml:
+    return TableElementHtml(mock_element)
+
+
+@pytest.fixture
+def link_element_html(mock_element: MockElement) -> LinkElementHtml:
+    return LinkElementHtml(mock_element)
+
+
+@pytest.fixture
+def unordered_list_element_html(mock_element: MockElement) -> UnorderedListElementHtml:
+    return UnorderedListElementHtml(mock_element)
+
+
+@pytest.fixture
+def elements_html() -> list[ElementHtml]:
+    return [
+        ListItemElementHtml(
+            MockElement(
+                text="Test List Item",
+                metadata=MockElementMetadata(page_number=1),
+                category=ElementType.LIST_ITEM,
+                id="test-element-1",
+            )
+        ),
+        TextElementHtml(
+            MockElement(
+                text="Test Text",
+                metadata=MockElementMetadata(page_number=None),
+                category=ElementType.TEXT,
+                id="test-element-2",
+            )
+        ),
+        TextElementHtml(
+            MockElement(
+                text="Test Text",
+                metadata=MockElementMetadata(page_number=2),
+                category=ElementType.TEXT,
+                id="test-element-3",
+            )
+        ),
+        ListItemElementHtml(
+            MockElement(
+                text="Test List Item",
+                metadata=MockElementMetadata(parent_id="test-element-3", page_number=2),
+                category=ElementType.LIST_ITEM,
+                id="test-element-4",
+            )
+        ),
+        ListItemElementHtml(
+            MockElement(
+                text="Test List Item",
+                metadata=MockElementMetadata(parent_id="test-element-3", page_number=2),
+                category=ElementType.LIST_ITEM,
+                id="test-element-5",
+            )
+        ),
+        TextElementHtml(
+            MockElement(
+                text="Test Text",
+                metadata=MockElementMetadata(page_number=3),
+                category=ElementType.TEXT,
+                id="test-element-6",
+            )
+        ),
+        ListItemElementHtml(
+            MockElement(
+                text="Test List Item Other",
+                metadata=MockElementMetadata(parent_id="test-element-6", page_number=3),
+                category=ElementType.LIST_ITEM_OTHER,
+                id="test-element-7",
+            )
+        ),
+        ListItemElementHtml(
+            MockElement(
+                text="Test List Item",
+                metadata=MockElementMetadata(parent_id="test-element-7", page_number=3),
+                category=ElementType.LIST_ITEM,
+                id="test-element-8",
+            )
+        ),
+        ListItemElementHtml(
+            MockElement(
+                text="Test List Item Other",
+                metadata=MockElementMetadata(parent_id="test-element-6", page_number=3),
+                category=ElementType.LIST_ITEM_OTHER,
+                id="test-element-9",
+            )
+        ),
+        TextElementHtml(
+            MockElement(
+                text="Test Text",
+                metadata=MockElementMetadata(parent_id="test-element-6", page_number=3),
+                category=ElementType.TEXT,
+                id="test-element-10",
+            )
+        ),
+    ]
+
+
+@pytest.fixture
+def elements(elements_html: list[ElementHtml]) -> list[Element]:
+    return [el.element for el in elements_html]
+
+
+@pytest.fixture
+def elements_small() -> list[Element]:
+    return [
+        MockElement(
+            text="Test Text 1",
+            category=ElementType.TEXT,
+            id="test-element-1",
+            metadata=MockElementMetadata(page_number=1),
+        ),
+        MockElement(
+            text="Test Text 2",
+            category=ElementType.TEXT,
+            id="test-element-2",
+            metadata=MockElementMetadata(page_number=2),
+        ),
+    ]
+
+
+def test_inject_html_element_content(element_html: ElementHtml) -> None:
+    soup = BeautifulSoup("", HTML_PARSER)
+    tag = soup.new_tag("div")
+    element_html._inject_html_element_content(tag)
+    assert tag.string == "Test Text"
+
+
+def test_get_text_as_html(element_html: ElementHtml) -> None:
+    tag = element_html.get_text_as_html()
+    assert isinstance(tag, Tag)
+    assert tag.name == "p"
+    assert tag.string == "Test Text"
+
+
+def test_get_children_html(element_html: ElementHtml) -> None:
+    soup = BeautifulSoup("", HTML_PARSER)
+    parent_tag = soup.new_tag("div")
+    child_element = MockElement(text="Child Text")
+    child_element_html = ElementHtml(child_element)
+    element_html.set_children([child_element_html])
+    result_tag = element_html._get_children_html(soup, parent_tag)
+    assert result_tag.name == "div"
+    assert len(result_tag.contents) == 2
+    assert result_tag.contents[1].string == "Child Text"
+
+
+def test_get_html_element_text_as_html(element_html: ElementHtml) -> None:
+    tag = element_html.get_html_element()
+    assert isinstance(tag, Tag)
+    assert tag.name == "p"
+    assert tag.string == "Test Text"
+    assert tag["class"] == "test-category"
+    assert tag["id"] == "test-id"
+
+
+def test_get_html_element_no_text_as_html(element_html: ElementHtml) -> None:
+    element_html.element.metadata.text_as_html = None
+    tag = element_html.get_html_element()
+    assert isinstance(tag, Tag)
+    assert tag.name == "div"
+    assert tag.string == "Test Text"
+    assert tag["class"] == "test-category"
+    assert tag["id"] == "test-id"
+
+
+def test_set_children(element_html: ElementHtml) -> None:
+    child_element = MockElement(text="Child Text")
+    child_element_html = ElementHtml(child_element)
+    element_html.set_children([child_element_html])
+    assert len(element_html.children) == 1
+    assert element_html.children[0].element.text == "Child Text"
+
+
+def test_title_element_html_tag(title_element_html: TitleElementHtml) -> None:
+    assert title_element_html.html_tag == "h1"
+
+
+def test_image_element_html_content(image_element_html: ImageElementHtml) -> None:
+    soup = BeautifulSoup("", HTML_PARSER)
+    tag = soup.new_tag("img")
+    image_element_html._inject_html_element_content(tag)
+    assert tag["alt"] == "Test Text"
+
+
+def test_image_element_html_content_with_base64(image_element_html: ImageElementHtml) -> None:
+    image_element_html.element.metadata.image_base64 = "base64data"
+    image_element_html.element.metadata.image_mime_type = "image/png"
+    soup = BeautifulSoup("", HTML_PARSER)
+    tag = soup.new_tag("img")
+    image_element_html._inject_html_element_content(tag)
+    assert tag["src"] == "data:image/png;base64,base64data"
+    assert tag["alt"] == "Test Text"
+
+
+def test_table_element_html_attrs(table_element_html: TableElementHtml) -> None:
+    soup = BeautifulSoup("", HTML_PARSER)
+    tag = soup.new_tag("table")
+    table_element_html._inject_html_element_attrs(tag)
+    assert tag["style"] == "border: 1px solid black; border-collapse: collapse;"
+
+
+def test_table_element_html_wraps_raw_text_in_a_cell(
+    table_element_html: TableElementHtml,
+) -> None:
+    # -- raw table text must sit inside a `<td>` so HTML5 parsers (e.g. the nh3
+    # -- sanitization pass) don't foster-parent it out of the empty table --
+    table_element_html.element.metadata.text_as_html = None
+    table_element_html.element.text = "cell text"
+    tag = table_element_html.get_html_element()
+    cell = tag.find("td")
+    assert cell is not None
+    assert cell.get_text() == "cell text"
+
+
+def test_table_element_html_leaves_empty_table_without_a_cell(
+    table_element_html: TableElementHtml,
+) -> None:
+    table_element_html.element.metadata.text_as_html = None
+    table_element_html.element.text = ""
+    tag = table_element_html.get_html_element()
+    assert tag.find("td") is None
+
+
+def test_link_element_html_attrs(link_element_html: LinkElementHtml) -> None:
+    link_element_html.element.metadata.url = "http://example.com"
+    soup = BeautifulSoup("", HTML_PARSER)
+    tag = soup.new_tag("a")
+    link_element_html._inject_html_element_attrs(tag)
+    assert tag["href"] == "http://example.com"
+
+
+def test_unordered_list_element_html(unordered_list_element_html: UnorderedListElementHtml) -> None:
+    soup = BeautifulSoup("", HTML_PARSER)
+    tag = soup.new_tag("ul")
+    child_element = MockElement(text="Child Text")
+    child_element_html = ListItemElementHtml(child_element)
+    unordered_list_element_html.set_children([child_element_html])
+    result_tag = unordered_list_element_html._get_children_html(soup, tag)
+    assert result_tag.name == "ul"
+    assert len(result_tag.contents) == 1
+    assert result_tag.contents[0].name == "li"
+    assert result_tag.contents[0].string == "Child Text"
+
+
+def test_group_element_children(elements_html: list[ElementHtml]) -> None:
+    grouped_children = _group_element_children(elements_html)
+    assert len(grouped_children) == 7
+    assert len(grouped_children[0].children) == 1
+    assert grouped_children[0].children[0].element.category == ElementType.LIST_ITEM
+    assert len(grouped_children[3].children) == 2
+    assert grouped_children[3].children[0].element.category == ElementType.LIST_ITEM
+    assert grouped_children[3].children[1].element.category == ElementType.LIST_ITEM
+    assert len(grouped_children[5].children) == 3
+    assert grouped_children[5].children[0].element.category == ElementType.LIST_ITEM_OTHER
+    assert grouped_children[5].children[1].element.category == ElementType.LIST_ITEM
+    assert grouped_children[5].children[2].element.category == ElementType.LIST_ITEM_OTHER
+
+
+def test_elements_to_html_tags_by_parent(
+    mocker: MockerFixture, elements_html: list[ElementHtml]
+) -> None:
+    mocker.patch(
+        "unstructured.partition.html.convert._group_element_children",
+        side_effect=lambda children: children,
+    )
+    result = _elements_to_html_tags_by_parent(elements_html)
+    assert len(result) == 4
+    assert result[0].element.id == "test-element-1"
+    assert len(result[0].children) == 0
+    assert result[1].element.id == "test-element-2"
+    assert len(result[1].children) == 0
+    assert result[2].element.id == "test-element-3"
+    assert len(result[2].children) == 2
+    assert result[2].children[0].element.id == "test-element-4"
+    assert result[2].children[1].element.id == "test-element-5"
+    assert result[3].element.id == "test-element-6"
+    assert len(result[3].children) == 3
+    assert result[3].children[0].element.id == "test-element-7"
+    assert len(result[3].children[0].children) == 1
+    assert result[3].children[0].children[0].element.id == "test-element-8"
+    assert result[3].children[1].element.id == "test-element-9"
+    assert result[3].children[2].element.id == "test-element-10"
+
+
+def test_elements_to_html_tags(mocker: MockerFixture, elements: list[Element]) -> None:
+    def _mock_get_html_element(self: ElementHtml, **kwargs: Any):
+        return BeautifulSoup(f"<div>{self.element.id}</div>", HTML_PARSER).find()
+
+    mocker.patch(
+        "unstructured.partition.html.convert._elements_to_html_tags_by_parent",
+        side_effect=lambda elements: elements,
+    )
+    mocker.patch(
+        "unstructured.partition.html.convert.ElementHtml.get_html_element",
+        side_effect=_mock_get_html_element,
+        autospec=True,
+    )
+    result = _elements_to_html_tags(elements)
+    assert len(result) == 10
+    assert all(isinstance(tag, Tag) for tag in result)
+    for i, el in enumerate(result, start=1):
+        assert el.string == f"test-element-{i}"
+
+
+def test_elements_to_html_tags_by_page(mocker: MockerFixture, elements: list[Element]) -> None:
+    def _mock_elements_to_html_tags(elements: list[Element], _: bool):
+        return [
+            BeautifulSoup(f"<div>{element.id}</div>", HTML_PARSER).find() for element in elements
+        ]
+
+    def _mock_group_elements_by_page(elements: list[Element]) -> list[list[Element]]:
+        pages_dict: defaultdict[int, list[Element]] = defaultdict(list)
+        for element in elements:
+            if element.metadata.page_number is not None:
+                pages_dict[element.metadata.page_number].append(element)
+        return list(pages_dict.values())
+
+    mocker.patch(
+        "unstructured.partition.html.convert._elements_to_html_tags",
+        side_effect=_mock_elements_to_html_tags,
+    )
+    mocker.patch(
+        "unstructured.partition.html.convert.group_elements_by_page",
+        side_effect=_mock_group_elements_by_page,
+    )
+    result = _elements_to_html_tags_by_page(elements)
+    assert len(result) == 3
+    assert all(isinstance(tag, Tag) for tag in result)
+    assert result[0].name == "div"
+    assert result[0]["data-page_number"] == 1
+    assert len(result[0].contents) == 1
+    assert result[0].contents[0].string == "test-element-1"
+    assert result[1].name == "div"
+    assert result[1]["data-page_number"] == 2
+    assert len(result[1].contents) == 3
+    for i, el in enumerate(result[1].contents, start=3):
+        assert el.string == f"test-element-{i}"
+    assert result[2]["data-page_number"] == 3
+    assert len(result[2].contents) == 5
+    for i, el in enumerate(result[2].contents, start=6):
+        assert el.string == f"test-element-{i}"
+
+
+def test_group_elements_by_page(caplog: pytest.LogCaptureFixture, elements: list[Element]) -> None:
+    result = group_elements_by_page(elements)
+    assert len(result) == 3
+    assert len(result[0]) == 1
+    assert len(result[1]) == 3
+    assert len(result[2]) == 5
+    assert result[0][0].id == "test-element-1"
+    for i, el in enumerate(result[1], start=3):
+        assert el.id == f"test-element-{i}"
+    for i, el in enumerate(result[2], start=6):
+        assert el.id == f"test-element-{i}"
+    assert "Page number is not set for an element test-element-2. Skipping." in caplog.text
+
+
+def test_elements_to_html_no_group_by_page(
+    mocker: MockerFixture, elements_small: list[Element]
+) -> None:
+    def _mock_elements_to_html_tags(elements: list[Element], _: bool):
+        return [
+            BeautifulSoup(f"<div>{element.id}</div>", HTML_PARSER).find() for element in elements
+        ]
+
+    mock_elements_to_html_tags = mocker.patch(
+        "unstructured.partition.html.convert._elements_to_html_tags",
+        side_effect=_mock_elements_to_html_tags,
+    )
+    mock_elements_to_html_tags_by_page = mocker.patch(
+        "unstructured.partition.html.convert._elements_to_html_tags_by_page",
+    )
+
+    result = elements_to_html(elements_small, exclude_binary_image_data=True, no_group_by_page=True)
+    assert "<div>\n   test-element-1\n  </div>" in result
+    assert "<div>\n   test-element-2\n  </div>" in result
+    assert "data-page_number" not in result
+    mock_elements_to_html_tags.assert_called_once_with(elements_small, True)
+    mock_elements_to_html_tags_by_page.assert_not_called()
+
+
+def test_elements_to_html_group_by_page(
+    mocker: MockerFixture, elements_small: list[Element]
+) -> None:
+    def _mock_elements_to_html_tags_by_page(elements: list[Element], _: bool):
+        return [
+            BeautifulSoup(
+                f"<div data-page_number='{element.metadata.page_number}'>{element.id}</div>",
+                HTML_PARSER,
+            ).find()
+            for element in elements
+        ]
+
+    mock_elements_to_html_tags_by_page = mocker.patch(
+        "unstructured.partition.html.convert._elements_to_html_tags_by_page",
+        side_effect=_mock_elements_to_html_tags_by_page,
+    )
+    mock_elements_to_html_tags = mocker.patch(
+        "unstructured.partition.html.convert._elements_to_html_tags"
+    )
+
+    result = elements_to_html(
+        elements_small, exclude_binary_image_data=True, no_group_by_page=False
+    )
+    soup = BeautifulSoup(result, HTML_PARSER)
+    assert soup.find("div", {"data-page_number": "1"}).string.strip() == "test-element-1"
+    assert soup.find("div", {"data-page_number": "2"}).string.strip() == "test-element-2"
+    mock_elements_to_html_tags_by_page.assert_called_once_with(elements_small, True)
+    mock_elements_to_html_tags.assert_not_called()
+
+
+def test_elements_to_html_invalid_html_template(
+    mocker: MockerFixture, elements: list[Element]
+) -> None:
+    mocker.patch(
+        "unstructured.partition.html.convert.HTML_TEMPLATE",
+        "<html><head><title>Test</title></head></html>",
+    )
+
+    with pytest.raises(ValueError, match="Body tag not found in the HTML template"):
+        elements_to_html(elements)

test_unstructured/partition/html/test_html_to_ontology_parsing.py

@@ -0,0 +1,697 @@
+from typing import Optional, Type
+
+import pytest
+from bs4 import BeautifulSoup
+
+from unstructured.documents.ontology import (
+    Checkbox,
+    Form,
+    FormFieldValue,
+    Image,
+    OntologyElement,
+    Page,
+    RadioButton,
+)
+from unstructured.partition.html.html_utils import indent_html
+from unstructured.partition.html.transformations import RECURSION_LIMIT, parse_html_to_ontology
+
+
+def _wrap_with_body(html: str) -> str:
+    return f'<body class="Document">{html}</body>'
+
+
+def remove_all_ids(html_str):
+    soup = BeautifulSoup(html_str, "html.parser")
+    for tag in soup.find_all(True):
+        if tag.has_attr("id"):
+            del tag["id"]
+    return str(soup)
+
+
+def test_parsing_header_and_footer_into_correct_ontologyelement():
+    input_html = """
+    <div class="Page">
+    <header class="Header">
+     this is a header
+    </header>
+    <footer class="Footer">
+     this is a footer
+    </footer>
+    </div>
+    """
+    page = parse_html_to_ontology(input_html)
+    assert len(page.children) == 2
+    header, footer = page.children
+    assert header.text == "this is a header"
+    assert header.html_tag_name == "header"
+    assert footer.text == "this is a footer"
+    assert footer.html_tag_name == "footer"
+
+
+def test_wrong_html_parser_causes_paragraph_to_be_nested_in_div():
+    # This test would fail if html5lib parser would be applied on the input HTML.
+    # It would result in Page: <p></p> <address></address>
+    #         instead of Page: <p><address></address></p>
+
+    # language=HTML
+    input_html = """
+    <div class="Page">
+    <p class="NarrativeText">
+     <address class="Address">
+      Mountain View, California
+     </address>
+    </p>
+    </div>
+    """
+    page = parse_html_to_ontology(input_html)
+
+    assert len(page.children) == 1
+    narrative_text = page.children[0]
+
+    assert len(narrative_text.children) == 1
+    address = narrative_text.children[0]
+
+    assert address.text == "Mountain View, California"
+
+
+def test_when_class_is_missing_it_can_be_inferred_from_type():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <aside>Some text</aside>
+    </div>
+    """)
+
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <aside class='Sidebar'><p class='Paragraph'>Some text</p></aside>
+    </div>
+    """)
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_when_class_is_wrong_tag_name_is_overwritten():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <p class='Sidebar'>Some text</p>
+    </div>
+    """)
+
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <aside class='Sidebar'><p class='Paragraph'>Some text</p></aside>
+    </div>
+    """)
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_when_tag_not_supported_by_ontology_and_wrong_then_consider_them_text():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <newtag class="wrongclass">Some text
+        </newtag>
+    </div>
+    """)
+
+    base_html = indent_html(base_html)
+
+    # TODO (Pluto): Maybe it should be considered as plain text?
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <span class="UncategorizedText">Some text
+        </span>
+    </div>
+    """)
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_div_are_ignored_when_no_attrs():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+      <div>
+           <input class="RadioButton" name="health-comparison" type="radio"/>
+      </div>
+    </div>
+    """)
+
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+       <input class="RadioButton" name="health-comparison" type="radio"/>
+    </div>
+    """)
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_ids_are_preserved():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+      <div style="background-color: lightblue" id="important_div">
+           <input class="RadioButton" name="health-comparison" type="radio"/>
+      </div>
+    </div>
+    """)
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+      <div class="UncategorizedText" style="background-color: lightblue" id="important_div">
+           <input class="RadioButton" name="health-comparison" type="radio"/>
+      </div>
+    </div>
+    """)
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+
+    page = ontology.children[0]
+    div_obj = page.children[0]
+    assert div_obj.additional_attributes["id"] == "important_div"
+
+
+def test_br_is_not_considered_uncategorized_text():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <br/>
+    </div>
+    """)
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <br/>
+    </div>
+    """)
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_text_without_tag_is_marked_as_uncategorized_text_when_there_are_other_elements():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        About the same
+        <input class="RadioButton" name="health-comparison" type="radio"/>
+        Some text
+    </div>
+    """)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <p class="Paragraph">
+            About the same
+        </p>
+        <input class="RadioButton" name="health-comparison" type="radio"/>
+        <p class="Paragraph">
+            Some text
+        </p>
+    </div>
+    """)
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_keyword_only_attributes_are_preserved_during_mapping():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <input class="FormFieldValue" type="radio" name="options" value="2" checked>
+    """)  # noqa: E501
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <input class="FormFieldValue" type="radio" name="options" value="2" checked>
+    """)  # noqa: E501
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_when_unknown_element_keyword_only_attributes_are_preserved_during_mapping():
+    # <input> can be assigned to multiple classes so it is not clear what it is
+    #  thus we assign it to UncategorizedText
+
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <form class="Form">
+            <label class="FormField" for="option1">
+                <input type="radio" name="option1" value="2" checked>
+                <span class="UncategorizedText">
+                    Option 1 (Checked)
+                </span>
+            </label>
+        </form>
+    </div>
+    """)
+    base_html = indent_html(base_html)
+
+    # TODO(Pluto): Maybe tag also should be overwritten? Or just leave it as it is?
+    #  We classify <input> as UncategorizedText but all the text is preserved
+    #  for UnstructuredElement so it make sense now as well
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <form class="Form">
+            <label class="FormField" for="option1">
+                <input class="RadioButton" type="radio" name="option1" value="2" checked />
+                <span class="UncategorizedText">
+                    Option 1 (Checked)
+                </span>
+            </label>
+        </form>
+    </div>
+    """)  # noqa: E501
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_broken_cell_is_not_raising_error():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <table class="Table">
+            <tbody class="TableBody">
+                <tr class="TableRow">
+                   <td class="TableCell&gt;11,442,231&lt;/td&gt;&lt;td class=" tablecell"="">
+                    83.64 GiB
+                   </td>
+                  <th class="TableCellHeader" rowspan="2">
+                    Fair Value
+                 </th>
+               </tr>
+           </tbody>
+        </table>
+    </div>
+    """)
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <table class="Table">
+            <tbody>
+                <tr>
+                   <td>
+                    83.64 GiB
+                   </td>
+                  <th rowspan="2">
+                    Fair Value
+                 </th>
+               </tr>
+           </tbody>
+        </table>
+    </div>
+    """)
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_table():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <table class="Table">
+            <tbody class="TableBody">
+                <tr class="TableRow">
+                  <td class="TableCell">
+                    Fair Value1
+                  </td>
+                  <th class="TableCellHeader" rowspan="2">
+                    Fair Value2
+                 </th>
+               </tr>
+           </tbody>
+        </table>
+    </div>
+    """)
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <table class="Table">
+            <tbody>
+                <tr>
+                  <td>
+                    Fair Value1
+                  </td>
+                  <th rowspan="2">
+                    Fair Value2
+                 </th>
+               </tr>
+           </tbody>
+        </table>
+    </div>
+    """)
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_table_and_time():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        <table class="Table">
+            <thead class='TableHeader'>
+                <tr class="TableRow">
+                    <th class="TableCellHeader"  colspan="6">
+                        Carrying Value
+                    </th>
+                </tr>
+            </thead>
+            <tbody class='TableBody'>
+                <tr class="TableRow">
+                    <td class="TableCell" colspan="5">
+                        <time class="CalendarDate">
+                            June 30, 2023
+                        </time>
+                    </td>
+                <td class="TableCell">
+                    <span class="Currency">
+                        $—
+                    </span>
+                </td>
+                </tr>
+            </tbody>
+        </table>
+    </div>
+    """)
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <table class="Table">
+            <thead>
+                <tr>
+                    <th colspan="6">
+                        Carrying Value
+                    </th>
+                </tr>
+            </thead>
+            <tbody>
+                <tr>
+                    <td colspan="5">
+                        <time>
+                            June 30, 2023
+                        </time>
+                    </td>
+                <td>
+                    <span>
+                        $—
+                    </span>
+                </td>
+                </tr>
+            </tbody>
+        </table>
+    </div>
+    """)
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_malformed_html():
+    # language=HTML
+    input_html = """
+    <!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+    <html>
+      <head>
+        <title>Super Malformed HTML</title>
+      </head>
+      <body class="Document">
+        <!-- Unclosed comment
+        <div class=>
+          <p>Paragraph with missing closing angle bracket
+        <div>
+          <span>
+            <p>Improperly nested paragraph within a span</span>
+          </p>
+        </div>
+        <script>
+          var x = "Unclosed script tag example;
+        </script>
+        <p>Paragraph with invalid characters: � � �</p>
+      </div>
+    </html>
+    """
+
+    # Such malformed HTML won't be returned by html_partitioning as it uses html5lib parser
+    # to imitate the same behaviour it will be first parsed the same way
+
+    input_html = indent_html(input_html, html_parser="html5lib")
+
+    # Ontology has 1 element and everything inside is just Text.
+    # SECURITY (GHSA-v5mq-3xhg-98m9): the collapsed markup stored as this element's
+    # text is HTML-escaped on output, so the embedded <script>/<div>/<p> survive as
+    # inert escaped text (&lt;script&gt; ...) instead of being re-materialized as live
+    # tags. Previously this test asserted a live <script> tag in the output.
+    # language=HTML
+    expected_html = """
+    <body class="Document">
+      <p class="Paragraph">
+       Unclosed comment
+        &lt;div class=&gt;
+          &lt;p&gt;Paragraph with missing closing angle bracket
+        &lt;div&gt;
+          &lt;span&gt;
+            &lt;p&gt;Improperly nested paragraph within a span&lt;/span&gt;
+          &lt;/p&gt;
+        &lt;/div&gt;
+        &lt;script&gt;
+          var x = "Unclosed script tag example;
+        &lt;/script&gt;
+        &lt;p&gt;Paragraph with invalid characters: � � �&lt;/p&gt;
+      &lt;/div&gt;
+    &lt;/html&gt;
+      </p>
+    </body>
+    """
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(input_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_text_is_wrapped_inside_layout_element():
+    # language=HTML
+    base_html = _wrap_with_body("""
+    <div class="Page">
+        Text
+    </div>
+    """)
+    base_html = indent_html(base_html)
+
+    # language=HTML
+    expected_html = _wrap_with_body("""
+    <div class="Page">
+        <p class='Paragraph'>Text</p>
+    </div>
+    """)
+
+    expected_html = indent_html(expected_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+    parsed_ontology = indent_html(remove_all_ids(ontology.to_html()))
+
+    assert parsed_ontology == expected_html
+
+
+def test_text_in_form_field_value():
+    # language=HTML
+    input_html = """
+    <div class="Page">
+    <input class="FormFieldValue" value="Random Input Value"/>
+    </div>
+    """
+    page = parse_html_to_ontology(input_html)
+
+    assert len(page.children) == 1
+    form_field_value = page.children[0]
+    assert form_field_value.text == ""
+    assert form_field_value.to_text() == "Random Input Value"
+
+
+def test_text_in_form_field_value_with_null_value():
+    # language=HTML
+    input_html = """
+    <div class="Page">
+    <input class="FormFieldValue" value=""/>
+    </div>
+    """
+    page = parse_html_to_ontology(input_html)
+
+    assert len(page.children) == 1
+    form_field_value = page.children[0]
+    assert form_field_value.text == ""
+    assert form_field_value.to_text() == ""
+
+
+def test_to_text_when_form_field():
+    ontology = Page(
+        children=[
+            Form(
+                tag="input",
+                additional_attributes={"value": "Random Input Value"},
+                children=[
+                    FormFieldValue(
+                        tag="input",
+                        additional_attributes={"value": "Random Input Value"},
+                    )
+                ],
+            )
+        ]
+    )
+    assert ontology.to_text(add_children=True) == "Random Input Value"
+
+
+def test_recursion_limit_is_limiting_parsing():
+    # language=HTML
+    broken_html = "some text"
+    for i in range(100):
+        broken_html = f"<p class='Paragraph'>{broken_html}</p>"
+    broken_html = _wrap_with_body(broken_html)
+    ontology = parse_html_to_ontology(broken_html)
+
+    iterator = 1
+    last_child = ontology.children[0]
+    while last_child.children:
+        last_child = last_child.children[0]
+        iterator += 1
+    assert last_child.text.startswith('<p class="Paragraph">')
+    assert iterator == RECURSION_LIMIT
+
+
+def test_get_text_when_recursion_limit_activated():
+    broken_html = "some text"
+    for i in range(100):
+        broken_html = f"<p class='Paragraph'>{broken_html}</p>"
+    broken_html = _wrap_with_body(broken_html)
+    ontology = parse_html_to_ontology(broken_html)
+
+    last_child = ontology.children[0]
+    while last_child.children:
+        last_child = last_child.children[0]
+
+    assert last_child.to_text() == "some text"
+
+
+def test_uncategorizedtest_has_image_and_no_text():
+    # language=HTML
+    base_html = _wrap_with_body("""
+        <div class="Page">
+    <div class="UncategorizedText">
+        <img src="https://www.example.com/image.jpg"/>
+    </div>
+    </div>
+    """)
+
+    base_html = indent_html(base_html)
+
+    ontology: OntologyElement = parse_html_to_ontology(base_html)
+
+    element = ontology.children[0].children[0]
+    assert type(element) is Image
+    assert element.css_class_name == "Image"
+
+
+@pytest.mark.parametrize(
+    ("input_type", "expected_class"),
+    [
+        ("checkbox", Checkbox),
+        ("radio", RadioButton),
+        ("text", FormFieldValue),  # explicit non-specialised type
+        (None, FormFieldValue),  # missing type attribute
+    ],
+)
+def test_input_tag_type_is_mapped_to_correct_ontology_class(
+    input_type: Optional[str], expected_class: Type[OntologyElement]
+) -> None:
+    """Ensure bare <input> tags are classified based on their *type* attribute."""
+
+    type_attr = f' type="{input_type}"' if input_type is not None else ""
+    html_snippet = f'<div class="Page"><input{type_attr} name="field" /></div>'
+
+    page = parse_html_to_ontology(html_snippet)
+    assert len(page.children) == 1
+    element = page.children[0]
+
+    # Validate chosen ontology class and preserved HTML semantics
+    assert isinstance(element, expected_class)
+    assert element.html_tag_name == "input"
+    assert element.css_class_name == expected_class.__name__

test_unstructured/partition/html/test_html_to_unstructured_and_back_parsing.py

@@ -0,0 +1,480 @@
+# End 2 End tests for 15 types
+
+from unstructured.documents.elements import (
+    Element,
+    ElementMetadata,
+    Header,
+    NarrativeText,
+    Table,
+    Text,
+)
+from unstructured.documents.ontology import Address, Paragraph
+from unstructured.partition.html.html_utils import indent_html
+from unstructured.partition.html.partition import partition_html
+from unstructured.partition.html.transformations import (
+    ontology_to_unstructured_elements,
+    parse_html_to_ontology,
+    unstructured_elements_to_ontology,
+)
+
+
+def _wrap_in_body_and_page(html_code):
+    return f'<body class="Document"><div class="Page" data-page-number="1">{html_code}</div></body>'
+
+
+_page_elements = [
+    Text(
+        text="",
+        metadata=ElementMetadata(text_as_html='<div class="Page" data-page-number="1" />'),
+    ),
+]
+
+
+def _assert_elements_equal(actual_elements: list[Element], expected_elements: list[Element]):
+    assert len(actual_elements) == len(expected_elements)
+    for actual, expected in zip(actual_elements, expected_elements):
+        assert actual == expected, f"Actual: {actual}, Expected: {expected}"
+        # Not all elements are considered be __eq__ Elements method
+        actual_html = indent_html(actual.metadata.text_as_html, html_parser="html.parser")
+        expected_html = indent_html(expected.metadata.text_as_html, html_parser="html.parser")
+        assert actual_html == expected_html, f"Actual: {actual_html}, Expected: {expected_html}"
+
+
+def _parse_to_unstructured_elements_and_back_to_html(html_as_str: str):
+    unstructured_elements = partition_html(
+        text=html_as_str, add_img_alt_text=False, html_parser_version="v2", unique_element_ids=True
+    )
+    parsed_ontology = unstructured_elements_to_ontology(unstructured_elements)
+    return unstructured_elements, parsed_ontology
+
+
+def test_simple_narrative_text_with_id():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+    <p class="NarrativeText">
+     DEALER ONLY
+    </p>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+
+    assert expected_html == parsed_html
+    expected_elements = _page_elements + [
+        NarrativeText(
+            text="DEALER ONLY",
+            metadata=ElementMetadata(
+                text_as_html='<p class="NarrativeText">DEALER ONLY</p>',
+            ),
+        )
+    ]
+
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_input_with_radio_button_checked():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+       <input class="RadioButton" name="health-comparison" type="radio" checked/>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+
+    assert expected_html == parsed_html
+    expected_elements = _page_elements + [
+        Text(
+            text="",
+            metadata=ElementMetadata(
+                text_as_html=(
+                    '<input class="RadioButton" name="health-comparison" type="radio" checked />'
+                ),
+            ),
+        )
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_multiple_elements():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+    <p class="Paragraph">
+        About the same
+    </p>
+    <input class="RadioButton" name="health-comparison" type="radio"/>
+    <p class="Paragraph">
+        Some text
+    </p>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+
+    assert expected_html == parsed_html
+    expected_elements = _page_elements + [
+        NarrativeText(
+            text="About the same",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph">About the same</p>',
+            ),
+        ),
+        Text(
+            text="",
+            metadata=ElementMetadata(
+                text_as_html='<input class="RadioButton" name="health-comparison" type="radio" />',
+            ),
+        ),
+        NarrativeText(
+            text="Some text",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph">Some text</p>',
+            ),
+        ),
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_multiple_pages():
+    # language=HTML
+    html_as_str = """
+    <body class="Document">
+        <div class="Page" data-page-number="1">
+            <p class="Paragraph">
+                Some text
+            </p>
+        </div>
+        <div class="Page" data-page-number="2">
+            <p class="Paragraph">
+                Another text
+            </p>
+        </div>
+    </body>
+    """
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+
+    assert expected_html == parsed_html
+
+    expected_elements = [
+        Text(
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page" data-page-number="1" />'),
+        ),
+        NarrativeText(
+            text="Some text",
+            metadata=ElementMetadata(text_as_html='<p class="Paragraph">Some text</p>'),
+        ),
+        Text(
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page" data-page-number="2" />'),
+        ),
+        NarrativeText(
+            text="Another text",
+            metadata=ElementMetadata(text_as_html='<p class="Paragraph">Another text</p>'),
+        ),
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_forms():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+        <form class="Form">
+            <label class="FormField" for="option1">
+                <input class="FormFieldValue" type="radio"
+                 name="options" value="2" checked>
+                <p class="Paragraph">
+                    Option 1 (Checked)
+                </p>
+            </label>
+        </form>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+    assert expected_html == parsed_html
+    expected_elements = _page_elements + [
+        Text(
+            text="2 Option 1 (Checked)",
+            metadata=ElementMetadata(
+                text_as_html=""
+                '<form class="Form">'
+                '<label class="FormField" '
+                'for="option1">'
+                '<input class="FormFieldValue" type="radio" '
+                'name="options" value="2" checked />'
+                '<p class="Paragraph">'
+                "Option 1 (Checked)"
+                "</p></label></form>",
+            ),
+        )
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_table():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+    <table class="Table">
+        <tbody class="TableBody">
+            <tr class="TableRow">
+              <td class="TableCell">
+                Fair Value1
+              </td>
+              <th class="TableCellHeader" rowspan="2">
+                Fair Value2
+             </th>
+           </tr>
+       </tbody>
+    </table>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+
+    expected_elements = _page_elements + [
+        Table(
+            text="Fair Value1 Fair Value2",
+            metadata=ElementMetadata(
+                text_as_html='<table class="Table">'
+                "<tbody>"
+                "<tr>"
+                "<td>"
+                "Fair Value1"
+                "</td>"
+                '<th rowspan="2">'
+                "Fair Value2"
+                "</th></tr></tbody></table>",
+            ),
+        )
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_very_nested_structure_is_preserved():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+    <section class='Section'>
+        <div class='Column'>
+            <header class='Header'>
+                <h1 class='Title'>
+                    Title
+                </h1>
+            </header>
+        </div>
+    </section>
+    <div class='Column'>
+            <header class='Header'>
+                Page 1
+            </header>
+            <blockquote class="Quote">
+                <p class="Paragraph">
+                    Clever Quote
+                </p>
+            </blockquote>
+            <div class='Footnote'>
+                <span class='UncategorizedText'>
+                    Uncategorized footnote text
+                </span>
+            </div>
+    </div>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+
+    assert expected_html == parsed_html
+    expected_elements = _page_elements + [
+        Text(
+            text="",
+            metadata=ElementMetadata(text_as_html='<section class="Section" />'),
+        ),
+        Text(
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Column" />'),
+        ),
+        Header(
+            text="Title",
+            metadata=ElementMetadata(
+                text_as_html='<header class="Header"><h1 class="Title">Title</h1></header>'
+            ),
+        ),
+        Text(
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Column" />'),
+        ),
+        Header(
+            text="Page 1",
+            metadata=ElementMetadata(text_as_html='<header class="Header">Page 1</header>'),
+        ),
+        NarrativeText(
+            text="Clever Quote",
+            metadata=ElementMetadata(
+                text_as_html='<blockquote class="Quote">'
+                '<p class="Paragraph">'
+                "Clever Quote"
+                "</p>"
+                "</blockquote>",
+            ),
+        ),
+        Text(
+            text="Uncategorized footnote text",
+            metadata=ElementMetadata(
+                text_as_html='<div class="Footnote">'
+                '<span class="UncategorizedText">'
+                "Uncategorized footnote text"
+                "</span>"
+                "</div>",
+            ),
+        ),
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_ordered_list():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+    <ul class="UnorderedList">
+        <li class="ListItem">
+            Item 1
+        </li>
+        <li class="ListItem">
+            Item 2
+        </li>
+        <li class="ListItem">
+            Item 3
+        </li>
+    </ul>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+
+    assert expected_html == parsed_html
+    expected_elements = _page_elements + [
+        Text(
+            text="Item 1 Item 2 Item 3",
+            metadata=ElementMetadata(
+                text_as_html='<ul class="UnorderedList">'
+                '<li class="ListItem">'
+                "Item 1"
+                "</li>"
+                '<li class="ListItem">'
+                "Item 2</li>"
+                '<li class="ListItem">'
+                "Item 3"
+                "</li></ul>",
+            ),
+        )
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_squeezed_elements_are_parsed_back():
+    # language=HTML
+    html_as_str = _wrap_in_body_and_page("""
+       <p class="NarrativeText">
+        Table of Contents
+       </p>
+       <address class="Address">
+        68 Prince Street Palmdale, CA 93550
+       </address>
+       <a class="Hyperlink">
+        www.google.com
+       </a>
+    """)
+
+    unstructured_elements, parsed_ontology = _parse_to_unstructured_elements_and_back_to_html(
+        html_as_str
+    )
+    expected_html = indent_html(html_as_str, html_parser="html.parser")
+    parsed_html = indent_html(parsed_ontology.to_html(), html_parser="html.parser")
+
+    assert expected_html == parsed_html
+    expected_elements = _page_elements + [
+        NarrativeText(
+            text="Table of Contents 68 Prince Street Palmdale, CA 93550 www.google.com",
+            metadata=ElementMetadata(
+                text_as_html='<p class="NarrativeText">Table of Contents</p>'
+                '<address class="Address">'
+                "68 Prince Street Palmdale, CA 93550"
+                "</address>"
+                '<a class="Hyperlink">www.google.com</a>',
+            ),
+        )
+    ]
+    _assert_elements_equal(unstructured_elements, expected_elements)
+
+
+def test_inline_elements_are_squeezed_when_text_wrapped_into_paragraphs():
+    # language=HTML
+    base_html = """
+        <div class="Page">
+            About the same
+            <address class="Address">
+                1356 Hornor Avenue Oklahoma
+            </address>
+            Some text
+        </div>
+        """
+    # Such HTML is transformed into Page: [Pargraph, Address, Paragraph]
+    # We would like it to be parsed to UnstructuredElements as [Page, NarrativeText]
+
+    ontology = parse_html_to_ontology(base_html)
+
+    p1, address, p2 = ontology.children
+    assert isinstance(p1, Paragraph)
+    assert isinstance(address, Address)
+    assert isinstance(p2, Paragraph)
+
+    unstructured_elements = ontology_to_unstructured_elements(ontology)
+
+    assert len(unstructured_elements) == 2
+    assert isinstance(unstructured_elements[0], Text)
+    assert isinstance(unstructured_elements[1], NarrativeText)
+
+
+def test_alternate_text_from_image_is_passed():
+    # language=HTML
+    input_html = """
+    <div class="Page">
+    <table>
+        <tr>
+            <td rowspan="2">Example image nested in the table:</td>
+            <td rowspan="2"><img src="my-logo.png" alt="ALT TEXT Logo"></td>
+        </tr>
+    </table>
+    </div>add_img_alt_text
+    """
+    page = parse_html_to_ontology(input_html)
+    unstructured_elements = ontology_to_unstructured_elements(page)
+    assert len(unstructured_elements) == 2
+    assert "ALT TEXT Logo" in unstructured_elements[1].text

test_unstructured/partition/html/test_html_utils.py

@@ -0,0 +1,26 @@
+import pytest
+
+from unstructured.partition.html.transformations import remove_empty_tags_from_html_content
+
+
+@pytest.mark.parametrize(
+    ("html_content, expected_output"),  # noqa PT006
+    [
+        ("<div></div>", ""),
+        ("<div><p></p></div>", "<div></div>"),
+        ("<div><input/></div>", "<div><input/></div>"),
+        ("<div><br/></div>", "<div><br/></div>"),
+        ('<div><p id="1"></p></div>', '<div><p id="1"></p></div>'),
+        ("<div><p>Content</p></div>", "<div><p>Content</p></div>"),
+        ("<div><p> </p></div>", "<div></div>"),
+        ("<div><p></p><span></span></div>", "<div></div>"),
+        ("<div><p>Content</p><span></span></div>", "<div><p>Content</p></div>"),
+        ("<div><p>Content</p><span> </span></div>", "<div><p>Content</p></div>"),
+        (
+            "<div><p>Content</p><span>Text</span></div>",
+            "<div><p>Content</p><span>Text</span></div>",
+        ),
+    ],
+)
+def test_removes_empty_tags(html_content, expected_output):
+    assert remove_empty_tags_from_html_content(html_content) == expected_output

test_unstructured/partition/html/test_partition_v2.py

@@ -0,0 +1,164 @@
+from unstructured.partition.html import partition_html
+
+
+def _by_text(elements):
+    return {(e.text or ""): e for e in elements}
+
+
+def test_category_depth_follows_heading_level_on_a_multi_level_page():
+    """ML-1328 AC1/AC3: a heading's `category_depth` is its heading level (h1->0, h2->1, h3->2),
+    and `parent_id` chains each subsection under its enclosing section heading."""
+    # language=HTML
+    html = """
+    <body class="Document">
+        <div class="Page" data-page-number="1">
+            <h1 class="Title">Cost Share Summary</h1>
+            <p class="NarrativeText">Intro paragraph.</p>
+            <h2 class="Heading">Accumulation Period</h2>
+            <p class="NarrativeText">Accumulation body.</p>
+            <h2 class="Heading">Cost Share Summary Tables by Benefit</h2>
+            <h3 class="Heading">How to read the Cost Share summary tables</h3>
+            <p class="NarrativeText">How-to body.</p>
+        </div>
+    </body>
+    """
+    elements = partition_html(text=html, html_parser_version="v2", unique_element_ids=False)
+    by_text = _by_text(elements)
+
+    section = by_text["Cost Share Summary"]
+    subsection_a = by_text["Accumulation Period"]
+    subsection_b = by_text["Cost Share Summary Tables by Benefit"]
+    sub_subsection = by_text["How to read the Cost Share summary tables"]
+
+    # -- depth tracks heading level --
+    assert section.metadata.category_depth == 0
+    assert subsection_a.metadata.category_depth == 1
+    assert subsection_b.metadata.category_depth == 1
+    assert sub_subsection.metadata.category_depth == 2
+
+    # -- parent_id chains section -> subsection -> sub-subsection --
+    assert subsection_a.metadata.parent_id == section.id
+    assert subsection_b.metadata.parent_id == section.id
+    assert sub_subsection.metadata.parent_id == subsection_b.id
+
+    # -- body text parents to its enclosing heading, not the page/column container --
+    assert by_text["How-to body."].metadata.parent_id == sub_subsection.id
+
+    # -- v2's defining behavior is preserved: every element keeps its text_as_html --
+    assert all(e.metadata.text_as_html for e in elements)
+
+
+def test_category_depth_does_not_change_with_multi_column_layout():
+    """ML-1328 AC2: the same heading reads the same `category_depth` whether the page is single- or
+    multi-column. Layout nesting (Page -> Column) must not bump the depth."""
+    # language=HTML
+    single_column = """
+    <body class="Document">
+        <div class="Page" data-page-number="1">
+            <h1 class="Title">Introduction</h1>
+            <h2 class="Heading">About</h2>
+        </div>
+    </body>
+    """
+    two_column = """
+    <body class="Document">
+        <div class="Page" data-page-number="1">
+            <div class="Column">
+                <h1 class="Title">Introduction</h1>
+                <h2 class="Heading">About</h2>
+            </div>
+            <div class="Column">
+                <h2 class="Heading">Contact</h2>
+            </div>
+        </div>
+    </body>
+    """
+    single = _by_text(
+        partition_html(text=single_column, html_parser_version="v2", unique_element_ids=False)
+    )
+    multi = _by_text(
+        partition_html(text=two_column, html_parser_version="v2", unique_element_ids=False)
+    )
+
+    # -- identical depth across layouts: h1 -> 0, h2 -> 1 -- column wrapping is irrelevant --
+    assert single["Introduction"].metadata.category_depth == 0
+    assert multi["Introduction"].metadata.category_depth == 0
+    assert single["About"].metadata.category_depth == 1
+    assert multi["About"].metadata.category_depth == 1
+    assert multi["Contact"].metadata.category_depth == 1
+
+
+def test_alternative_image_text_can_be_included():
+    # language=HTML
+    html = """
+    <div class="Page">
+        <img src="my-logo.png" alt="ALT TEXT Logo"/>
+    </div>
+    """
+    _, image_to_text_alt_mode = partition_html(
+        text=html,
+        image_alt_mode="to_text",
+        html_parser_version="v2",
+    )
+    assert "ALT TEXT Logo" in image_to_text_alt_mode.text
+
+    _, image_none_alt_mode = partition_html(
+        text=html,
+        image_alt_mode=None,
+        html_parser_version="v2",
+    )
+    assert "ALT TEXT Logo" not in image_none_alt_mode.text
+
+
+def test_alternative_image_text_can_be_included_when_nested_in_paragraph():
+    # language=HTML
+    html = """
+    <div class="Page">
+        <p class="Paragraph">
+            <img src="my-logo.png" alt="ALT TEXT Logo"/>
+        </p>
+    </div>
+    """
+    _, paragraph_to_text_alt_mode = partition_html(
+        text=html,
+        image_alt_mode="to_text",
+        html_parser_version="v2",
+    )
+    assert "ALT TEXT Logo" in paragraph_to_text_alt_mode.text
+
+    _, paragraph_none_alt_mode = partition_html(
+        text=html,
+        image_alt_mode=None,
+        html_parser_version="v2",
+    )
+    assert "ALT TEXT Logo" not in paragraph_none_alt_mode.text
+
+
+def test_attr_and_html_inside_table_cell_is_kept():
+    # language=HTML
+    html = """
+    <div class="Page">
+        <table class="Table">
+            <tbody>
+                <tr>
+                    <td colspan="2">
+                        Some text
+                    </td>
+                    <td>
+                        <input checked="" class="Checkbox" type="checkbox"/>
+                    </td>
+                </tr>
+            </tbody>
+        </table>
+    </div>
+    """
+    page, table = partition_html(
+        text=html,
+        image_alt_mode="to_text",
+        html_parser_version="v2",
+    )
+
+    assert (
+        '<input checked="" class="Checkbox" type="checkbox"/>' in table.metadata.text_as_html
+    )  # class is removed
+    assert 'colspan="2"' in table.metadata.text_as_html

test_unstructured/partition/html/test_unstructured_elements_to_ontology_parsing.py

@@ -0,0 +1,294 @@
+from unstructured.documents.elements import ElementMetadata, NarrativeText, Text
+from unstructured.documents.ontology import Column, Document, Page, Paragraph
+from unstructured.partition.html.transformations import unstructured_elements_to_ontology
+
+
+def test_when_first_elements_does_not_have_id():
+    unstructured_elements = [
+        Text(
+            element_id="1",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        NarrativeText(
+            element_id="2",
+            text="Example text",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Example text </p>', parent_id="1"
+            ),
+        ),
+    ]
+    ontology = unstructured_elements_to_ontology(unstructured_elements)
+
+    assert isinstance(ontology, Document)
+
+    assert len(ontology.children) == 1
+    page = ontology.children[0]
+
+    assert isinstance(page, Page)
+    assert len(page.children) == 1
+    paragraph = page.children[0]
+
+    assert isinstance(paragraph, Paragraph)
+    assert paragraph.text == "Example text"
+
+
+def test_elements_without_text_as_html_are_skipped_not_fatal():
+    # An element with no HTML payload (text_as_html=None) carries nothing to rebuild the
+    # ontology tree from. It must be skipped per-element, not crash the whole conversion.
+    unstructured_elements = [
+        Text(
+            element_id="1",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        NarrativeText(
+            element_id="2",
+            text="no html payload",
+            metadata=ElementMetadata(text_as_html=None, parent_id="1"),
+        ),
+        NarrativeText(
+            element_id="3",
+            text="Example text",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Example text </p>', parent_id="1"
+            ),
+        ),
+    ]
+
+    ontology = unstructured_elements_to_ontology(unstructured_elements)  # must not raise
+
+    assert isinstance(ontology, Document)
+    page = ontology.children[0]
+    assert isinstance(page, Page)
+    # the None-payload element is skipped; the valid one still reconstructs
+    assert len(page.children) == 1
+    assert isinstance(page.children[0], Paragraph)
+    assert page.children[0].text == "Example text"
+
+
+def test_when_two_combined_elements_have_the_same_parent():
+    unstructured_elements = [
+        Text(
+            element_id="1",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        NarrativeText(
+            element_id="2",
+            text="Example text",
+            metadata=ElementMetadata(
+                text_as_html=(
+                    '<p class="Paragraph"> Example text </p>'
+                    '<p class="Paragraph"> Example text 2 </p>'
+                ),
+                parent_id="1",
+            ),
+        ),
+        NarrativeText(
+            element_id="3",
+            text="Example text 2",
+            metadata=ElementMetadata(
+                text_as_html=(
+                    '<p class="Paragraph"> Example text 3 </p>'
+                    '<p class="Paragraph"> Example text 4 </p>'
+                ),
+                parent_id="1",
+            ),
+        ),
+    ]
+    ontology = unstructured_elements_to_ontology(unstructured_elements)
+
+    assert isinstance(ontology, Document)
+
+    assert len(ontology.children) == 1
+    page = ontology.children[0]
+
+    assert isinstance(page, Page)
+    assert len(page.children) == 4
+
+
+def test_content_element_without_parent_nests_in_current_container():
+    """ML-1328: tree reconstruction is layout-container driven, not content-`parent_id` driven.
+
+    Content elements nest in the innermost open layout container regardless of their (now
+    heading-based, possibly absent) `parent_id`. A paragraph with no `parent_id` therefore lands in
+    the current Page rather than being lifted to a Document-level sibling -- and is not lost.
+    """
+    unstructured_elements = [
+        Text(
+            element_id="1",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        NarrativeText(
+            element_id="2",
+            text="Example text",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Example text </p>', parent_id="1"
+            ),
+        ),
+        NarrativeText(
+            element_id="3",
+            text="Example text without parent",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Example text without parent </p>'
+            ),
+        ),
+    ]
+    ontology = unstructured_elements_to_ontology(unstructured_elements)
+
+    assert isinstance(ontology, Document)
+    assert len(ontology.children) == 1
+    page = ontology.children[0]
+    assert isinstance(page, Page)
+    assert len(page.children) == 2
+    first, second = page.children
+    assert first.text == "Example text"
+    assert second.text == "Example text without parent"
+
+
+def test_multiple_pages_can_be_combined():
+    unstructured_elements = [
+        Text(
+            element_id="1",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        NarrativeText(
+            element_id="2",
+            text="Example text on page 1",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Example text on page 1 </p>', parent_id="1"
+            ),
+        ),
+        Text(
+            element_id="3",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        NarrativeText(
+            element_id="4",
+            text="Example text on page 2",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Example text on page 2 </p>', parent_id="3"
+            ),
+        ),
+    ]
+    ontology = unstructured_elements_to_ontology(unstructured_elements)
+
+    assert isinstance(ontology, Document)
+    assert len(ontology.children) == 2
+    page1 = ontology.children[0]
+    page2 = ontology.children[1]
+    assert isinstance(page1, Page)
+    assert isinstance(page2, Page)
+    assert len(page1.children) == 1
+    assert len(page2.children) == 1
+    paragraph1 = page1.children[0]
+    paragraph2 = page2.children[0]
+    assert isinstance(paragraph1, Paragraph)
+    assert isinstance(paragraph2, Paragraph)
+    assert paragraph1.text == "Example text on page 1"
+    assert paragraph2.text == "Example text on page 2"
+
+
+def test_empty_input_returns_empty_document():
+    """ML-1328: empty input must return an empty Document, not raise IndexError."""
+    ontology = unstructured_elements_to_ontology([])
+
+    assert isinstance(ontology, Document)
+    assert ontology.children == []
+
+
+def test_nested_layout_containers_rebuild_column_nesting():
+    """ML-1328: a Column nests inside its Page (container `parent_id` drives the layout tree).
+
+    Two columns under one page; content nests in the innermost open container. This is the
+    multi-column round-trip that exercises container-under-container nesting.
+    """
+    unstructured_elements = [
+        Text(
+            element_id="page",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        Text(
+            element_id="col1",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Column"/>', parent_id="page"),
+        ),
+        NarrativeText(
+            element_id="c1",
+            text="Left column text",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Left column text </p>', parent_id="col1"
+            ),
+        ),
+        Text(
+            element_id="col2",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Column"/>', parent_id="page"),
+        ),
+        NarrativeText(
+            element_id="c2",
+            text="Right column text",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Right column text </p>', parent_id="col2"
+            ),
+        ),
+    ]
+    ontology = unstructured_elements_to_ontology(unstructured_elements)
+
+    assert isinstance(ontology, Document)
+    assert len(ontology.children) == 1
+    page = ontology.children[0]
+    assert isinstance(page, Page)
+    # -- both columns nest under the page; the second column did not pop past the page to root --
+    assert len(page.children) == 2
+    col1, col2 = page.children
+    assert isinstance(col1, Column)
+    assert isinstance(col2, Column)
+    assert [c.text for c in col1.children] == ["Left column text"]
+    assert [c.text for c in col2.children] == ["Right column text"]
+
+
+def test_layout_container_with_unknown_parent_id_does_not_pop_to_root():
+    """ML-1328: a container whose `parent_id` matches no open container nests in the current one.
+
+    Malformed/reordered input (violating the documented parent-before-child precondition) must not
+    pop past valid ancestors to the Document root and mis-nest subsequent content. The Column here
+    references a non-existent parent; it should stay inside the open Page, and nothing is lost.
+    """
+    unstructured_elements = [
+        Text(
+            element_id="page",
+            text="",
+            metadata=ElementMetadata(text_as_html='<div class="Page"/>'),
+        ),
+        Text(
+            element_id="col",
+            text="",
+            metadata=ElementMetadata(
+                text_as_html='<div class="Column"/>', parent_id="DOES_NOT_EXIST"
+            ),
+        ),
+        NarrativeText(
+            element_id="c1",
+            text="Body text",
+            metadata=ElementMetadata(
+                text_as_html='<p class="Paragraph"> Body text </p>', parent_id="col"
+            ),
+        ),
+    ]
+    ontology = unstructured_elements_to_ontology(unstructured_elements)
+
+    assert isinstance(ontology, Document)
+    assert len(ontology.children) == 1
+    page = ontology.children[0]
+    assert isinstance(page, Page)
+    # -- the Column stayed nested in the Page rather than being lifted to a Document-level sibling
+    assert len(page.children) == 1
+    column = page.children[0]
+    assert isinstance(column, Column)
+    assert [c.text for c in column.children] == ["Body text"]

test_unstructured/partition/html/test_xss_sanitization.py

@@ -0,0 +1,254 @@
+"""End-to-end regression tests for the stored-XSS fix (GHSA-v5mq-3xhg-98m9).
+
+These exercise the two production sinks:
+
+* ``elements_to_html`` — the assembled HTML document,
+* ``ElementMetadata.text_as_html`` — which some callers return to clients verbatim,
+
+feeding them the exact proof-of-concept from the advisory and asserting every
+vector is neutralized, while legitimate formatting is preserved.
+"""
+
+from __future__ import annotations
+
+import re
+from urllib.parse import urlparse
+
+from bs4 import BeautifulSoup
+
+from unstructured.documents.elements import ElementMetadata, Text
+from unstructured.partition.html import partition_html
+from unstructured.partition.html.convert import elements_to_html
+
+# -- The advisory's proof-of-concept document. --
+MALICIOUS_HTML = (
+    '<body class="Document">'
+    '<p class="Paragraph" onmouseover="alert(1)">hello'
+    '<img src=x onerror="alert(document.domain)"></p>'
+    '<a class="Hyperlink" href="javascript:alert(document.cookie)">click me</a>'
+    "</body>"
+)
+
+
+def _partition_v2(html_text: str):
+    return list(partition_html(text=html_text, html_parser_version="v2"))
+
+
+def _all_text_as_html(elements) -> str:
+    return " ".join(e.metadata.text_as_html or "" for e in elements)
+
+
+def _href_values(html: str) -> list[str]:
+    return [
+        str(anchor["href"])
+        for anchor in BeautifulSoup(html, "html.parser").find_all("a", href=True)
+    ]
+
+
+class DescribeElementsToHtmlIsInert:
+    """The four advisory vectors must not survive into elements_to_html output."""
+
+    def it_strips_event_handler_attributes(self):
+        out = elements_to_html(_partition_v2(MALICIOUS_HTML), no_group_by_page=True)
+        assert "onmouseover" not in out
+        assert "onerror" not in out
+        assert "onload" not in out
+
+    def it_neutralizes_javascript_hrefs(self):
+        out = elements_to_html(_partition_v2(MALICIOUS_HTML), no_group_by_page=True)
+        assert "javascript:" not in out.lower()
+
+    def it_has_no_live_svg_from_attribute_breakout(self):
+        # -- title='"><svg onload=...>' must not break out of the quoted value --
+        breakout = (
+            '<body class="Document">'
+            '<p class="Paragraph" title=\'"><svg onload=alert(1)>\'>x</p>'
+            "</body>"
+        )
+        out = elements_to_html(_partition_v2(breakout), no_group_by_page=True)
+        # -- no live <svg> tag; the payload is escaped to inert text --
+        assert not re.search(r"<svg", out)
+        assert "&lt;svg" in out
+
+    def it_adds_safe_rel_for_blank_targets(self):
+        blank_target = (
+            '<body class="Document">'
+            '<a class="Hyperlink" href="https://example.com" target="_blank">x</a>'
+            "</body>"
+        )
+        out = elements_to_html(_partition_v2(blank_target), no_group_by_page=True)
+        assert 'target="_blank"' in out
+        assert "noopener" in out
+        assert "noreferrer" in out
+
+    def it_filters_overlay_styles(self):
+        styled = (
+            '<body class="Document">'
+            '<p class="Paragraph" style="position:fixed;inset:0;z-index:9999;color:red">x</p>'
+            "</body>"
+        )
+        out = elements_to_html(_partition_v2(styled), no_group_by_page=True)
+        assert "position" not in out
+        assert "inset" not in out
+        assert "z-index" not in out
+        assert "color:red" in out
+
+
+class DescribeTextAsHtmlIsInert:
+    """`text_as_html` is a value some callers return to clients directly."""
+
+    def it_strips_event_handlers_from_text_as_html(self):
+        tah = _all_text_as_html(_partition_v2(MALICIOUS_HTML))
+        assert "onmouseover" not in tah
+        assert "onerror" not in tah
+
+    def it_neutralizes_javascript_hrefs_in_text_as_html(self):
+        tah = _all_text_as_html(_partition_v2(MALICIOUS_HTML))
+        assert "javascript:" not in tah.lower()
+
+    def it_escapes_the_attribute_breakout_quote(self):
+        breakout = (
+            '<body class="Document">'
+            '<p class="Paragraph" title=\'"><svg onload=alert(1)>\'>x</p>'
+            "</body>"
+        )
+        tah = _all_text_as_html(_partition_v2(breakout))
+        # -- the double-quote that would break out is encoded --
+        assert "&quot;" in tah
+        assert "&lt;svg" in tah
+        assert not re.search(r"<svg", tah)
+
+    def it_drops_meta_refresh_attributes_from_text_as_html(self):
+        meta_refresh = (
+            '<body class="Document">'
+            '<meta class="Keywords" http-equiv="refresh" content="0;url=javascript:alert(1)">'
+            "</body>"
+        )
+        tah = _all_text_as_html(_partition_v2(meta_refresh))
+        assert "http-equiv" not in tah
+        assert "refresh" not in tah
+
+    def it_drops_form_navigation_attributes_from_text_as_html(self):
+        form_payload = (
+            '<body class="Document">'
+            '<form class="Form" action="javascript:alert(1)">'
+            '<input class="FormFieldValue" value="x" formaction="javascript:alert(2)">'
+            "</form>"
+            "</body>"
+        )
+        tah = _all_text_as_html(_partition_v2(form_payload))
+        assert "action=" not in tah
+        assert "formaction" not in tah
+        assert "javascript:" not in tah.lower()
+
+    def it_drops_srcset_from_text_as_html(self):
+        srcset_payload = (
+            '<body class="Document">'
+            '<img class="Image" src="https://example.com/ok.png" '
+            'srcset="javascript:alert(1) 1x">'
+            "</body>"
+        )
+        tah = _all_text_as_html(_partition_v2(srcset_payload))
+        assert "srcset" not in tah
+        assert "javascript:" not in tah.lower()
+
+    def it_rejects_svg_data_images_in_text_as_html(self):
+        svg_payload = (
+            '<body class="Document">'
+            '<img class="Image" src="data:image/svg+xml;base64,PHN2Zz48L3N2Zz4=">'
+            "</body>"
+        )
+        tah = _all_text_as_html(_partition_v2(svg_payload))
+        assert "data:image/svg" not in tah
+
+    def it_adds_safe_rel_for_blank_targets_in_text_as_html(self):
+        blank_target = (
+            '<body class="Document">'
+            '<a class="Hyperlink" href="https://example.com" target="_blank">x</a>'
+            "</body>"
+        )
+        tah = _all_text_as_html(_partition_v2(blank_target))
+        assert 'target="_blank"' in tah
+        assert "noopener" in tah
+        assert "noreferrer" in tah
+
+    def it_filters_overlay_styles_in_text_as_html(self):
+        styled = (
+            '<body class="Document">'
+            '<p class="Paragraph" style="position:fixed;inset:0;z-index:9999;color:red">x</p>'
+            "</body>"
+        )
+        tah = _all_text_as_html(_partition_v2(styled))
+        assert "position" not in tah
+        assert "inset" not in tah
+        assert "z-index" not in tah
+        assert "color: red" in tah
+
+
+class DescribeLinkUrlInjectionViaMetadata:
+    """`elements_to_html` builds `href` from `metadata.url` outside the ontology
+    emitter, so the nh3 sweep must cover it too."""
+
+    def it_drops_a_javascript_url_from_link_metadata(self):
+        el = Text(
+            text="click me",
+            metadata=ElementMetadata(url="javascript:alert(1)"),
+        )
+        el.category = "Link"  # type: ignore[assignment]
+        out = elements_to_html([el], no_group_by_page=True)
+        assert "javascript:" not in out.lower()
+
+    def it_keeps_an_https_url_from_link_metadata(self):
+        el = Text(
+            text="click me",
+            metadata=ElementMetadata(url="https://example.com"),
+        )
+        el.category = "Link"  # type: ignore[assignment]
+        out = elements_to_html([el], no_group_by_page=True)
+        hrefs = _href_values(out)
+        assert any(
+            (p := urlparse(href)).scheme == "https" and p.hostname == "example.com"
+            for href in hrefs
+        )
+
+
+class DescribeLegitimateFormattingPreserved:
+    """The fix must not strip benign structure/formatting."""
+
+    def it_preserves_tables_with_colspan_and_borders(self):
+        tbl = (
+            '<body class="Document"><table class="Table"><tbody>'
+            '<tr><td colspan="2">A&amp;B</td></tr>'
+            "<tr><td>1</td><td>2</td></tr></tbody></table></body>"
+        )
+        out = elements_to_html(_partition_v2(tbl), no_group_by_page=True)
+        assert "<table" in out
+        assert 'colspan="2"' in out
+        # -- the ampersand text is HTML-escaped, not dropped --
+        assert "A&amp;B" in out
+
+    def it_preserves_safe_links_and_headings(self):
+        doc = (
+            '<body class="Document">'
+            '<h1 class="Title">Heading</h1>'
+            '<a class="Hyperlink" href="https://example.com/page">link</a>'
+            "</body>"
+        )
+        out = elements_to_html(_partition_v2(doc), no_group_by_page=True)
+        assert "https://example.com/page" in _href_values(out)
+        assert "Heading" in out
+
+    def it_preserves_mailto_scheme_in_text_as_html(self):
+        # -- `mailto:` is a safe scheme and must survive into `text_as_html`
+        # -- (a value some callers return to clients directly) --
+        doc = '<body class="Document"><a class="Hyperlink" href="mailto:a@b.com">email</a></body>'
+        tah = _all_text_as_html(_partition_v2(doc))
+        assert "mailto:a@b.com" in tah
+
+    def it_preserves_base64_images(self):
+        img_src = "data:image/png;base64,iVBORw0KGgo="
+        el = Text(text="", metadata=ElementMetadata(image_base64="iVBORw0KGgo="))
+        el.category = "Image"  # type: ignore[assignment]
+        el.metadata.image_mime_type = "image/png"
+        out = elements_to_html([el], no_group_by_page=True)
+        assert img_src in out