wehub-resource-sync
461bf6fd40
CI / lint (3.11) (push) Blocked by required conditions
CI / lint (3.12) (push) Blocked by required conditions
CI / lint (3.13) (push) Blocked by required conditions
CI / shellcheck (push) Waiting to run
CI / shfmt (push) Waiting to run
CI / setup (3.11) (push) Waiting to run
CI / setup (3.12) (push) Waiting to run
CI / setup (3.13) (push) Waiting to run
CI / check-licenses (3.12) (push) Blocked by required conditions
CI / test_unit (3.11) (push) Blocked by required conditions
CI / test_unit (3.12) (push) Blocked by required conditions
CI / test_unit (3.13) (push) Blocked by required conditions
CI / test_unit_no_extras (3.11) (push) Blocked by required conditions
CI / test_unit_no_extras (3.12) (push) Blocked by required conditions
CI / test_json_to_html (3.12) (push) Blocked by required conditions
CI / test_unit_no_extras (3.13) (push) Blocked by required conditions
CI / test_unit_dependency_extras (csv, 3.11, --extra csv) (push) Blocked by required conditions
CI / test_unit_dependency_extras (csv, 3.12, --extra csv) (push) Blocked by required conditions
CI / test_unit_dependency_extras (csv, 3.13, --extra csv) (push) Blocked by required conditions
CI / test_unit_dependency_extras (docx, 3.11, --extra docx) (push) Blocked by required conditions
CI / test_unit_dependency_extras (docx, 3.12, --extra docx) (push) Blocked by required conditions
CI / test_unit_dependency_extras (docx, 3.13, --extra docx) (push) Blocked by required conditions
CI / test_unit_dependency_extras (markdown, 3.11, --extra md) (push) Blocked by required conditions
CI / test_unit_dependency_extras (markdown, 3.12, --extra md) (push) Blocked by required conditions
CI / test_unit_dependency_extras (markdown, 3.13, --extra md) (push) Blocked by required conditions
CI / test_unit_dependency_extras (odt, 3.11, --extra odt) (push) Blocked by required conditions
CI / test_unit_dependency_extras (odt, 3.12, --extra odt) (push) Blocked by required conditions
CI / test_unit_dependency_extras (pdf-image, 3.12, --extra pdf --extra image --extra paddleocr) (push) Blocked by required conditions
CI / test_unit_dependency_extras (pdf-image, 3.13, --extra pdf --extra image --extra paddleocr) (push) Blocked by required conditions
CI / test_unit_dependency_extras (pptx, 3.13, --extra pptx) (push) Blocked by required conditions
CI / test_unit_dependency_extras (odt, 3.13, --extra odt) (push) Blocked by required conditions
CI / test_unit_dependency_extras (pdf-image, 3.11, --extra pdf --extra image --extra paddleocr) (push) Blocked by required conditions
CI / test_unit_dependency_extras (pptx, 3.11, --extra pptx) (push) Blocked by required conditions
CI / test_unit_dependency_extras (pptx, 3.12, --extra pptx) (push) Blocked by required conditions
CI / test_unit_dependency_extras (pypandoc, 3.11, --extra epub --extra org --extra rtf --extra rst) (push) Blocked by required conditions
CI / test_unit_dependency_extras (pypandoc, 3.12, --extra epub --extra org --extra rtf --extra rst) (push) Blocked by required conditions
CI / test_unit_dependency_extras (pypandoc, 3.13, --extra epub --extra org --extra rtf --extra rst) (push) Blocked by required conditions
CI / test_unit_dependency_extras (xlsx, 3.11, --extra xlsx) (push) Blocked by required conditions
CI / test_unit_dependency_extras (xlsx, 3.12, --extra xlsx) (push) Blocked by required conditions
CI / test_unit_dependency_extras (xlsx, 3.13, --extra xlsx) (push) Blocked by required conditions
CI / test_ingest_src (3.12) (push) Blocked by required conditions
CI / test_json_to_markdown (3.12) (push) Blocked by required conditions
CI / changelog (push) Waiting to run
CI / test_dockerfile (push) Blocked by required conditions
CodeQL / Analyze (python) (push) Waiting to run
Build And Push Docker Image / set-short-sha (push) Waiting to run
Build And Push Docker Image / build-images (linux/amd64, opensource-linux-8core) (push) Blocked by required conditions
Build And Push Docker Image / build-images (linux/arm64, ubuntu-24.04-arm) (push) Blocked by required conditions
Build And Push Docker Image / publish-images (push) Blocked by required conditions
Partition Benchmark / setup (push) Waiting to run
Partition Benchmark / Measure and compare partition() runtime (push) Blocked by required conditions
302 lines
12 KiB
Python
302 lines
12 KiB
Python
# pyright: reportPrivateUsage=false
|
|
|
|
"""Unit-test suite for the `unstructured.common.html_table` module."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from lxml.html import fragment_fromstring
|
|
|
|
from unstructured.common.html_table import (
|
|
HtmlCell,
|
|
HtmlRow,
|
|
HtmlTable,
|
|
htmlify_matrix_of_cell_texts,
|
|
)
|
|
|
|
|
|
class Describe_htmlify_matrix_of_cell_texts:
|
|
"""Unit-test suite for `unstructured.common.html_table.htmlify_matrix_of_cell_texts()`."""
|
|
|
|
def test_htmlify_matrix_handles_empty_cells(self):
|
|
assert htmlify_matrix_of_cell_texts([["cell1", "", "cell3"], ["", "cell5", ""]]) == (
|
|
"<table>"
|
|
"<tr><td>cell1</td><td/><td>cell3</td></tr>"
|
|
"<tr><td/><td>cell5</td><td/></tr>"
|
|
"</table>"
|
|
)
|
|
|
|
def test_htmlify_matrix_handles_special_characters(self):
|
|
assert htmlify_matrix_of_cell_texts([['<>&"', "newline\n"]]) == (
|
|
"<table><tr><td><>&"</td><td>newline<br/></td></tr></table>"
|
|
)
|
|
|
|
def test_htmlify_matrix_handles_multiple_rows_and_cells(self):
|
|
assert htmlify_matrix_of_cell_texts([["cell1", "cell2"], ["cell3", "cell4"]]) == (
|
|
"<table>"
|
|
"<tr><td>cell1</td><td>cell2</td></tr>"
|
|
"<tr><td>cell3</td><td>cell4</td></tr>"
|
|
"</table>"
|
|
)
|
|
|
|
def test_htmlify_matrix_handles_empty_matrix(self):
|
|
assert htmlify_matrix_of_cell_texts([]) == ""
|
|
|
|
|
|
class DescribeHtmlTable:
|
|
"""Unit-test suite for `unstructured.common.html_table.HtmlTable`."""
|
|
|
|
def it_can_construct_from_html_text(self):
|
|
html_table = HtmlTable.from_html_text("<table><tr><td>foobar</td></tr></table>")
|
|
|
|
assert isinstance(html_table, HtmlTable)
|
|
assert html_table._table.tag == "table"
|
|
|
|
@pytest.mark.parametrize(
|
|
"html_text",
|
|
[
|
|
"<table><tr><td>foobar</td></tr></table>",
|
|
"<body><table><tr><td>foobar</td></tr></table></body>",
|
|
"<html><body><table><tr><td>foobar</td></tr></table></body></html>",
|
|
],
|
|
)
|
|
def it_can_find_a_table_wrapped_in_an_html_or_body_element(self, html_text: str):
|
|
html_table = HtmlTable.from_html_text(html_text)
|
|
|
|
assert isinstance(html_table, HtmlTable)
|
|
assert html_table._table.tag == "table"
|
|
|
|
def but_it_raises_when_no_table_element_is_present_in_the_html(self):
|
|
with pytest.raises(ValueError, match="`html_text` contains no `<table>` element"):
|
|
HtmlTable.from_html_text("<html><body><tr><td>foobar</td></tr></body></html>")
|
|
|
|
def it_removes_any_attributes_present_on_the_table_element(self):
|
|
html_table = HtmlTable.from_html_text(
|
|
'<table border="1", class="foobar"><tr><td>foobar</td></tr></table>',
|
|
)
|
|
assert html_table.html == "<table><tr><td>foobar</td></tr></table>"
|
|
|
|
def but_it_preserves_colspan_and_rowspan_as_structural_cell_attributes(self):
|
|
html_table = HtmlTable.from_html_text(
|
|
"<table>"
|
|
"<tr><th colspan='2' class='hdr' style='x'>A</th>"
|
|
"<th rowspan='2' id='foo'>B</th></tr>"
|
|
"<tr><td colspan='2' rowspan='3' data-k='v'>C</td><td>D</td></tr>"
|
|
"</table>"
|
|
)
|
|
table = fragment_fromstring(html_table.html)
|
|
|
|
# -- colspan/rowspan survive compactification --
|
|
assert table.xpath("./tr[1]/td[1]/@colspan") == ["2"]
|
|
assert table.xpath("./tr[1]/td[2]/@rowspan") == ["2"]
|
|
assert table.xpath("./tr[2]/td[1]/@colspan") == ["2"]
|
|
assert table.xpath("./tr[2]/td[1]/@rowspan") == ["3"]
|
|
# -- cosmetic / arbitrary attributes are still stripped --
|
|
assert table.xpath("./tr[1]/td[1]/@class") == []
|
|
assert table.xpath("./tr[1]/td[1]/@style") == []
|
|
assert table.xpath("./tr[1]/td[2]/@id") == []
|
|
assert table.xpath("./tr[2]/td[1]/@data-k") == []
|
|
|
|
@pytest.mark.parametrize(
|
|
"html_text",
|
|
[
|
|
"<table><thead><tr><td>foobar</td></tr></thead></table>",
|
|
"<table><thead><tr><td>foobar</td></tr></thead><tbody></tbody></table>",
|
|
"<table><tbody><tr><td>foobar</td></tr></tbody><tfoot></tfoot></table>",
|
|
],
|
|
)
|
|
def it_removes_any_thead_tbody_or_tfoot_elements_present_within_the_table_element(
|
|
self, html_text: str
|
|
):
|
|
html_table = HtmlTable.from_html_text(html_text)
|
|
assert html_table.html == "<table><tr><td>foobar</td></tr></table>"
|
|
|
|
def it_changes_any_th_elements_to_td_elements_for_cell_element_uniformity(self):
|
|
html_table = HtmlTable.from_html_text(
|
|
"<table> <tr><th>a</th><th/><th>b</th></tr> <tr><td/><td>c</td><td/></tr></table>"
|
|
)
|
|
assert html_table.html == (
|
|
"<table><tr><td>a</td><td/><td>b</td></tr><tr><td/><td>c</td><td/></tr></table>"
|
|
)
|
|
|
|
def it_removes_any_extra_whitespace_between_elements_and_normalizes_whitespace_in_text(self):
|
|
html_table = HtmlTable.from_html_text(
|
|
"\n <table>\n <tr>\n <td>\tabc def\nghi </td>\n </tr>\n</table>\n ",
|
|
)
|
|
assert html_table.html == "<table><tr><td>abc def ghi</td></tr></table>"
|
|
|
|
def it_preserves_tail_text_in_mixed_content_cells(self):
|
|
"""Tail text (between inline children) carries real content and must not be dropped."""
|
|
html_table = HtmlTable.from_html_text(
|
|
"<table><tr>"
|
|
"<td><b>foo</b> bar <b>baz</b></td>"
|
|
"<td><b>x</b><br/>y<br/>z</td>"
|
|
"</tr></table>"
|
|
)
|
|
assert html_table.html == (
|
|
"<table><tr>"
|
|
"<td><b>foo</b> bar <b>baz</b></td>"
|
|
"<td><b>x</b><br/>y<br/>z</td>"
|
|
"</tr></table>"
|
|
)
|
|
|
|
def and_it_normalizes_whitespace_within_tail_text(self):
|
|
"""Tail whitespace runs are collapsed, but leading/trailing spaces are kept."""
|
|
html_table = HtmlTable.from_html_text(
|
|
"<table><tr><td><b>a</b> b\n c <b>d</b></td></tr></table>"
|
|
)
|
|
assert html_table.html == "<table><tr><td><b>a</b> b c <b>d</b></td></tr></table>"
|
|
|
|
def it_can_serialize_the_table_element_to_str_html_text(self):
|
|
table = fragment_fromstring("<table><tr><td>foobar</td></tr></table>")
|
|
html_table = HtmlTable(table)
|
|
|
|
assert html_table.html == "<table><tr><td>foobar</td></tr></table>"
|
|
|
|
def it_can_iterate_the_rows_in_the_table(self):
|
|
html_table = HtmlTable.from_html_text(
|
|
"<table>"
|
|
" <tr><td>abc</td><td>def</td><td>ghi</td></tr>"
|
|
" <tr><td>jkl</td><td>mno</td><td>pqr</td></tr>"
|
|
" <tr><td>stu</td><td>vwx</td><td>yz</td></tr>"
|
|
"</table>"
|
|
)
|
|
|
|
row_iter = html_table.iter_rows()
|
|
|
|
row = next(row_iter)
|
|
assert isinstance(row, HtmlRow)
|
|
assert row.html == "<tr><td>abc</td><td>def</td><td>ghi</td></tr>"
|
|
# --
|
|
row = next(row_iter)
|
|
assert isinstance(row, HtmlRow)
|
|
assert row.html == "<tr><td>jkl</td><td>mno</td><td>pqr</td></tr>"
|
|
# --
|
|
row = next(row_iter)
|
|
assert isinstance(row, HtmlRow)
|
|
assert row.html == "<tr><td>stu</td><td>vwx</td><td>yz</td></tr>"
|
|
# --
|
|
with pytest.raises(StopIteration):
|
|
next(row_iter)
|
|
|
|
def it_preserves_row_header_semantics_when_iterating_rows(self):
|
|
html_table = HtmlTable.from_html_text(
|
|
"<table>"
|
|
" <thead><tr><td>head-from-thead</td></tr></thead>"
|
|
" <tbody>"
|
|
" <tr><th>head-from-th</th></tr>"
|
|
" <tr><td>body</td></tr>"
|
|
" </tbody>"
|
|
"</table>"
|
|
)
|
|
|
|
assert [row.is_header for row in html_table.iter_rows()] == [True, True, False]
|
|
|
|
def and_it_preserves_source_row_html_before_compactification(self):
|
|
html_table = HtmlTable.from_html_text(
|
|
"<table>"
|
|
" <thead><tr data-row='header'><th scope='col'>Header</th></tr></thead>"
|
|
" <tbody><tr><td class='body-cell'>Body</td></tr></tbody>"
|
|
"</table>"
|
|
)
|
|
rows = list(html_table.iter_rows())
|
|
header_row = fragment_fromstring(rows[0].source_html or "<tr/>")
|
|
body_row = fragment_fromstring(rows[1].source_html or "<tr/>")
|
|
|
|
assert header_row.xpath("./@data-row") == ["header"]
|
|
assert header_row.xpath("./th/@scope") == ["col"]
|
|
assert body_row.xpath("./td/@class") == ["body-cell"]
|
|
|
|
# -- compactified row HTML contract remains unchanged --
|
|
assert rows[0].html == "<tr><td>Header</td></tr>"
|
|
assert rows[1].html == "<tr><td>Body</td></tr>"
|
|
|
|
def it_provides_access_to_the_clear_concatenated_text_of_the_table(self):
|
|
html_table = HtmlTable.from_html_text(
|
|
"<table>"
|
|
" <tr><th> a\n b c </th><th/><th>def</th></tr>"
|
|
" <tr><td>gh \ti</td><td/><td>\n jk l </td></tr>"
|
|
" <tr><td/><td> m n op\n</td><td/></tr>"
|
|
"</table>"
|
|
)
|
|
assert html_table.text == "a b c def gh i jk l m n op"
|
|
|
|
|
|
class DescribeHtmlRow:
|
|
"""Unit-test suite for `unstructured.common.html_table.HtmlRow`."""
|
|
|
|
def it_can_serialize_the_row_to_html(self):
|
|
assert HtmlRow(fragment_fromstring("<tr><td>a</td><td>b</td><td/></tr>")).html == (
|
|
"<tr><td>a</td><td>b</td><td/></tr>"
|
|
)
|
|
|
|
def it_can_iterate_the_cells_in_the_row(self):
|
|
row = HtmlRow(fragment_fromstring("<tr><td>a</td><td>b</td><td/></tr>"))
|
|
|
|
cell_iter = row.iter_cells()
|
|
|
|
cell = next(cell_iter)
|
|
assert isinstance(cell, HtmlCell)
|
|
assert cell.html == "<td>a</td>"
|
|
# --
|
|
cell = next(cell_iter)
|
|
assert isinstance(cell, HtmlCell)
|
|
assert cell.html == "<td>b</td>"
|
|
# --
|
|
cell = next(cell_iter)
|
|
assert isinstance(cell, HtmlCell)
|
|
assert cell.html == "<td/>"
|
|
# --
|
|
with pytest.raises(StopIteration):
|
|
next(cell_iter)
|
|
|
|
def it_can_iterate_the_texts_of_the_cells_in_the_row(self):
|
|
row = HtmlRow(fragment_fromstring("<tr><td>a</td><td>b</td><td/></tr>"))
|
|
|
|
text_iter = row.iter_cell_texts()
|
|
|
|
assert next(text_iter) == "a"
|
|
assert next(text_iter) == "b"
|
|
with pytest.raises(StopIteration):
|
|
next(text_iter)
|
|
|
|
def and_it_includes_descendant_inline_text_in_cell_texts(self):
|
|
row = HtmlRow(
|
|
fragment_fromstring(
|
|
"<tr>"
|
|
"<td>ID</td>"
|
|
"<td><a href='#'>Category Link</a></td>"
|
|
"<td><span> Extra spacing </span></td>"
|
|
"</tr>"
|
|
)
|
|
)
|
|
|
|
assert list(row.iter_cell_texts()) == ["ID", "Category Link", "Extra spacing"]
|
|
|
|
def it_knows_when_it_represents_a_header_row(self):
|
|
assert HtmlRow(fragment_fromstring("<tr><td>a</td></tr>")).is_header is False
|
|
assert HtmlRow(fragment_fromstring("<tr><td>a</td></tr>"), is_header=True).is_header is True
|
|
|
|
|
|
class DescribeHtmlCell:
|
|
"""Unit-test suite for `unstructured.common.html_table.HtmlCell`."""
|
|
|
|
def it_can_serialize_the_cell_to_html(self):
|
|
assert HtmlCell(fragment_fromstring("<td>a b c</td>")).html == "<td>a b c</td>"
|
|
|
|
def and_it_preserves_nested_markup_when_serializing_nonempty_cells(self):
|
|
assert HtmlCell(fragment_fromstring("<td><a href='#'>Category Link</a></td>")).html == (
|
|
'<td><a href="#">Category Link</a></td>'
|
|
)
|
|
|
|
@pytest.mark.parametrize(
|
|
("cell_html", "expected_value"),
|
|
[
|
|
("<td> Lorem ipsum </td>", "Lorem ipsum"),
|
|
("<td><a href='#'>Category Link</a></td>", "Category Link"),
|
|
("<td/>", ""),
|
|
],
|
|
)
|
|
def it_knows_the_text_in_the_cell(self, cell_html: str, expected_value: str):
|
|
assert HtmlCell(fragment_fromstring(cell_html)).text == expected_value
|