'
p = etree.fromstring(html, html_parser).xpath(".//p")[0]
accum = _ElementAccumulator(p)
accum.add(TextSegment("Ford... you're turning into a penguin.", {}))
(element,) = accum.flush(None)
assert element.metadata.page_number == 2
def it_leaves_page_number_None_when_no_data_page_number_in_tree(self):
p = etree.fromstring("", html_parser).xpath(".//p")[0]
accum = _ElementAccumulator(p)
accum.add(TextSegment("Ford... you're turning into a penguin.", {}))
(element,) = accum.flush(None)
assert element.metadata.page_number is None
# -- fixtures --------------------------------------------------------------------------------
@pytest.fixture()
def html_element(self) -> etree.ElementBase:
return etree.fromstring("", html_parser).xpath(".//p")[0]
class Describe_PreElementAccumulator:
"""Isolated unit-test suite for `unstructured.partition.html.parser._PreElementAccumulator`."""
def it_computes_the_normalized_text_of_its_text_segments_to_help(self):
html_element = etree.fromstring("", html_parser).xpath(".//p")[0]
accum = _PreElementAccumulator(html_element)
accum.add(TextSegment("\n\n", {}))
accum.add(TextSegment(" The panel lit up\n", {}))
accum.add(TextSegment(" with the words 'Please do not press\n", {}))
accum.add(TextSegment(" this button again'\n\n", {}))
# -- note single leading and trailing newline stripped --
assert accum._normalized_text == (
"\n"
" The panel lit up\n"
" with the words 'Please do not press\n"
" this button again'\n"
)
# -- FLOW (BLOCK-ITEM) ELEMENTS ------------------------------------------------------------------
class DescribeFlow:
"""Isolated unit-test suite for `unstructured.partition.html.parser.Flow`.
The `Flow` class provides most behaviors for flow (block-level) elements.
"""
# -- .is_phrasing -----------------------------------------------------
def it_knows_it_is_NOT_a_phrasing_element(self):
p = etree.fromstring("
Hello
", html_parser).xpath(".//p")[0]
assert isinstance(p, Flow)
assert p.is_phrasing is False
# -- .iter_elements() -------------------------------------------------
def it_generates_the_document_elements_from_the_Flow_element(self):
"""Phrasing siblings of child block elements are processed with text or tail.
In the general case, a Flow element can contain text, phrasing content, and child flow
elements.
Each of these five lines in this example is a "paragraph" and gives rise to a distinct
document-element.
"""
html_text = """
Text of div with hierarchical\nphrasing content before first block item
tail of block item with hierarchical phrasing content
second block item
tail of block item with hierarchical phrasing content
"""
div = etree.fromstring(html_text, html_parser).xpath(".//div")[0]
elements = div.iter_elements()
e = next(elements)
assert e == Text("Text of div with hierarchical phrasing content before first block item")
assert e.metadata.to_dict() == {
"emphasized_text_contents": ["with", "hierarchical", "phrasing"],
"emphasized_text_tags": ["b", "bi", "b"],
}
e = next(elements)
assert e == NarrativeText("Click here to see the blurb for this block item.")
assert e.metadata.to_dict() == {"link_texts": ["here"], "link_urls": ["http://blurb.io"]}
e = next(elements)
assert e == Text("tail of block item with hierarchical phrasing content")
assert e.metadata.to_dict() == {
"emphasized_text_contents": ["with", "hierarchical", "phrasing"],
"emphasized_text_tags": ["b", "bi", "b"],
}
e = next(elements)
assert e == Text("second block item")
assert e.metadata.to_dict() == {}
e = next(elements)
assert e == Text("tail of block item with hierarchical phrasing content")
assert e.metadata.to_dict() == {
"emphasized_text_contents": ["with", "hierarchical"],
"emphasized_text_tags": ["b", "bi"],
}
with pytest.raises(StopIteration):
e = next(elements)
# -- ._page_number ----------------------------------------------------
def it_returns_None_when_no_data_page_number_in_tree(self):
p = etree.fromstring("
text
", html_parser).xpath(".//p")[0]
assert p._page_number is None
def it_finds_page_number_from_ancestor(self):
html = '
text
'
p = etree.fromstring(html, html_parser).xpath(".//p")[0]
assert p._page_number == 1
def it_finds_page_number_on_self(self):
html = '
text
'
div = etree.fromstring(html, html_parser).xpath(".//div")[0]
assert div._page_number == 3
def it_returns_nearest_ancestors_page_number(self):
html = '
text
'
p = etree.fromstring(html, html_parser).xpath(".//p")[0]
assert p._page_number == 2
def it_returns_None_for_non_numeric_data_page_number(self):
html = '
text
'
p = etree.fromstring(html, html_parser).xpath(".//p")[0]
assert p._page_number is None
def it_falls_back_to_outer_page_number_when_inner_is_non_numeric(self):
html = '
text
'
p = etree.fromstring(html, html_parser).xpath(".//p")[0]
assert p._page_number == 1
# -- ._element_from_text_or_tail() ------------------------------------
def it_assembles_text_and_tail_document_elements_to_help(self):
"""Text and tails and their phrasing content are both processed the same way."""
html_text = "
The \n Roman poet Virgil gave his pet fly
"
div = etree.fromstring(html_text, html_parser).xpath(".//div")[0]
elements = div._element_from_text_or_tail(div.text, deque(div), Text)
e = next(elements)
# -- element text is normalized --
assert e == Text("The Roman poet Virgil gave his pet fly")
# -- individual annotations are consolidated --
assert e.metadata.to_dict() == {
"emphasized_text_contents": ["poet", "Virgil", "gave"],
"emphasized_text_tags": ["b", "bi", "b"],
}
def but_it_does_not_generate_a_document_element_when_only_whitespace_is_contained(self):
html_text = "
\n \n \n \n
"
div = etree.fromstring(html_text, html_parser).xpath(".//div")[0]
elements = div._element_from_text_or_tail(div.text, deque(div), Text)
with pytest.raises(StopIteration):
next(elements)
def it_uses_the_specified_element_class_to_form_the_document_element(self):
html_text = "
\n The line-storm clouds fly tattered and swift\n
"
div = etree.fromstring(html_text, html_parser).xpath(".//div")[0]
elements = div._element_from_text_or_tail(div.text, deque(div), Address)
e = next(elements)
assert e == Address("The line-storm clouds fly tattered and swift")
assert e.metadata.to_dict() == {}
with pytest.raises(StopIteration):
next(elements)
def and_it_selects_the_document_element_class_by_analyzing_the_text_when_not_specified(self):
html_text = "
\n The line-storm clouds fly tattered and swift,\n
"
div = etree.fromstring(html_text, html_parser).xpath(".//div")[0]
elements = div._element_from_text_or_tail(div.text, deque(div))
assert next(elements) == NarrativeText("The line-storm clouds fly tattered and swift,")
def but_it_does_not_generate_a_document_element_when_only_a_bullet_character_is_contained(self):
html_text = "
*
"
div = etree.fromstring(html_text, html_parser).xpath(".//div")[0]
elements = div._element_from_text_or_tail(div.text, deque(div))
with pytest.raises(StopIteration):
next(elements)
# -- ._iter_text_segments() -------------------------------------------
@pytest.mark.parametrize(
("html_text", "expected_value"),
[
( # -- text with no phrasing --
"
Ford... you're turning into a penguin.
",
[("Ford... you're turning into a penguin.", {})],
),
( # -- text with phrasing --
"
` element can contain only phrasing content."""
html_text = (
"
\n"
" The Answer to the Great Question... Of Life, the Universe and Everything...\n"
" Is... Forty-two, said Deep Thought, with infinite majesty and calm.\n"
"
\n"
)
pre = etree.fromstring(html_text, html_parser).xpath(".//pre")[0]
elements = pre.iter_elements()
e = next(elements)
assert e == CodeSnippet(
" The Answer to the Great Question... Of Life, the Universe and Everything...\n"
" Is... Forty-two, said Deep Thought, with infinite majesty and calm."
)
with pytest.raises(StopIteration):
next(elements)
@pytest.mark.parametrize(
("html_text", "expected_value"),
[
# -- a newline in the 0th position of pre.text is dropped --
("
\n foo
", " foo "),
# -- but not when preceded by any other whitespace --
("
\n foo
", " \n foo "),
# -- and only one is dropped --
("
\n\n foo
", "\n foo "),
# -- a newline in the -1th position is dropped --
("
foo \n
", " foo "),
# -- but not when followed by any other whitespace --
("
foo \n
", " foo \n "),
# -- and only one is dropped --
("
foo \n\n
", " foo \n"),
# -- a newline in both positions are both dropped --
("
\n foo \n
", " foo "),
# -- or not when not at the absolute edge --
("
\n foo \n
", " \n foo \n "),
],
)
def but_it_strips_a_single_leading_or_trailing_newline(
self, html_text: str, expected_value: str
):
"""Content starts on next line when opening `
` tag is immediately followed by `\n`"""
pre = etree.fromstring(html_text, html_parser).xpath(".//pre")[0]
e = next(pre.iter_elements())
assert e.text == expected_value
def it_assigns_emphasis_and_link_metadata_when_contents_have_those_phrasing_elements(self):
html_text = '
'
pre = etree.fromstring(html_text, html_parser).xpath(".//pre")[0]
e = next(pre.iter_elements())
assert e.text == "You're turning into a penguin."
assert e.metadata.emphasized_text_contents == ["turning"]
assert e.metadata.emphasized_text_tags == ["b"]
assert e.metadata.link_texts == ["penguin"]
assert e.metadata.link_urls == ["http://eie.io"]
def it_generates_CodeSnippet_elements_to_preserve_code_formatting(self):
"""Pre elements should generate CodeSnippet elements, not generic Text elements.
This ensures code formatting (whitespace, line breaks) is preserved during chunking.
"""
html_text = "
def hello():\n print('Hello')\n return True
"
pre = etree.fromstring(html_text, html_parser).xpath(".//pre")[0]
e = next(pre.iter_elements())
assert isinstance(e, CodeSnippet)
assert e.text == "def hello():\n print('Hello')\n return True"
class DescribeRemovedBlock:
"""Isolated unit-test suite for `unstructured.partition.html.parser.RemovedBlock`.
This class is used for block level items we want to skip like `` and ``.
"""
def it_is_skipped_during_parsing(self):
html_text = """
An elephant at sunset
Content we want.
"""
div = etree.fromstring(html_text, html_parser).xpath(".//div")[0]
assert list(div.iter_elements()) == [NarrativeText("Content we want.")]
# -- PHRASING (INLINE) ELEMENTS ------------------------------------------------------------------
class DescribePhrasing:
"""Isolated unit-test suite for `unstructured.partition.html.parser.Phrasing`.
The `Phrasing` class provides most behaviors for phrasing (inline) elements.
"""
# -- .is_phrasing -----------------------------------------------------
def it_knows_it_is_a_phrasing_element(self):
b = etree.fromstring("Hello", html_parser).xpath(".//b")[0]
assert isinstance(b, Phrasing)
assert b.is_phrasing is True
# -- .iter_text_segments() --------------------------------------------
@pytest.mark.parametrize(
("html_text", "expected_value"),
[
# -- an empty element produces no text segments --
("", []),
# -- element text produces one segment --
(" foo ", [(" foo ", {})]),
# -- element tail produces one segment --
(" bar ", [(" bar ", {})]),
# -- element descendants each produce one segment --
("foo bar", [("foo ", {}), ("bar", {})]),
# -- and any combination produces a segment for each text, child, and tail --
(
" foo bar baz ",
[
(" ", {}),
("foo ", {}),
("bar", {}),
(" baz", {}),
(" ", {}),
],
),
],
)
def it_generates_text_segments_for_its_text_and_children_and_tail(
self, html_text: str, expected_value: list[TextSegment]
):
e = etree.fromstring(html_text, html_parser).xpath(".//body")[0][0]
assert list(e.iter_text_segments()) == expected_value
@pytest.mark.parametrize(
("html_text", "expected_value"),
[
# -- Phrasing with nested block but no text or tail produces only element for block --
("
aaa
", [Text("aaa")]),
# -- Phrasing with text produces annotated text-segment for the text --
(
"aaa
bbb
",
[
TextSegment(
"aaa", {"emphasized_text_contents": "aaa", "emphasized_text_tags": "b"}
),
Text("bbb"),
],
),
# -- Phrasing with tail produces annotated text-segment for the tail --
(
"
aaa
bbb",
[
Text("aaa"),
TextSegment(
"bbb", {"emphasized_text_contents": "bbb", "emphasized_text_tags": "b"}
),
],
),
# -- Phrasing with text, nested block, and tail produces all three --
(
"aaa
bbb
ccc",
[
TextSegment(
"aaa", {"emphasized_text_contents": "aaa", "emphasized_text_tags": "b"}
),
Text("bbb"),
TextSegment(
"ccc", {"emphasized_text_contents": "ccc", "emphasized_text_tags": "b"}
),
],
),
],
)
def but_it_can_also_generate_an_element_when_it_has_a_nested_block_element(
self, html_text: str, expected_value: list[TextSegment | Element]
):
e = etree.fromstring(html_text, html_parser).xpath(".//body")[0][0]
assert list(e.iter_text_segments()) == expected_value
# -- ._annotation() ---------------------------------------------------
def it_forms_its_annotations_from_emphasis(self):
cite = etree.fromstring("", html_parser).xpath(".//cite")[0]
assert cite._annotation("\n foobar\n ", "bi") == {
"emphasized_text_contents": "foobar",
"emphasized_text_tags": "bi",
}
@pytest.mark.parametrize("text", ["", "\n \t "])
def but_not_when_text_is_empty_or_whitespace(self, text: str):
cite = etree.fromstring("", html_parser).xpath(".//cite")[0]
assert cite._annotation(text, "bi") == {}
def and_not_when_there_is_no_emphasis(self):
cite = etree.fromstring("", html_parser).xpath(".//cite")[0]
assert cite._annotation("foobar", "") == {}
# -- ._inside_emphasis() ----------------------------------------------
@pytest.mark.parametrize("enclosing_emphasis", ["", "b", "bi"])
def it_uses_the_enclosing_emphasis_as_the_default_inside_emphasis(
self, enclosing_emphasis: str
):
"""Inside emphasis is applied to text inside the phrasing element (but not its tail).
The `._inside_emphasis()` method is overridden by Bold and Italic classes which add their
specific emphasis characters.
"""
abbr = etree.fromstring("", html_parser).xpath(".//abbr")[0]
assert abbr._inside_emphasis(enclosing_emphasis) == enclosing_emphasis
# -- ._iter_child_text_segments() -------------------------------------
@pytest.mark.parametrize(
("html_text", "expected_value"),
[
# -- a phrasing element with no children produces no text segments
# -- (element text is handled elsewhere)
("aaa", []),
# -- child phrasing element produces text-segment for its text --
("xbbb", [TextSegment("bbb", {})]),
# -- and also for its tail when it has one --
("xbbbccc", [TextSegment("bbb", {}), TextSegment("ccc", {})]),
# -- nested phrasing recursively each produce a segment for text and tail, in order --
(
"xxxaaabbbcccdddeeefff",
[
TextSegment("aaa", {}),
TextSegment("bbb", {}),
TextSegment("ccc", {}),
TextSegment("ddd", {}),
TextSegment("eee", {}),
TextSegment("fff", {}),
],
),
],
)
def it_generates_text_segments_for_its_children_and_their_tails(
self, html_text: str, expected_value: list[TextSegment]
):
e = etree.fromstring(html_text, html_parser).xpath(".//body")[0][0]
assert list(e._iter_child_text_segments("")) == expected_value
@pytest.mark.parametrize(
("html_text", "inside_emphasis", "expected_value"),
[
# -- a phrasing element with no block children produces no elements --
("", "", []),
# -- a child block element produces an element --
("
aaa
", "", [Text("aaa")]),
# -- a child block element with a tail also produces a text-segment for the tail --
("
aaa
bbb", "", [Text("aaa"), TextSegment("bbb", {})]),
# -- and also text-segments for phrasing following the tail --
(
"
aaa
bbbcccddd",
"",
[
Text("aaa"),
TextSegment("bbb", {}),
TextSegment("ccc", {}),
TextSegment("ddd", {}),
],
),
# -- and emphasis is applied before and after block-item --
(
"aaa
dddeee",
"",
[
TextSegment("aaa", {}),
TextSegment("bbb", {}),
Text("ccc"),
TextSegment("ddd", {}),
TextSegment("eee", {}),
],
),
],
)
def it_generates_text_segments_from_the_tail_and_contiguous_phrasing(
self, html_text: str, emphasis: str, expected_value: list[TextSegment | Element]
):
e = etree.fromstring(html_text, html_parser).xpath(".//body")[0][0]
p = e.xpath("./p")[0]
tail = p.tail or ""
q = deque(e[1:])
assert (
list(e._iter_text_segments_from_block_tail_and_phrasing(tail, q, emphasis))
== expected_value
)
class DescribeAnchor:
"""Isolated unit-test suite for `unstructured.partition.html.parser.Anchor`.
The `Anchor` class is used for `` tags and provides link metadata.
"""
# -- .iter_text_segments() --------------------------------------------
@pytest.mark.parametrize(
("html_text", "emphasis", "expected_value"),
[
# -- produces no text-segment or annotation for anchor.text when there is none --
('', "", []),
# -- but it produces a text-segment for the tail if there is one --
(' long tail ', "", [TextSegment(" long tail ", {})]),
# -- produces text-segment but no annotation for anchor.text when it is whitespace --
('', "", [TextSegment(" ", {})]),
# -- produces text-segment and annotation for anchor text. Note `link_texts:`
# -- annotation value is whitespace-normalized but text-segment text is not.
(
' click here ',
"",
[
TextSegment(
" click here ",
{"link_texts": ["click here"], "link_urls": ["http://abc.com"]},
)
],
),
# -- produces text-segment for both text and tail when present --
(
' click here long tail',
"",
[
TextSegment(
" click here ",
{"link_texts": ["click here"], "link_urls": ["http://abc.com"]},
),
TextSegment(" long tail", {}),
],
),
# -- nested phrasing inside element is handled as expected --
(
'
the Force."""
a = etree.fromstring(html_text, html_parser).xpath(".//a")[0]
actual = list(a.iter_text_segments("i"))
assert actual == [
TextSegment(" \n ", {}),
NarrativeText("I am one with"),
TextSegment(
" the Force.",
{
"emphasized_text_contents": "the Force.",
"emphasized_text_tags": "i",
},
),
]
element = actual[1]
assert element.metadata.link_texts == ["I am one with"]
assert element.metadata.link_urls == ["http://eie.io"]
# -- ._iter_phrases_and_elements() ------------------------------------
def it_divides_the_anchor_contents_but_not_tail_into_phrases_and_elements(self):
html_text = """
But always
Otherwise you
"""
a = etree.fromstring(html_text, html_parser).xpath(".//a")[0]
assert list(a._iter_phrasing(text=a.text, q=deque(a), emphasis="")) == [
(
TextSegment("But ", {}),
TextSegment(
"always ",
{
"emphasized_text_contents": "always",
"emphasized_text_tags": "b",
},
),
),
NarrativeText("see first."),
(
TextSegment(
"Otherwise",
{
"emphasized_text_contents": "Otherwise",
"emphasized_text_tags": "b",
},
),
TextSegment(" you ", {}),
),
]
# -- ._link_annotate_element() ----------------------------------------
def it_adds_link_metadata_to_an_element_to_help(self):
html_text = """"""
a = etree.fromstring(html_text, html_parser).xpath(".//a")[0]
element = Text("aaa")
e = a._link_annotate_element(element)
assert e is element
assert e.metadata.link_texts == ["aaa"]
assert e.metadata.link_urls == ["http://eie.io"]
def and_it_preserves_any_existing_link_metadata_on_the_element(self):
# -- nested anchors shouldn't be possible but easier to test than prove it can't happen --
html_text = """"""
a = etree.fromstring(html_text, html_parser).xpath(".//a")[0]
element = Text("bbb")
element.metadata.link_texts = ["abc"]
element.metadata.link_urls = ["http://abc.com"]
e = a._link_annotate_element(element)
assert e is element
assert e.metadata.link_texts == ["abc", "bbb"]
assert e.metadata.link_urls == ["http://abc.com", "http://eie.io"]
def but_not_when_the_text_is_empty(self):
html_text = """"""
a = etree.fromstring(html_text, html_parser).xpath(".//a")[0]
element = Text("")
e = a._link_annotate_element(element)
assert e is element
assert e.metadata.link_texts is None
assert e.metadata.link_urls is None
def and_not_when_there_is_no_url(self):
html_text = """"""
a = etree.fromstring(html_text, html_parser).xpath(".//a")[0]
element = Text("zzz")
e = a._link_annotate_element(element)
assert e is element
assert e.metadata.link_texts is None
assert e.metadata.link_urls is None
# -- ._link_text_segment() --------------------------------------------
def it_consolidates_a_phrase_into_a_single_link_annotated_TextSegment_to_help(self):
html_text = """"""
a = etree.fromstring(html_text, html_parser).xpath(".//a")[0]
phrase = (
TextSegment(
"Otherwise you will only ",
{
"emphasized_text_contents": ["Otherwise"],
"emphasized_text_tags": ["i"],
},
),
TextSegment(
"see what you were expecting.\n",
{
"emphasized_text_contents": "expecting",
"emphasized_text_tags": "b",
},
),
)
link_text_segment = a._link_text_segment(phrase)
assert link_text_segment == TextSegment(
"Otherwise you will only see what you were expecting.\n",
{
"emphasized_text_contents": ["Otherwise", "expecting"],
"emphasized_text_tags": ["i", "b"],
"link_texts": ["Otherwise you will only see what you were expecting."],
"link_urls": ["http://eie.io"],
},
)
@pytest.mark.parametrize("text", ["", " \n \t "])
def but_not_when_the_text_is_empty_or_whitespace_only(self, text: str):
html_text = """"""
a = etree.fromstring(html_text, html_parser).xpath(".//a")[0]
phrase = (TextSegment(text, {}), TextSegment(text, {}), TextSegment(text, {}))
assert a._link_text_segment(phrase) is None
def and_not_when_the_anchor_has_no_href_url(self):
html_text = """foobar"""
a = etree.fromstring(html_text, html_parser).xpath(".//a")[0]
phrase = (TextSegment("Otherwise", {}), TextSegment(" you will", {}))
assert a._link_text_segment(phrase) is None
class DescribeBold:
"""Isolated unit-test suite for `unstructured.partition.html.parser.Bold`.
The `Bold` class is used for `` and `` tags and adds emphasis metadata.
"""
def it_annotates_its_text_segment_with_bold_emphasis(self):
b = etree.fromstring("rhombus", html_parser).xpath(".//b")[0]
text_segments = b.iter_text_segments()
text, annotation = next(text_segments)
assert text == "rhombus"
assert annotation == {
"emphasized_text_contents": "rhombus",
"emphasized_text_tags": "b",
}
def and_its_children_are_also_annotated_with_bold_emphasis(self):
b = etree.fromstring("rhombus pentagon", html_parser).xpath(".//b")[0]
text_segments = b.iter_text_segments()
text, annotation = next(text_segments)
assert text == "rhombus "
assert annotation == {
"emphasized_text_contents": "rhombus",
"emphasized_text_tags": "b",
}
text, annotation = next(text_segments)
assert text == "pentagon"
assert annotation == {
"emphasized_text_contents": "pentagon",
"emphasized_text_tags": "bi",
}
def but_not_its_tail(self):
b = etree.fromstring("rhombus pentagon", html_parser).xpath(".//b")[0]
text_segments = b.iter_text_segments()
text, annotation = next(text_segments)
assert text == "rhombus"
assert annotation == {
"emphasized_text_contents": "rhombus",
"emphasized_text_tags": "b",
}
text, annotation = next(text_segments)
assert text == " pentagon"
assert annotation == {}
class DescribeItalic:
"""Isolated unit-test suite for `unstructured.partition.html.parser.Italic`.
The `Italic` class is used for `` and `` tags and adds emphasis metadata.
"""
def it_annotates_its_text_segment_with_italic_emphasis(self):
i = etree.fromstring("rhombus", html_parser).xpath(".//i")[0]
text_segments = i.iter_text_segments()
text, annotation = next(text_segments)
assert text == "rhombus"
assert annotation == {
"emphasized_text_contents": "rhombus",
"emphasized_text_tags": "i",
}
def and_its_children_are_also_annotated_with_italic_emphasis(self):
em = etree.fromstring("rhombus pentagon", html_parser).xpath(".//em")[0]
text_segments = em.iter_text_segments()
text, annotation = next(text_segments)
assert text == "rhombus "
assert annotation == {
"emphasized_text_contents": "rhombus",
"emphasized_text_tags": "i",
}
text, annotation = next(text_segments)
assert text == "pentagon"
assert annotation == {
"emphasized_text_contents": "pentagon",
"emphasized_text_tags": "bi",
}
def but_not_its_tail(self):
i = etree.fromstring("rhombus pentagon", html_parser).xpath(".//i")[0]
text_segments = i.iter_text_segments()
text, annotation = next(text_segments)
assert text == "rhombus"
assert annotation == {
"emphasized_text_contents": "rhombus",
"emphasized_text_tags": "i",
}
text, annotation = next(text_segments)
assert text == " pentagon"
assert annotation == {}
class DescribeLineBreak:
"""Isolated unit-test suite for `unstructured.partition.html.parser.LineBreak`.
Used for ` ` elements, it's only special behavior is to add whitespace such that phrasing
butted up tight on both sides of the ` ` element is not joined, like `abc def` should
become "abc def", not "abcdef".
"""
def it_adds_a_newline_in_its_place(self):
cite = etree.fromstring(
"spaceships of the Vogon Constructor Fleet", html_parser
).xpath(".//cite")[0]
text_segments = cite.iter_text_segments()
texts = [ts.text for ts in text_segments]
assert texts == ["spaceships of the", "\n", "Vogon Constructor Fleet"]
assert _normalize_text("".join(texts)) == "spaceships of the Vogon Constructor Fleet"
class DescribeRemovedPhrasing:
"""Isolated unit-test suite for `unstructured.partition.html.parser.RemovedPhrasing`.
Used for phrasing elements like `
