import random import unittest from types import SimpleNamespace from unittest.mock import patch from olmocr.synth.mine_html_templates import generate_tests_from_html class TestFootnoteTestGeneration(unittest.TestCase): def setUp(self): self.random_gen = random.Random(42) self.pdf_id = "test_pdf" self.page_num = 1 self.uuid_counter = 0 uuid_patch = patch( "olmocr.synth.mine_html_templates.uuid.uuid4", side_effect=self._fake_uuid, ) self.addCleanup(uuid_patch.stop) uuid_patch.start() def _fake_uuid(self): current = f"{self.uuid_counter:032x}" self.uuid_counter += 1 return SimpleNamespace(hex=current) @staticmethod def _hashable_tests(tests): return {tuple(sorted(test.items())) for test in tests} def _generate_footnote_tests(self, html_content): tests = generate_tests_from_html( html_content, self.pdf_id, self.page_num, self.random_gen, False, ) footnote_tests = [] for test in tests: if test.get("type") == "footnote": sanitized = {k: v for k, v in test.items() if k != "id"} footnote_tests.append(sanitized) return self._hashable_tests(footnote_tests) def test_single_marker_generates_marker_only(self): html_content = """
Alpha with reference1 and more text.
""" footnote_tests = self._generate_footnote_tests(html_content) self.assertSetEqual( footnote_tests, self._hashable_tests( [ { "pdf": "test_pdf_page1.pdf", "page": 1, "type": "footnote", "marker": "1", "max_diffs": 0, "appears_before_marker": "Alpha with reference", } ] ), ) def test_marker_with_definition_includes_text(self): html_content = """Alpha with reference1 and more text.
1This is the footnote text that elaborates on the reference in detail.
""" footnote_tests = self._generate_footnote_tests(html_content) self.assertSetEqual( footnote_tests, self._hashable_tests( [ { "pdf": "test_pdf_page1.pdf", "page": 1, "type": "footnote", "marker": "1", "max_diffs": 0, "appears_before_marker": "Alpha with reference", "appears_after_marker": "This is the", } ] ), ) def test_multiple_markers_mixed_occurrences(self): html_content = """First paragraph ends with marker1 in the sentence.
Second paragraph carries marker2 for another note.
Here is a lone marker3 with no definition below.
1Definition for footnote one with ample descriptive content to qualify.
2Definition for footnote two including enough characters to be captured correctly.
""" footnote_tests = self._generate_footnote_tests(html_content) self.assertSetEqual( footnote_tests, self._hashable_tests( [ { "pdf": "test_pdf_page1.pdf", "page": 1, "type": "footnote", "marker": "1", "max_diffs": 0, "appears_before_marker": "ends with marker", "appears_after_marker": "Definition for footnote", }, { "pdf": "test_pdf_page1.pdf", "page": 1, "type": "footnote", "marker": "2", "max_diffs": 0, "appears_before_marker": "paragraph carries marker", "appears_after_marker": "Definition for footnote", }, { "pdf": "test_pdf_page1.pdf", "page": 1, "type": "footnote", "marker": "3", "max_diffs": 0, "appears_before_marker": "a lone marker", }, ] ), ) def test_cutoff(self): html_content = """Your personal data is processed in accordance with Regulation (EU) No 2018/17251 on the protection of individuals with regard to the processing of personal data by the Union institutions, bodies, offices and agencies and on the free movement of such data.""" footnote_tests = self._generate_footnote_tests(html_content) self.assertEqual(len(footnote_tests), 1) self.assertSetEqual( footnote_tests, self._hashable_tests( [ { "pdf": "test_pdf_page1.pdf", "page": 1, "type": "footnote", "marker": "1", "max_diffs": 0, "appears_before_marker": "(EU) No 2018/1725", }, ] ), ) def test_footnotes_in_footer(self): # Sometimes footnotes appear in a