import random import unittest from unittest.mock import MagicMock, patch from bs4 import BeautifulSoup from olmocr.bench.tests import TestType from olmocr.synth.mine_html_templates import ( PreserveTablesConverter, extract_html_metadata, generate_tests_from_html, html_to_markdown_with_frontmatter, ) class TestMathExtraction(unittest.TestCase): """Test the math extraction functionality in mine_html_templates.py""" def setUp(self): self.random_generator = random.Random(42) return super().setUp() def test_math_extraction_from_html(self): """Test that math equations are properly extracted from HTML content""" html_content = """

Some text with inline math \\(x = 2\\) here.

Display math: \\[E = mc^2\\]

Another inline: \\(\\alpha + \\beta = \\gamma\\)

Complex display: \\[\\int_0^\\infty e^{-x} dx = 1\\]

""" # Generate tests from HTML tests = generate_tests_from_html(html_content, "test_pdf", 1, self.random_generator) # Filter math tests math_tests = [t for t in tests if t.get("type") == "math"] # Check that we extracted math equations self.assertTrue(len(math_tests) > 0, "Should extract at least one math equation") # Check that specific equations were extracted math_contents = [t["math"] for t in math_tests] self.assertIn("x = 2", math_contents) self.assertIn("E = mc^2", math_contents) self.assertIn("\\alpha + \\beta = \\gamma", math_contents) self.assertIn("\\int_0^\\infty e^{-x} dx = 1", math_contents) def test_math_extraction_with_multiline(self): """Test extraction of multiline math equations""" html_content = """

Multiline equation: \\[ e_i = \\frac{e_i + \\varphi(e_i)}{2} + \\frac{e_i - \\varphi(e_i)}{2}, \\quad \\text{for } i \\in \\mathbb{N}. \\]

""" tests = generate_tests_from_html(html_content, "test_pdf", 1, self.random_generator) math_tests = [t for t in tests if t.get("type") == "math"] # Check multiline equation is captured self.assertTrue(len(math_tests) > 0) # Check that the multiline content is preserved (without excessive newlines) found_multiline = False for test in math_tests: if "\\frac{e_i + \\varphi(e_i)}{2}" in test["math"] and "\\mathbb{N}" in test["math"]: found_multiline = True break self.assertTrue(found_multiline, "Should extract multiline equation correctly") def test_math_extraction_deduplication(self): """Test that duplicate math equations are deduplicated""" html_content = """

First occurrence: \\[x^2 + y^2 = z^2\\]

Second occurrence: \\[x^2 + y^2 = z^2\\]

Third occurrence: \\[x^2 + y^2 = z^2\\]

""" tests = generate_tests_from_html(html_content, "test_pdf", 1, self.random_generator) math_tests = [t for t in tests if t.get("type") == "math"] # Count how many times the equation appears equation_count = sum(1 for t in math_tests if "x^2 + y^2 = z^2" in t["math"]) # Should only appear once due to deduplication self.assertEqual(equation_count, 1, "Duplicate equations should be deduplicated") def test_math_extraction_patterns(self): """Test different math delimiter patterns""" html_content = """

Pattern 1: \\(inline1\\)

Pattern 2: \\[display1\\]

Pattern 3: $$display2$$

""" tests = generate_tests_from_html(html_content, "test_pdf", 1, self.random_generator) math_tests = [t for t in tests if t.get("type") == "math"] math_contents = [t["math"] for t in math_tests] # Check all patterns are captured self.assertIn("inline1", math_contents) self.assertIn("display1", math_contents) self.assertIn("display2", math_contents) def test_math_extraction_minimum_length(self): """Test that very short equations are filtered out""" html_content = """

Short: \\(x\\)

Also short: \\[y\\]

Long enough: \\(x=1\\)

""" tests = generate_tests_from_html(html_content, "test_pdf", 1, self.random_generator) math_tests = [t for t in tests if t.get("type") == "math"] math_contents = [t["math"] for t in math_tests] # Short equations (length <= 2) should be filtered out self.assertNotIn("x", math_contents) self.assertNotIn("y", math_contents) # Longer equation should be included self.assertIn("x=1", math_contents) def test_math_validation_passes(self): """Test that valid math tests pass validation against markdown""" html_content = """

Test equation: \\[E = mc^2\\]

""" # Mock the validation to always pass for math tests with patch("olmocr.bench.tests.load_single_test") as mock_load: mock_test = MagicMock() mock_test.run.return_value = (True, None) mock_load.return_value = mock_test tests = generate_tests_from_html(html_content, "test_pdf", 1, self.random_generator) math_tests = [t for t in tests if t.get("type") == "math"] # Verify math test was created self.assertTrue(len(math_tests) > 0) # Verify test has correct structure for test in math_tests: self.assertEqual(test["type"], "math") self.assertIn("math", test) self.assertEqual(test["max_diffs"], 0) self.assertIn("id", test) self.assertIn("pdf", test) self.assertEqual(test["page"], 1) def test_complex_markdown_example(self): """Test with the complex markdown example provided by the user""" # Convert markdown to HTML-like structure for testing html_content = '\n\n\n \n \n Automorphisms of Order Two\n \n \n \n \n\n\n
\n

If \\(\\varphi \\in \\text{Aut}(E)\\) with \\(\\varphi^2 = id\\) we observe that

\n \\[e_i = \\frac{e_i + \\varphi(e_i)}{2} + \\frac{e_i - \\varphi(e_i)}{2}, \\quad \\text{for } i \\in \\mathbb{N}.\\]\n \n

Setting \\(a_i = e_i + \\varphi(e_i)/2\\) we have:

\n \n \n
\n \n
\n
Definition 5
\n

Let \\(\\varphi \\in \\text{Aut}(E)\\). We say that \\(\\varphi\\) is of canonical type if \\(\\varphi(e_i) \\in E_{(1)}\\) for all \\(i\\).

\n \n

If \\(\\varphi\\) is an automorphism of order 2 on \\(E\\), we have that \\(\\varphi\\) is of canonical type if and only if \\(a_i \\in E_{(1)}\\) for all \\(i\\). Let us fix a basis \\(\\beta = \\{e_1, e_2, \\ldots, e_n, \\ldots\\}\\) of the vector space \\(L\\) and an automorphism \\(\\varphi \\in \\text{Aut}(E)\\) such that \\(\\varphi^2 = id\\). Then \\(\\varphi\\), as a linear transformation, has eigenvalues \\(\\pm 1\\) and \\(-1\\) only, and moreover, there exists a basis of the vector space \\(E\\) consisting of eigenvectors. (It is well known from elementary Linear Algebra that this fact does not depend on the dimension of the vector space as long as the characteristic of \\(F\\) is different from 2.) Then \\(E = E(1) \\oplus E(-1)\\) where \\(E(t)\\) is the eigenspace for the eigenvalue \\(t\\) of the linear transformation \\(\\varphi\\). One considers the intersections \\(L(t) = L \\cap E(t)\\), \\(t = \\pm 1\\). Changing the basis \\(\\beta\\), if necessary, one may assume that \\(L(t)\\) is the span of \\(\\beta \\cap L(t)\\). Clearly this change of basis gives rise to a homogeneous automorphism of \\(E\\) and we can take the composition of it and then \\(\\varphi\\). We shall assume that such a change of basis has been done.

\n \n

Denote

\n \\[I_\\varphi = \\{n \\in \\mathbb{N} \\mid \\varphi(e_n) = \\pm e_n\\}.\\]\n
\n \n

We shall distinguish the following four possibilities:

\n \n
    \n
  1. \\(I_\\varphi = \\mathbb{N}\\).
    2. \n
  2. \\(I_\\varphi \\neq \\mathbb{N}\\) is infinite.
    3. \n
  3. \\(I_\\varphi\\) is finite and nonempty.
    4. \n
  4. \\(I_\\gamma = \\emptyset\\) for every linear basis \\(\\gamma\\) of \\(L\\).
    5. \n
\n \n

We shall call these automorphisms (and also the corresponding \\(\\mathbb{Z}_2\\)-gradings), automorphisms (or gradings) of type 1, 2, 3, and 4, respectively.

\n \n

The automorphisms of type 1 induce \\(\\mathbb{Z}_2\\)-gradings on \\(E\\) in which all generators of \\(E\\) are homogeneous. Such structures are called homogeneous \\(\\mathbb{Z}_2\\)-gradings on \\(E\\). The corresponding graded identities were completely studied in [22, 24, 29].

\n \n

We conclude this section with the following lemma.

\n \n
\n
Lemma 6
\n

Let \\(\\varphi\\) be an automorphism of order two of \\(E\\). Then \\(\\varphi\\) is of type 4 if and only if, for every \\(v \\in L\\) such that \\(\\varphi(v) = \\pm v\\), one has \\(v = 0\\).

\n \n
\n Proof Assume that \\(\\varphi\\) is of type 4 and let \\(v \\in L\\) with \\(\\varphi(v) = \\pm v\\). If \\(v \\neq 0\\), choose a basis \\(\\gamma\\) of \\(L\\) such that \\(v \\in \\gamma\\). Then \\(I_\\gamma \\neq \\emptyset\\), a contradiction. The converse follows by the same argument.\n ■\n
\n
\n \n

3    Automorphisms of order two of E

\n \n

From this point on, our goal is to survey recent developments regarding automorphisms of order two and the corresponding \\(\\mathbb{Z}_2\\)-gradings of the infinite-dimensional Grassmann algebra.

\n \n

Let \\(X = \\{e_1, \\ldots, e_n, \\ldots\\}\\). For each map \\(\\lambda : X \\to E\\), we can define the linear transformation \\(\\varphi : E \\to E\\) by

\n \n
\n \\[\\varphi(e_{i_1} \\cdots e_{i_n}) = \\lambda(e_{i_1}) \\cdots \\lambda(e_{i_n}),\\] (1)\n
\n \n

for all \\(n \\in \\mathbb{N}\\).

\n \n

We start with the next lemma.

\n \n
\n
Lemma 7
\n

The linear transformation \\(\\varphi\\) is an endomorphism of \\(E\\) if and only if

\n \\[\\lambda(e_i)\\lambda(e_j) + \\lambda(e_j)\\lambda(e_i) = 0, \\quad \\text{for all } i, j.\\]\n
\n \n \n\n' tests = generate_tests_from_html(html_content, "test_pdf", 1, self.random_generator) math_tests = [t for t in tests if t.get("type") == "math"] for test in math_tests: print(test) def test_math_extraction_strips_whitespace(self): """Test that extracted math equations have whitespace properly stripped""" html_content = """

\\[ x = y + z \\]

""" tests = generate_tests_from_html(html_content, "test_pdf", 1, self.random_generator) math_tests = [t for t in tests if t.get("type") == "math"] self.assertTrue(len(math_tests) > 0) # The equation should be stripped of leading/trailing whitespace self.assertEqual(math_tests[0]["math"].strip(), math_tests[0]["math"]) def test_math_not_in_present_tests(self): """Test that extracted math equations have whitespace properly stripped""" html_content = """

\\[x = y + z \\mathcal{x} \\]

\\[y = x + z \\mathcal{x} \\]

\\[q = r + z \\mathcal{x} \\]

""" tests = generate_tests_from_html(html_content, "test_pdf", 1, self.random_generator) present_tests = [t for t in tests if t.get("type") == "present"] self.assertTrue(len(present_tests) == 0) if __name__ == "__main__": unittest.main()