learningcircuit--local-deep…/tests/content_fetcher/test_html_content_extraction.py

"""Unit tests for HTMLDownloader._extract_content and _format_extracted_content.

Production code: src/local_deep_research/research_library/downloaders/html.py
"""

from local_deep_research.research_library.downloaders.html import HTMLDownloader


# ---------------------------------------------------------------------------
# _extract_content
# ---------------------------------------------------------------------------


def _long_text(n=120):
    """Generate text longer than the 100-char fallback threshold."""
    return "A" * n


class TestExtractContent:
    """Tests for HTMLDownloader._extract_content HTML parsing."""

    def setup_method(self):
        self.downloader = HTMLDownloader()

    def test_minimal_html_with_body(self):
        long = _long_text()
        html = f"<html><body><p>{long}</p></body></html>"
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is not None
        assert long[:50] in result["content"]

    def test_article_tag_preferred(self):
        long = _long_text()
        html = f"""<html><body>
            <div>sidebar stuff that is long enough to pass {_long_text()}</div>
            <article><p>{long}</p></article>
        </body></html>"""
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is not None
        assert long[:50] in result["content"]

    def test_main_tag_fallback(self):
        long = _long_text()
        html = f"<html><body><main><p>{long}</p></main></body></html>"
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is not None
        assert long[:50] in result["content"]

    def test_content_div_fallback(self):
        long = _long_text()
        html = f'<html><body><div class="content"><p>{long}</p></div></body></html>'
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is not None
        assert long[:50] in result["content"]

    def test_script_style_nav_removed(self):
        long = _long_text()
        html = f"""<html><body>
            <script>var x = 1;</script>
            <style>.foo {{ color: red; }}</style>
            <nav>navigation links</nav>
            <p>{long}</p>
        </body></html>"""
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is not None
        assert "var x" not in result["content"]
        assert "color: red" not in result["content"]

    def test_sidebar_pattern_removed(self):
        long = _long_text()
        html = f"""<html><body>
            <nav><a href="/">Home</a><a href="/about">About</a></nav>
            <div class="sidebar">sidebar content with some navigation links and additional information</div>
            <article><p>{long}</p></article>
            <footer><p>Copyright 2024 All rights reserved terms privacy contact</p></footer>
        </body></html>"""
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is not None
        assert long[:50] in result["content"]

    def test_short_content_extracts_via_justext(self):
        """Short but meaningful content is extracted by justext."""
        html = "<html><body><p>Short text here but enough to be above fifty characters for the final check.</p></body></html>"
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is not None
        assert "Short text" in result["content"]

    def test_no_body_returns_none(self):
        # No meaningful content in head-only document
        html = "<html><head><title>Title</title></head></html>"
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is None

    def test_title_extraction(self):
        long = _long_text()
        html = f"<html><head><title>My Title</title></head><body><p>{long}</p></body></html>"
        result = self.downloader._extract_content(html, "http://example.com")
        assert result is not None
        assert result["title"] == "My Title"

    def test_url_passed_through(self):
        long = _long_text()
        html = f"<html><body><p>{long}</p></body></html>"
        result = self.downloader._extract_content(
            html, "http://example.com/page"
        )
        assert result["url"] == "http://example.com/page"


# ---------------------------------------------------------------------------
# _format_extracted_content
# ---------------------------------------------------------------------------


class TestFormatExtractedContent:
    """Tests for HTMLDownloader._format_extracted_content."""

    def setup_method(self):
        self.downloader = HTMLDownloader()

    def test_full_dict(self):
        extracted = {
            "title": "My Title",
            "description": "A description",
            "url": "http://example.com",
            "content": "Main content here",
        }
        result = self.downloader._format_extracted_content(extracted)
        assert "# My Title" in result
        assert "*A description*" in result
        assert "Source: http://example.com" in result
        assert "Main content here" in result

    def test_missing_optional_fields(self):
        extracted = {
            "title": None,
            "description": None,
            "url": "http://example.com",
            "content": "Main content",
        }
        result = self.downloader._format_extracted_content(extracted)
        assert "#" not in result  # no title heading
        assert "*" not in result  # no description italic
        assert "Source: http://example.com" in result
        assert "Main content" in result

    def test_empty_content(self):
        extracted = {
            "title": None,
            "description": None,
            "url": None,
            "content": None,
        }
        result = self.downloader._format_extracted_content(extracted)
        # Should not crash, just return minimal/empty output
        assert isinstance(result, str)