"""Unit tests for HTMLDownloader._extract_content and _format_extracted_content. Production code: src/local_deep_research/research_library/downloaders/html.py """ from local_deep_research.research_library.downloaders.html import HTMLDownloader # --------------------------------------------------------------------------- # _extract_content # --------------------------------------------------------------------------- def _long_text(n=120): """Generate text longer than the 100-char fallback threshold.""" return "A" * n class TestExtractContent: """Tests for HTMLDownloader._extract_content HTML parsing.""" def setup_method(self): self.downloader = HTMLDownloader() def test_minimal_html_with_body(self): long = _long_text() html = f"

{long}

" result = self.downloader._extract_content(html, "http://example.com") assert result is not None assert long[:50] in result["content"] def test_article_tag_preferred(self): long = _long_text() html = f"""
sidebar stuff that is long enough to pass {_long_text()}

{long}

""" result = self.downloader._extract_content(html, "http://example.com") assert result is not None assert long[:50] in result["content"] def test_main_tag_fallback(self): long = _long_text() html = f"

{long}

" result = self.downloader._extract_content(html, "http://example.com") assert result is not None assert long[:50] in result["content"] def test_content_div_fallback(self): long = _long_text() html = f'

{long}

' result = self.downloader._extract_content(html, "http://example.com") assert result is not None assert long[:50] in result["content"] def test_script_style_nav_removed(self): long = _long_text() html = f"""

{long}

""" result = self.downloader._extract_content(html, "http://example.com") assert result is not None assert "var x" not in result["content"] assert "color: red" not in result["content"] def test_sidebar_pattern_removed(self): long = _long_text() html = f"""
sidebar content with some navigation links and additional information

{long}

""" result = self.downloader._extract_content(html, "http://example.com") assert result is not None assert long[:50] in result["content"] def test_short_content_extracts_via_justext(self): """Short but meaningful content is extracted by justext.""" html = "

Short text here but enough to be above fifty characters for the final check.

" result = self.downloader._extract_content(html, "http://example.com") assert result is not None assert "Short text" in result["content"] def test_no_body_returns_none(self): # No meaningful content in head-only document html = "Title" result = self.downloader._extract_content(html, "http://example.com") assert result is None def test_title_extraction(self): long = _long_text() html = f"My Title

{long}

" result = self.downloader._extract_content(html, "http://example.com") assert result is not None assert result["title"] == "My Title" def test_url_passed_through(self): long = _long_text() html = f"

{long}

" result = self.downloader._extract_content( html, "http://example.com/page" ) assert result["url"] == "http://example.com/page" # --------------------------------------------------------------------------- # _format_extracted_content # --------------------------------------------------------------------------- class TestFormatExtractedContent: """Tests for HTMLDownloader._format_extracted_content.""" def setup_method(self): self.downloader = HTMLDownloader() def test_full_dict(self): extracted = { "title": "My Title", "description": "A description", "url": "http://example.com", "content": "Main content here", } result = self.downloader._format_extracted_content(extracted) assert "# My Title" in result assert "*A description*" in result assert "Source: http://example.com" in result assert "Main content here" in result def test_missing_optional_fields(self): extracted = { "title": None, "description": None, "url": "http://example.com", "content": "Main content", } result = self.downloader._format_extracted_content(extracted) assert "#" not in result # no title heading assert "*" not in result # no description italic assert "Source: http://example.com" in result assert "Main content" in result def test_empty_content(self): extracted = { "title": None, "description": None, "url": None, "content": None, } result = self.downloader._format_extracted_content(extracted) # Should not crash, just return minimal/empty output assert isinstance(result, str)