learningcircuit--local-deep-research/tests/content_fetcher/test_html_downloader.py

"""
Tests for HTML downloader.
"""

from unittest.mock import MagicMock, patch

from local_deep_research.research_library.downloaders.html import HTMLDownloader
from local_deep_research.research_library.downloaders.base import ContentType


class TestHTMLDownloaderCanHandle:
    """Test URL handling capability."""

    def test_can_handle_http(self):
        """Test HTTP URLs are handled."""
        downloader = HTMLDownloader()
        assert downloader.can_handle("http://example.com/article")

    def test_can_handle_https(self):
        """Test HTTPS URLs are handled."""
        downloader = HTMLDownloader()
        assert downloader.can_handle("https://example.com/article")

    def test_cannot_handle_ftp(self):
        """Test FTP URLs are not handled."""
        downloader = HTMLDownloader()
        assert not downloader.can_handle("ftp://example.com/file")

    def test_cannot_handle_invalid(self):
        """Test invalid URLs are not handled."""
        downloader = HTMLDownloader()
        assert not downloader.can_handle("not-a-url")


class TestHTMLDownloaderExtraction:
    """Test content extraction from HTML."""

    @patch(
        "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html"
    )
    def test_extract_title(self, mock_fetch, mock_html_response):
        """Test title extraction."""
        mock_fetch.return_value = mock_html_response

        downloader = HTMLDownloader()
        result = downloader.download_text("https://example.com/article")

        assert result is not None
        assert "Test Article Title" in result

    @patch(
        "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html"
    )
    def test_extract_content(self, mock_fetch, mock_html_response):
        """Test main content extraction."""
        mock_fetch.return_value = mock_html_response

        downloader = HTMLDownloader()
        result = downloader.download_text("https://example.com/article")

        assert result is not None
        assert "first paragraph" in result
        assert "second paragraph" in result

    @patch(
        "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html"
    )
    def test_removes_navigation(self, mock_fetch, mock_html_response):
        """Test navigation elements are removed."""
        mock_fetch.return_value = mock_html_response

        downloader = HTMLDownloader()
        result = downloader.download_text("https://example.com/article")

        assert result is not None
        assert "Navigation menu" not in result

    @patch(
        "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html"
    )
    def test_removes_footer(self, mock_fetch, mock_html_response):
        """Test footer elements are removed."""
        mock_fetch.return_value = mock_html_response

        downloader = HTMLDownloader()
        result = downloader.download_text("https://example.com/article")

        assert result is not None
        assert "Footer content" not in result

    @patch(
        "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html"
    )
    def test_extract_description(self, mock_fetch, mock_html_response):
        """Test meta description extraction."""
        mock_fetch.return_value = mock_html_response

        downloader = HTMLDownloader()
        result = downloader.download_text("https://example.com/article")

        assert result is not None
        assert "test article description" in result


class TestHTMLDownloaderDownload:
    """Test download functionality."""

    @patch(
        "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html"
    )
    def test_download_returns_bytes(self, mock_fetch, mock_html_response):
        """Test download returns bytes."""
        mock_fetch.return_value = mock_html_response

        downloader = HTMLDownloader()
        result = downloader.download(
            "https://example.com/article", ContentType.TEXT
        )

        assert result is not None
        assert isinstance(result, bytes)

    @patch(
        "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html"
    )
    def test_download_pdf_not_supported(self, mock_fetch):
        """Test PDF download returns None."""
        downloader = HTMLDownloader()
        result = downloader.download(
            "https://example.com/article", ContentType.PDF
        )

        assert result is None
        mock_fetch.assert_not_called()

    @patch(
        "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html"
    )
    def test_download_with_result_success(self, mock_fetch, mock_html_response):
        """Test download_with_result on success."""
        mock_fetch.return_value = mock_html_response

        downloader = HTMLDownloader()
        result = downloader.download_with_result(
            "https://example.com/article", ContentType.TEXT
        )

        assert result.is_success
        assert result.content is not None
        assert result.skip_reason is None

    @patch(
        "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html"
    )
    def test_download_with_result_failure(self, mock_fetch):
        """Test download_with_result on failure."""
        mock_fetch.return_value = None

        downloader = HTMLDownloader()
        result = downloader.download_with_result(
            "https://example.com/article", ContentType.TEXT
        )

        assert not result.is_success
        assert result.content is None
        assert result.skip_reason is not None


class TestHTMLDownloaderFetchHTML:
    """Test HTML fetching."""

    @patch("local_deep_research.security.SafeSession")
    def test_fetch_html_success(
        self, mock_session_class, mock_successful_response
    ):
        """Test successful HTML fetch."""
        mock_session = MagicMock()
        mock_session.headers = {}
        mock_session.get.return_value = mock_successful_response
        mock_session_class.return_value = mock_session

        downloader = HTMLDownloader()
        downloader.session = mock_session

        result = downloader._fetch_html("https://example.com/article")

        assert result is not None
        mock_session.get.assert_called_once()

    @patch("local_deep_research.security.SafeSession")
    def test_fetch_html_404(self, mock_session_class):
        """Test 404 response."""
        mock_session = MagicMock()
        mock_session.headers = {}
        mock_response = MagicMock()
        mock_response.status_code = 404
        mock_session.get.return_value = mock_response
        mock_session_class.return_value = mock_session

        downloader = HTMLDownloader()
        downloader.session = mock_session

        result = downloader._fetch_html("https://example.com/notfound")

        assert result is None

    @patch("local_deep_research.security.SafeSession")
    def test_fetch_html_wrong_content_type(self, mock_session_class):
        """Test non-HTML content type."""
        mock_session = MagicMock()
        mock_session.headers = {}
        mock_response = MagicMock()
        mock_response.status_code = 200
        mock_response.headers = {"content-type": "application/json"}
        mock_session.get.return_value = mock_response
        mock_session_class.return_value = mock_session

        downloader = HTMLDownloader()
        downloader.session = mock_session

        result = downloader._fetch_html("https://api.example.com/data")

        assert result is None


class TestHTMLDownloaderMetadata:
    """Test metadata extraction."""

    @patch(
        "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html"
    )
    def test_get_metadata(self, mock_fetch, mock_html_response):
        """Test metadata extraction."""
        mock_fetch.return_value = mock_html_response

        downloader = HTMLDownloader()
        metadata = downloader.get_metadata("https://example.com/article")

        assert metadata.get("url") == "https://example.com/article"
        assert metadata.get("title") == "Test Article Title"
        assert (
            metadata.get("description") == "This is a test article description."
        )
        assert metadata.get("author") == "Test Author"

    @patch(
        "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html"
    )
    def test_get_metadata_fetch_failure(self, mock_fetch):
        """Test metadata extraction when fetch fails."""
        mock_fetch.return_value = None

        downloader = HTMLDownloader()
        metadata = downloader.get_metadata("https://example.com/article")

        assert metadata == {}


class TestHTMLDownloaderEdgeCases:
    """Test edge cases."""

    @patch(
        "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html"
    )
    def test_empty_html(self, mock_fetch):
        """Test handling of empty HTML."""
        mock_fetch.return_value = "<html><body></body></html>"

        downloader = HTMLDownloader()
        result = downloader.download_text("https://example.com/empty")

        # Should return None for empty content
        assert result is None

    @patch(
        "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html"
    )
    def test_malformed_html(self, mock_fetch):
        """Test handling of malformed HTML."""
        mock_fetch.return_value = "<html><body><p>Unclosed paragraph"

        downloader = HTMLDownloader()
        # Should not raise exception
        result = downloader.download_text("https://example.com/malformed")

        # BeautifulSoup handles malformed HTML gracefully
        assert result is None or isinstance(result, str)

    @patch(
        "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html"
    )
    def test_unicode_content(self, mock_fetch):
        """Test handling of Unicode content."""
        mock_fetch.return_value = """
        <html>
        <head><title>Unicode Test</title></head>
        <body>
            <article>
                <p>This has émojis 🎉 and spëcial çharacters.</p>
                <p>Chinese: 中文内容</p>
                <p>Japanese: 日本語のテキスト</p>
            </article>
        </body>
        </html>
        """

        downloader = HTMLDownloader()
        result = downloader.download_text("https://example.com/unicode")

        assert result is not None
        assert "émojis" in result or "mojis" in result  # Unicode handling
        assert "中文" in result or "Chinese" in result

    @patch(
        "local_deep_research.research_library.downloaders.html.HTMLDownloader._fetch_html"
    )
    def test_very_long_content(self, mock_fetch):
        """Test handling of very long content."""
        # Create HTML with many paragraphs
        paragraphs = "\n".join(
            f"<p>Paragraph {i} with some content that is long enough to matter.</p>"
            for i in range(100)
        )
        mock_fetch.return_value = f"""
        <html>
        <head><title>Long Article</title></head>
        <body><article>{paragraphs}</article></body>
        </html>
        """

        downloader = HTMLDownloader()
        result = downloader.download_text("https://example.com/long")

        assert result is not None
        assert len(result) > 1000