learningcircuit--local-deep-research/tests/content_fetcher/test_fetcher_coverage.py

"""
Comprehensive tests for ContentFetcher to improve coverage of fetcher.py.

Covers: get_url_info, _get_downloader (caching + all branches + ImportError),
fetch (invalid, SSRF, success, PDF, unicode errors, truncation, metadata,
failed download, exceptions, no downloader fallback), fetch_text, and
DEFAULT_MAX_CONTENT_LENGTH usage.
"""

import pytest
from unittest.mock import patch, MagicMock

from local_deep_research.content_fetcher.fetcher import (
    ContentFetcher,
    DEFAULT_MAX_CONTENT_LENGTH,
)
from local_deep_research.content_fetcher.url_classifier import URLType


# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------


def _make_download_result(
    content: bytes, is_success: bool = True, skip_reason=None
):
    """Build a mock DownloadResult."""
    result = MagicMock()
    result.is_success = is_success
    result.content = content
    result.skip_reason = skip_reason
    return result


# ---------------------------------------------------------------------------
# get_url_info
# ---------------------------------------------------------------------------


class TestGetUrlInfo:
    def test_arxiv_url(self):
        fetcher = ContentFetcher()
        info = fetcher.get_url_info("https://arxiv.org/abs/2301.12345")
        assert info["url_type"] == "arxiv"
        assert info["source_name"] == "arXiv"
        assert info["extracted_id"] == "2301.12345"

    def test_pubmed_url(self):
        fetcher = ContentFetcher()
        info = fetcher.get_url_info("https://pubmed.ncbi.nlm.nih.gov/12345678/")
        assert info["url_type"] == "pubmed"
        assert info["source_name"] == "PubMed"
        assert info["extracted_id"] == "12345678"

    def test_html_url(self):
        fetcher = ContentFetcher()
        info = fetcher.get_url_info("https://example.com/article")
        assert info["url_type"] == "html"
        assert info["source_name"] == "Web Page"
        assert info["extracted_id"] is None

    def test_invalid_url(self):
        fetcher = ContentFetcher()
        info = fetcher.get_url_info("javascript:alert(1)")
        assert info["url_type"] == "invalid"
        assert info["source_name"] == "Invalid URL"

    def test_doi_url(self):
        fetcher = ContentFetcher()
        info = fetcher.get_url_info("https://doi.org/10.1234/test")
        assert info["url_type"] == "doi"
        assert info["extracted_id"] == "10.1234/test"

    def test_pmc_url(self):
        fetcher = ContentFetcher()
        info = fetcher.get_url_info(
            "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC1234567/"
        )
        assert info["url_type"] == "pmc"
        assert info["extracted_id"] == "PMC1234567"

    def test_semantic_scholar_url(self):
        fetcher = ContentFetcher()
        hex40 = "a" * 40
        info = fetcher.get_url_info(
            f"https://www.semanticscholar.org/paper/Title/{hex40}"
        )
        assert info["url_type"] == "semantic_scholar"
        assert info["extracted_id"] == hex40

    def test_biorxiv_url(self):
        fetcher = ContentFetcher()
        info = fetcher.get_url_info(
            "https://www.biorxiv.org/content/10.1101/2023"
        )
        assert info["url_type"] == "biorxiv"

    def test_medrxiv_url(self):
        fetcher = ContentFetcher()
        info = fetcher.get_url_info(
            "https://www.medrxiv.org/content/10.1101/2023"
        )
        assert info["url_type"] == "medrxiv"

    def test_pdf_url(self):
        fetcher = ContentFetcher()
        info = fetcher.get_url_info("https://example.com/paper.pdf")
        assert info["url_type"] == "pdf"


# ---------------------------------------------------------------------------
# _get_downloader
# ---------------------------------------------------------------------------


class TestGetDownloader:
    """Test _get_downloader caching and all URLType branches."""

    def test_caching(self):
        """Second call for same type returns cached downloader."""
        fetcher = ContentFetcher()
        mock_dl = MagicMock()
        fetcher._downloaders[URLType.HTML] = mock_dl
        assert fetcher._get_downloader(URLType.HTML) is mock_dl

    def test_downloader_caching_per_type(self):
        """Pre-cached downloader for a specific type is returned directly."""
        fetcher = ContentFetcher()
        mock_dl = MagicMock()
        fetcher._downloaders[URLType.ARXIV] = mock_dl
        assert fetcher._get_downloader(URLType.ARXIV) is mock_dl
        # A different type should NOT return the cached one
        assert fetcher._get_downloader(URLType.INVALID) is None

    @pytest.mark.parametrize(
        "url_type,module_path,class_name",
        [
            (
                URLType.ARXIV,
                "local_deep_research.research_library.downloaders.arxiv",
                "ArxivDownloader",
            ),
            (
                URLType.PUBMED,
                "local_deep_research.research_library.downloaders.pubmed",
                "PubMedDownloader",
            ),
            (
                URLType.PMC,
                "local_deep_research.research_library.downloaders.pubmed",
                "PubMedDownloader",
            ),
            (
                URLType.SEMANTIC_SCHOLAR,
                "local_deep_research.research_library.downloaders.semantic_scholar",
                "SemanticScholarDownloader",
            ),
            (
                URLType.BIORXIV,
                "local_deep_research.research_library.downloaders.biorxiv",
                "BioRxivDownloader",
            ),
            (
                URLType.MEDRXIV,
                "local_deep_research.research_library.downloaders.biorxiv",
                "BioRxivDownloader",
            ),
            (
                URLType.PDF,
                "local_deep_research.research_library.downloaders.direct_pdf",
                "DirectPDFDownloader",
            ),
            (
                URLType.HTML,
                "local_deep_research.research_library.downloaders.playwright_html",
                "AutoHTMLDownloader",
            ),
            (
                URLType.DOI,
                "local_deep_research.research_library.downloaders.playwright_html",
                "AutoHTMLDownloader",
            ),
        ],
    )
    def test_import_error_returns_none(
        self, url_type, module_path, class_name, monkeypatch
    ):
        """When the downloader module cannot be imported, returns None."""
        import sys

        fetcher = ContentFetcher()
        # Use monkeypatch for safe sys.modules manipulation (auto-restored after test)
        monkeypatch.setitem(sys.modules, module_path, None)
        result = fetcher._get_downloader(url_type)
        assert result is None

    def test_returns_none_for_invalid(self):
        """INVALID type has no branch, returns None."""
        fetcher = ContentFetcher()
        assert fetcher._get_downloader(URLType.INVALID) is None


# ---------------------------------------------------------------------------
# fetch
# ---------------------------------------------------------------------------


class TestFetch:
    """Tests for ContentFetcher.fetch()."""

    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def _fetch_with_mocks(
        self,
        url,
        url_type,
        downloader,
        classifier_mock,
        validate_mock,
        **kwargs,
    ):
        """Helper: run fetch() with classifier and validate_url mocked."""
        classifier_mock.classify.return_value = url_type
        classifier_mock.get_source_name.return_value = "Test Source"
        fetcher = ContentFetcher()
        if downloader is not None:
            fetcher._downloaders[url_type] = downloader
        # Also patch _get_downloader to return what's cached or None
        if downloader is None:
            with patch.object(fetcher, "_get_downloader", return_value=None):
                # Also block the HTML fallback import
                with patch(
                    "local_deep_research.research_library.downloaders.html.HTMLDownloader",
                    create=True,
                    side_effect=ImportError("no html"),
                ):
                    return fetcher.fetch(url, **kwargs)
        return fetcher.fetch(url, **kwargs)

    # --- Invalid URL ---
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_invalid_url_returns_error(self, classifier_mock):
        classifier_mock.classify.return_value = URLType.INVALID
        classifier_mock.get_source_name.return_value = "Invalid URL"
        fetcher = ContentFetcher()
        result = fetcher.fetch("javascript:alert(1)")
        assert result["status"] == "error"
        assert "Invalid" in result["error"]

    # --- SSRF blocked ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=False,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_ssrf_blocked(self, classifier_mock, validate_mock):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"
        fetcher = ContentFetcher()
        result = fetcher.fetch("http://169.254.169.254/metadata")
        assert result["status"] == "error"
        assert "SSRF" in result["error"]

    # --- Successful UTF-8 fetch ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_success_utf8(self, classifier_mock, validate_mock):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        mock_dl = MagicMock()
        mock_dl.download_with_result.return_value = _make_download_result(
            b"Hello, world!"
        )
        # Remove get_metadata so the hasattr check is False
        del mock_dl.get_metadata

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl
        result = fetcher.fetch("https://example.com")
        assert result["status"] == "success"
        assert result["content"] == "Hello, world!"

    # --- PDF magic bytes -> text extraction success ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_pdf_content(self, classifier_mock, validate_mock):
        classifier_mock.classify.return_value = URLType.PDF
        classifier_mock.get_source_name.return_value = "PDF"

        pdf_bytes = b"%PDF-1.4 fake pdf content"
        mock_dl = MagicMock()
        mock_dl.download_with_result.return_value = _make_download_result(
            pdf_bytes
        )
        del mock_dl.get_metadata

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.PDF] = mock_dl

        with patch(
            "local_deep_research.content_fetcher.fetcher.BaseDownloader",
            create=True,
        ) as base_mock:
            base_mock.extract_text_from_pdf.return_value = "Extracted PDF text"
            # Need to patch the actual import inside fetch
            with patch(
                "local_deep_research.research_library.downloaders.base.BaseDownloader"
            ) as real_base:
                real_base.extract_text_from_pdf.return_value = (
                    "Extracted PDF text"
                )
                result = fetcher.fetch("https://example.com/paper.pdf")

        assert result["status"] == "success"
        assert result["content"] == "Extracted PDF text"

    # --- PDF text extraction failure ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_pdf_extraction_failure(self, classifier_mock, validate_mock):
        classifier_mock.classify.return_value = URLType.PDF
        classifier_mock.get_source_name.return_value = "PDF"

        pdf_bytes = b"%PDF-1.4 corrupt"
        mock_dl = MagicMock()
        mock_dl.download_with_result.return_value = _make_download_result(
            pdf_bytes
        )

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.PDF] = mock_dl

        with patch(
            "local_deep_research.research_library.downloaders.base.BaseDownloader"
        ) as real_base:
            real_base.extract_text_from_pdf.return_value = None
            result = fetcher.fetch("https://example.com/paper.pdf")

        assert result["status"] == "error"
        assert "Could not extract text from PDF" in result["error"]

    # --- UnicodeDecodeError ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_unicode_decode_error(self, classifier_mock, validate_mock):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        # Content that is NOT pdf and NOT valid UTF-8
        bad_bytes = b"\x80\x81\x82\x83"
        mock_dl = MagicMock()
        mock_dl.download_with_result.return_value = _make_download_result(
            bad_bytes
        )

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl
        result = fetcher.fetch("https://example.com")
        assert result["status"] == "error"
        assert "not valid UTF-8" in result["error"]

    # --- Content truncation ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_content_truncation(self, classifier_mock, validate_mock):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        long_content = b"A" * 200
        mock_dl = MagicMock()
        mock_dl.download_with_result.return_value = _make_download_result(
            long_content
        )
        del mock_dl.get_metadata

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl
        result = fetcher.fetch("https://example.com", max_length=50)
        assert result["status"] == "success"
        assert len(result["content"]) < 200
        assert "[... content truncated ...]" in result["content"]
        # First 50 chars should be preserved
        assert result["content"].startswith("A" * 50)

    # --- Metadata extraction ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_with_metadata(self, classifier_mock, validate_mock):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        mock_dl = MagicMock()
        mock_dl.download_with_result.return_value = _make_download_result(
            b"Content"
        )
        mock_dl.get_metadata.return_value = {
            "title": "Test Title",
            "author": "Author Name",
            "published_date": "2025-01-01",
        }

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl
        result = fetcher.fetch("https://example.com")
        assert result["status"] == "success"
        assert result["title"] == "Test Title"
        assert result["author"] == "Author Name"
        assert result["published_date"] == "2025-01-01"

    # --- Metadata extraction raises exception (should be silenced) ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_metadata_exception_silenced(
        self, classifier_mock, validate_mock
    ):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        mock_dl = MagicMock()
        mock_dl.download_with_result.return_value = _make_download_result(
            b"Content"
        )
        mock_dl.get_metadata.side_effect = RuntimeError("metadata boom")

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl
        result = fetcher.fetch("https://example.com")
        assert result["status"] == "success"
        # title should be None because metadata retrieval failed
        assert result["title"] is None

    # --- Download failed (is_success=False) ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_failed_download(self, classifier_mock, validate_mock):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        mock_dl = MagicMock()
        mock_dl.download_with_result.return_value = _make_download_result(
            b"", is_success=False, skip_reason="403 Forbidden"
        )

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl
        result = fetcher.fetch("https://example.com")
        assert result["status"] == "error"
        assert result["error"] == "403 Forbidden"

    # --- Download failed with no skip_reason ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_failed_download_no_reason(
        self, classifier_mock, validate_mock
    ):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        mock_dl = MagicMock()
        mock_dl.download_with_result.return_value = _make_download_result(
            b"", is_success=False, skip_reason=None
        )

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl
        result = fetcher.fetch("https://example.com")
        assert result["status"] == "error"
        assert result["error"] == "Download failed"

    # --- Downloader throws exception ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_downloader_exception(self, classifier_mock, validate_mock):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        mock_dl = MagicMock()
        mock_dl.download_with_result.side_effect = ConnectionError("timeout")

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl
        result = fetcher.fetch("https://example.com")
        assert result["status"] == "error"
        assert "timeout" in result["error"]

    # --- No downloader and HTML fallback also fails (ImportError) ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_no_downloader_no_html_fallback(
        self, classifier_mock, validate_mock, monkeypatch
    ):
        import sys

        classifier_mock.classify.return_value = URLType.SEMANTIC_SCHOLAR
        classifier_mock.get_source_name.return_value = "Semantic Scholar"

        fetcher = ContentFetcher()
        # _get_downloader returns None, AND the inline HTML fallback import fails
        # Block the playwright_html module so the fallback import raises ImportError
        monkeypatch.setitem(
            sys.modules,
            "local_deep_research.research_library.downloaders.playwright_html",
            None,
        )
        with patch.object(fetcher, "_get_downloader", return_value=None):
            result = fetcher.fetch("https://www.semanticscholar.org/paper/xyz")

        assert result["status"] == "error"
        assert "No suitable downloader" in result["error"]

    # --- No downloader but HTML fallback works ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_html_fallback_success(self, classifier_mock, validate_mock):
        """When specialized downloader import fails, fall back to HTML via _get_downloader."""
        classifier_mock.classify.return_value = URLType.SEMANTIC_SCHOLAR
        classifier_mock.get_source_name.return_value = "Semantic Scholar"

        mock_html_dl = MagicMock()
        mock_html_dl.download_with_result.return_value = _make_download_result(
            b"Fallback content"
        )
        del mock_html_dl.get_metadata

        fetcher = ContentFetcher()

        # First call (SEMANTIC_SCHOLAR) returns None (import failed),
        # second call (HTML fallback) returns the HTML downloader.
        def side_effect(url_type):
            if url_type == URLType.HTML:
                return mock_html_dl
            return None

        with patch.object(fetcher, "_get_downloader", side_effect=side_effect):
            result = fetcher.fetch("https://www.semanticscholar.org/paper/xyz")

        assert result["status"] == "success"
        assert result["content"] == "Fallback content"

    # --- Default max_length is applied ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_default_max_length_applied(self, classifier_mock, validate_mock):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        # Content larger than DEFAULT_MAX_CONTENT_LENGTH
        huge_content = b"X" * (DEFAULT_MAX_CONTENT_LENGTH + 100)
        mock_dl = MagicMock()
        mock_dl.download_with_result.return_value = _make_download_result(
            huge_content
        )
        del mock_dl.get_metadata

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl
        # Do NOT pass max_length -> should use DEFAULT_MAX_CONTENT_LENGTH
        result = fetcher.fetch("https://example.com")
        assert result["status"] == "success"
        assert "[... content truncated ...]" in result["content"]
        # The content before truncation suffix should be DEFAULT_MAX_CONTENT_LENGTH chars
        assert result["content"].startswith("X" * DEFAULT_MAX_CONTENT_LENGTH)

    # --- prefer_text=False sets PDF content type ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_prefer_text_false(self, classifier_mock, validate_mock):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        mock_dl = MagicMock()
        mock_dl.download_with_result.return_value = _make_download_result(
            b"data"
        )
        del mock_dl.get_metadata

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl
        fetcher.fetch("https://example.com", prefer_text=False)

        # Verify download_with_result was called with ContentType.PDF
        from local_deep_research.research_library.downloaders.base import (
            ContentType,
        )

        call_args = mock_dl.download_with_result.call_args
        assert call_args[0][1] == ContentType.PDF

    # --- is_success True but content is None/empty ---
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_success_but_empty_content(
        self, classifier_mock, validate_mock
    ):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        mock_dl = MagicMock()
        result_obj = MagicMock()
        result_obj.is_success = True
        result_obj.content = None
        result_obj.skip_reason = None
        mock_dl.download_with_result.return_value = result_obj

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl
        result = fetcher.fetch("https://example.com")
        # Falls into the else branch because content is falsy
        assert result["status"] == "error"
        assert result["error"] == "Download failed"


# ---------------------------------------------------------------------------
# fetch_text
# ---------------------------------------------------------------------------


class TestFetchText:
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_text_success(self, classifier_mock, validate_mock):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        mock_dl = MagicMock()
        mock_dl.download_with_result.return_value = _make_download_result(
            b"Text content"
        )
        del mock_dl.get_metadata

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl
        text = fetcher.fetch_text("https://example.com")
        assert text == "Text content"

    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_text_failure_returns_none(
        self, classifier_mock, validate_mock
    ):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        mock_dl = MagicMock()
        mock_dl.download_with_result.return_value = _make_download_result(
            b"", is_success=False, skip_reason="error"
        )

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl
        text = fetcher.fetch_text("https://example.com")
        assert text is None

    def test_fetch_text_invalid_url_returns_none(self):
        fetcher = ContentFetcher()
        text = fetcher.fetch_text("javascript:void(0)")
        assert text is None

    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_fetch_text_with_max_length(self, classifier_mock, validate_mock):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        mock_dl = MagicMock()
        mock_dl.download_with_result.return_value = _make_download_result(
            b"A" * 200
        )
        del mock_dl.get_metadata

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl
        text = fetcher.fetch_text("https://example.com", max_length=50)
        assert text is not None
        assert "[... content truncated ...]" in text


# ---------------------------------------------------------------------------
# __init__
# ---------------------------------------------------------------------------


class TestInit:
    def test_default_timeout(self):
        fetcher = ContentFetcher()
        assert fetcher.timeout == 30

    def test_custom_timeout(self):
        fetcher = ContentFetcher(timeout=60)
        assert fetcher.timeout == 60

    def test_downloaders_initially_empty(self):
        fetcher = ContentFetcher()
        assert fetcher._downloaders == {}


# ---------------------------------------------------------------------------
# DEFAULT_MAX_CONTENT_LENGTH constant
# ---------------------------------------------------------------------------


class TestConstants:
    def test_default_max_content_length_value(self):
        assert DEFAULT_MAX_CONTENT_LENGTH == 500_000


# ---------------------------------------------------------------------------
# language parameter
# ---------------------------------------------------------------------------


class TestLanguageParam:
    def test_default_language(self):
        fetcher = ContentFetcher()
        assert fetcher.language == "English"

    def test_custom_language(self):
        fetcher = ContentFetcher(language="German")
        assert fetcher.language == "German"

    def test_language_forwarded_to_html_downloader(self):
        """_get_downloader(HTML) passes language to AutoHTMLDownloader."""
        fetcher = ContentFetcher(language="French")

        with patch(
            "local_deep_research.research_library.downloaders."
            "playwright_html.AutoHTMLDownloader"
        ) as dl_cls:
            dl_cls.return_value = MagicMock()
            downloader = fetcher._get_downloader(URLType.HTML)
            dl_cls.assert_called_once_with(
                timeout=30,
                language="French",
                enable_js_rendering=False,
            )
            assert downloader is not None


# ---------------------------------------------------------------------------
# HTML fallback for specialized downloader failures
# ---------------------------------------------------------------------------


class TestHtmlFallback:
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_specialized_fails_html_fallback_succeeds(
        self, classifier_mock, validate_mock
    ):
        """When ArXiv downloader fails, HTML fallback returns content."""
        classifier_mock.classify.return_value = URLType.ARXIV
        classifier_mock.get_source_name.return_value = "arXiv"

        # Specialized downloader fails
        mock_arxiv = MagicMock()
        mock_arxiv.download_with_result.return_value = _make_download_result(
            content=None, is_success=False, skip_reason="PDF unavailable"
        )

        # HTML fallback succeeds
        mock_html = MagicMock()
        mock_html.download_with_result.return_value = _make_download_result(
            content=b"arXiv abstract content"
        )
        del mock_html.get_metadata

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.ARXIV] = mock_arxiv
        fetcher._downloaders[URLType.HTML] = mock_html

        result = fetcher.fetch("https://arxiv.org/abs/2301.12345")
        assert result["status"] == "success"
        assert "arXiv abstract content" in result["content"]

    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_specialized_fails_html_fallback_also_fails(
        self, classifier_mock, validate_mock
    ):
        """When both specialized and HTML downloaders fail, return error."""
        classifier_mock.classify.return_value = URLType.PUBMED
        classifier_mock.get_source_name.return_value = "PubMed"

        mock_pubmed = MagicMock()
        mock_pubmed.download_with_result.return_value = _make_download_result(
            content=None, is_success=False, skip_reason="Paywalled"
        )

        mock_html = MagicMock()
        mock_html.download_with_result.return_value = _make_download_result(
            content=None, is_success=False, skip_reason="No content"
        )

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.PUBMED] = mock_pubmed
        fetcher._downloaders[URLType.HTML] = mock_html

        result = fetcher.fetch("https://pubmed.ncbi.nlm.nih.gov/12345/")
        assert result["status"] == "error"

    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_pdf_type_no_html_fallback(self, classifier_mock, validate_mock):
        """URLType.PDF failure does NOT trigger HTML fallback."""
        classifier_mock.classify.return_value = URLType.PDF
        classifier_mock.get_source_name.return_value = "Direct PDF"

        mock_pdf = MagicMock()
        mock_pdf.download_with_result.return_value = _make_download_result(
            content=None, is_success=False, skip_reason="404"
        )

        mock_html = MagicMock()

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.PDF] = mock_pdf
        fetcher._downloaders[URLType.HTML] = mock_html

        result = fetcher.fetch("https://example.com/paper.pdf")
        assert result["status"] == "error"
        # HTML downloader should NOT have been called
        mock_html.download_with_result.assert_not_called()


# ---------------------------------------------------------------------------
# fetch_batch
# ---------------------------------------------------------------------------


class TestFetchBatch:
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_returns_dict_with_correct_keys(
        self, classifier_mock, validate_mock
    ):
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        mock_dl = MagicMock()
        mock_dl.download_with_result.return_value = _make_download_result(
            b"Page content"
        )
        del mock_dl.get_metadata

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl

        urls = ["https://a.com", "https://b.com"]
        result = fetcher.fetch_batch(urls)

        assert isinstance(result, dict)
        assert set(result.keys()) == set(urls)
        assert all(v is not None for v in result.values())

    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    @patch("local_deep_research.content_fetcher.fetcher.URLClassifier")
    def test_mixed_results(self, classifier_mock, validate_mock):
        """One URL succeeds, one fails — both appear in output."""
        classifier_mock.classify.return_value = URLType.HTML
        classifier_mock.get_source_name.return_value = "Web Page"

        success_result = _make_download_result(b"OK content")
        fail_result = _make_download_result(
            content=None, is_success=False, skip_reason="Error"
        )

        mock_dl = MagicMock()
        mock_dl.download_with_result.side_effect = [
            success_result,
            fail_result,
        ]
        del mock_dl.get_metadata

        fetcher = ContentFetcher()
        fetcher._downloaders[URLType.HTML] = mock_dl

        result = fetcher.fetch_batch(["https://ok.com", "https://fail.com"])
        assert result["https://ok.com"] is not None
        assert result["https://fail.com"] is None