learningcircuit--local-deep-research/tests/content_fetcher/test_mcp_integration.py

"""
Integration tests for download_content tool in MCP strategy.
"""

from unittest.mock import MagicMock, patch

from local_deep_research.content_fetcher import ContentFetcher
from local_deep_research.research_library.downloaders.base import DownloadResult


class TestMCPDownloadContentTool:
    """Test download_content tool integration with MCP strategy."""

    @patch(
        "local_deep_research.content_fetcher.fetcher.ContentFetcher._get_downloader"
    )
    def test_execute_download_content_success(self, mock_get_downloader):
        """Test successful content download via MCP strategy executor."""
        # Setup mock
        mock_downloader = MagicMock()
        mock_downloader.download_with_result.return_value = DownloadResult(
            content=b"# Research Paper\n\nThis is the abstract of the paper discussing important findings.",
            is_success=True,
        )
        mock_downloader.get_metadata.return_value = {
            "title": "Important Research Paper",
            "author": "Dr. Smith",
        }
        mock_get_downloader.return_value = mock_downloader

        # Import and test the executor pattern
        from local_deep_research.content_fetcher import ContentFetcher

        fetcher = ContentFetcher()
        result = fetcher.fetch(
            "https://arxiv.org/abs/2301.12345",
            max_length=5000,
        )

        assert result["status"] == "success"
        assert "abstract" in result["content"].lower()
        assert result["source_type"] == "arXiv"

    @patch(
        "local_deep_research.content_fetcher.fetcher.ContentFetcher._get_downloader"
    )
    def test_execute_download_content_with_truncation(
        self, mock_get_downloader
    ):
        """Test content truncation for very long papers."""
        # Create a very long paper content
        long_content = "Introduction. " * 5000  # ~70k characters

        mock_downloader = MagicMock()
        mock_downloader.download_with_result.return_value = DownloadResult(
            content=long_content.encode("utf-8"),
            is_success=True,
        )
        mock_downloader.get_metadata.return_value = {}
        mock_get_downloader.return_value = mock_downloader

        fetcher = ContentFetcher()
        result = fetcher.fetch(
            "https://arxiv.org/abs/2301.12345",
            max_length=1000,
        )

        assert result["status"] == "success"
        assert len(result["content"]) < 2000  # Should be truncated
        assert "truncated" in result["content"].lower()

    @patch(
        "local_deep_research.content_fetcher.fetcher.ContentFetcher._get_downloader"
    )
    def test_execute_download_content_paper_not_found(
        self, mock_get_downloader
    ):
        """Test handling when paper is not available."""
        mock_downloader = MagicMock()
        mock_downloader.download_with_result.return_value = DownloadResult(
            content=None,
            is_success=False,
            skip_reason="Paper not found or requires subscription",
        )
        mock_get_downloader.return_value = mock_downloader

        fetcher = ContentFetcher()
        result = fetcher.fetch("https://example.com/paywalled-paper")

        assert result["status"] == "error"
        assert (
            "subscription" in result["error"].lower()
            or "not found" in result["error"].lower()
        )

    @patch(
        "local_deep_research.content_fetcher.fetcher.ContentFetcher._get_downloader"
    )
    @patch(
        "local_deep_research.content_fetcher.fetcher.policy_aware_validate_url",
        return_value=True,
    )
    def test_execute_download_content_timeout(
        self, mock_validate, mock_get_downloader
    ):
        """Test handling of timeout during download."""
        mock_downloader = MagicMock()
        mock_downloader.download_with_result.side_effect = TimeoutError(
            "Connection timed out"
        )
        mock_get_downloader.return_value = mock_downloader

        fetcher = ContentFetcher()
        result = fetcher.fetch("https://slow-server.example.com/paper")

        assert result["status"] == "error"
        assert (
            "timed out" in result["error"].lower()
            or "timeout" in result["error"].lower()
        )


class TestDownloadContentWithRealURLPatterns:
    """Test with real-world URL patterns."""

    def test_arxiv_url_variants(self):
        """Test various arXiv URL formats are recognized."""
        fetcher = ContentFetcher()

        # Standard abstract
        info = fetcher.get_url_info("https://arxiv.org/abs/2301.12345")
        assert info["url_type"] == "arxiv"
        assert info["extracted_id"] == "2301.12345"

        # With version
        info = fetcher.get_url_info("https://arxiv.org/abs/2301.12345v3")
        assert info["url_type"] == "arxiv"
        assert info["extracted_id"] == "2301.12345v3"

        # PDF link
        info = fetcher.get_url_info("https://arxiv.org/pdf/2301.12345.pdf")
        assert info["url_type"] == "arxiv"

        # HTML version
        info = fetcher.get_url_info("https://arxiv.org/html/2301.12345")
        assert info["url_type"] == "arxiv"

        # ar5iv (HTML rendering)
        info = fetcher.get_url_info("https://ar5iv.org/abs/2301.12345")
        assert info["url_type"] == "arxiv"

        # Old format (pre-2007)
        info = fetcher.get_url_info("https://arxiv.org/abs/hep-th/9901001")
        assert info["url_type"] == "arxiv"

    def test_pubmed_url_variants(self):
        """Test various PubMed URL formats."""
        fetcher = ContentFetcher()

        # Standard PubMed
        info = fetcher.get_url_info("https://pubmed.ncbi.nlm.nih.gov/12345678")
        assert info["url_type"] == "pubmed"
        assert info["extracted_id"] == "12345678"

        # With trailing slash
        info = fetcher.get_url_info("https://pubmed.ncbi.nlm.nih.gov/12345678/")
        assert info["url_type"] == "pubmed"

        # Old format
        info = fetcher.get_url_info(
            "https://www.ncbi.nlm.nih.gov/pubmed/12345678"
        )
        assert info["url_type"] == "pubmed"

    def test_pmc_url_variants(self):
        """Test various PMC URL formats."""
        fetcher = ContentFetcher()

        # Standard PMC
        info = fetcher.get_url_info(
            "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC1234567/"
        )
        assert info["url_type"] == "pmc"
        assert info["extracted_id"] == "PMC1234567"

        # Europe PMC
        info = fetcher.get_url_info("https://europepmc.org/article/PMC/1234567")
        assert info["url_type"] == "pmc"

    def test_semantic_scholar_url_variants(self):
        """Test Semantic Scholar URL formats."""
        fetcher = ContentFetcher()

        # With paper title in URL
        info = fetcher.get_url_info(
            "https://www.semanticscholar.org/paper/Attention-Is-All-You-Need-Vaswani-Shazeer/abcdef1234567890abcdef1234567890abcdef12"
        )
        assert info["url_type"] == "semantic_scholar"

        # API URL
        info = fetcher.get_url_info(
            "https://api.semanticscholar.org/v1/paper/abcd1234"
        )
        assert info["url_type"] == "semantic_scholar"

    def test_biorxiv_medrxiv_url_variants(self):
        """Test bioRxiv and medRxiv URL formats."""
        fetcher = ContentFetcher()

        # bioRxiv
        info = fetcher.get_url_info(
            "https://www.biorxiv.org/content/10.1101/2021.01.01.123456v1"
        )
        assert info["url_type"] == "biorxiv"

        # medRxiv
        info = fetcher.get_url_info(
            "https://www.medrxiv.org/content/10.1101/2021.01.01.123456v1"
        )
        assert info["url_type"] == "medrxiv"

    def test_doi_url_variants(self):
        """Test DOI URL formats."""
        fetcher = ContentFetcher()

        # doi.org
        info = fetcher.get_url_info("https://doi.org/10.1038/nature12373")
        assert info["url_type"] == "doi"
        assert "10.1038" in info["extracted_id"]

        # dx.doi.org
        info = fetcher.get_url_info(
            "https://dx.doi.org/10.1126/science.1234567"
        )
        assert info["url_type"] == "doi"


class TestDownloadContentEdgeCases:
    """Test edge cases and error handling."""

    @patch(
        "local_deep_research.content_fetcher.fetcher.ContentFetcher._get_downloader"
    )
    def test_empty_url(self, mock_get_downloader):
        """Test handling of empty URL."""
        fetcher = ContentFetcher()
        # Empty URL should still try to classify and fetch
        info = fetcher.get_url_info("")
        assert info["url_type"] == "html"  # Falls back to HTML

    @patch(
        "local_deep_research.content_fetcher.fetcher.ContentFetcher._get_downloader"
    )
    def test_malformed_url(self, mock_get_downloader):
        """Test handling of malformed URL."""
        mock_downloader = MagicMock()
        mock_downloader.download_with_result.side_effect = ValueError(
            "Invalid URL"
        )
        mock_get_downloader.return_value = mock_downloader

        fetcher = ContentFetcher()
        result = fetcher.fetch("not-a-valid-url")

        assert result["status"] == "error"

    @patch(
        "local_deep_research.content_fetcher.fetcher.ContentFetcher._get_downloader"
    )
    def test_binary_pdf_content(self, mock_get_downloader):
        """Test handling of binary PDF content that can't be decoded as UTF-8."""
        # Simulate PDF bytes that fail UTF-8 decoding
        pdf_bytes = b"%PDF-1.4\x00\x01\x02\x03binary content"

        mock_downloader = MagicMock()
        mock_downloader.download_with_result.return_value = DownloadResult(
            content=pdf_bytes,
            is_success=True,
        )
        mock_downloader.get_metadata.return_value = {}
        mock_get_downloader.return_value = mock_downloader

        # Mock the PDF text extraction to return extracted text
        with patch(
            "local_deep_research.research_library.downloaders.base.BaseDownloader.extract_text_from_pdf"
        ) as mock_extract:
            mock_extract.return_value = "Extracted PDF text content"

            fetcher = ContentFetcher()
            result = fetcher.fetch("https://example.com/paper.pdf")

            # Should attempt PDF extraction
            assert result["status"] == "success" or result["status"] == "error"

    @patch(
        "local_deep_research.content_fetcher.fetcher.ContentFetcher._get_downloader"
    )
    def test_rate_limited_response(self, mock_get_downloader):
        """Test handling of rate-limited responses."""
        mock_downloader = MagicMock()
        mock_downloader.download_with_result.return_value = DownloadResult(
            content=None,
            is_success=False,
            skip_reason="Rate limited - too many requests (429)",
        )
        mock_get_downloader.return_value = mock_downloader

        fetcher = ContentFetcher()
        result = fetcher.fetch("https://arxiv.org/abs/2301.12345")

        assert result["status"] == "error"
        assert "rate" in result["error"].lower() or "429" in result["error"]

    @patch(
        "local_deep_research.content_fetcher.fetcher.ContentFetcher._get_downloader"
    )
    def test_subscription_required(self, mock_get_downloader):
        """Test handling of paywalled content."""
        mock_downloader = MagicMock()
        mock_downloader.download_with_result.return_value = DownloadResult(
            content=None,
            is_success=False,
            skip_reason="Subscription required to access full text",
        )
        mock_get_downloader.return_value = mock_downloader

        fetcher = ContentFetcher()
        result = fetcher.fetch("https://nature.com/articles/paywalled")

        assert result["status"] == "error"
        assert "subscription" in result["error"].lower()


class TestDownloadContentCaching:
    """Test downloader caching behavior."""

    def test_downloader_reuse(self):
        """Test that downloaders are cached and reused."""
        fetcher = ContentFetcher()

        # Get HTML downloader twice
        d1 = fetcher._get_downloader(
            __import__(
                "local_deep_research.content_fetcher.url_classifier",
                fromlist=["URLType"],
            ).URLType.HTML
        )
        d2 = fetcher._get_downloader(
            __import__(
                "local_deep_research.content_fetcher.url_classifier",
                fromlist=["URLType"],
            ).URLType.HTML
        )

        # Should be the same instance
        assert d1 is d2

    def test_different_downloaders_for_different_types(self):
        """Test that different URL types get different downloaders."""
        from local_deep_research.content_fetcher.url_classifier import URLType

        fetcher = ContentFetcher()

        html_downloader = fetcher._get_downloader(URLType.HTML)
        # Other downloaders may not be available, but HTML should always work
        assert html_downloader is not None