learningcircuit--local-deep-research/tests/search_engines/test_search_engine_wayback.py

"""
Comprehensive tests for the Wayback Machine search engine.
Tests initialization, URL extraction, snapshot retrieval, and content fetching.

Note: These tests mock HTTP requests to avoid requiring API connections.
"""

import pytest
from unittest.mock import Mock


class TestWaybackSearchEngineInit:
    """Tests for Wayback Machine search engine initialization."""

    def test_init_default_values(self):
        """Test initialization with default values."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        engine = WaybackSearchEngine()

        assert engine.max_results == 10
        assert engine.max_snapshots_per_url == 3
        assert engine.language == "English"
        assert engine.closest_only is False
        assert engine.is_public is True

    def test_init_with_custom_max_results(self):
        """Test initialization with custom max results."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        engine = WaybackSearchEngine(max_results=25)
        assert engine.max_results == 25

    def test_init_with_custom_max_snapshots(self):
        """Test initialization with custom max snapshots per URL."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        engine = WaybackSearchEngine(max_snapshots_per_url=10)
        assert engine.max_snapshots_per_url == 10

    def test_init_with_closest_only(self):
        """Test initialization with closest only mode."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        engine = WaybackSearchEngine(closest_only=True)
        assert engine.closest_only is True

    def test_init_with_language(self):
        """Test initialization with different language."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        engine = WaybackSearchEngine(language="German")
        assert engine.language == "German"

    def test_api_endpoints_set(self):
        """Test that API endpoints are correctly set."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        engine = WaybackSearchEngine()
        assert engine.available_api == "https://archive.org/wayback/available"
        assert engine.cdx_api == "https://web.archive.org/cdx/search/cdx"


class TestWaybackEngineType:
    """Tests for Wayback engine type identification."""

    def test_engine_type_set(self):
        """Test that engine type is properly set."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        engine = WaybackSearchEngine()
        assert "wayback" in engine.engine_type.lower()

    def test_engine_is_public(self):
        """Test that engine is marked as public."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        engine = WaybackSearchEngine()
        assert engine.is_public is True

    def test_engine_is_not_generic(self):
        """Test that engine is not marked as generic."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        engine = WaybackSearchEngine()
        assert (
            not hasattr(engine, "is_generic") or engine.is_generic is not True
        )


class TestWaybackURLExtraction:
    """Tests for Wayback URL extraction from queries."""

    @pytest.fixture
    def engine(self):
        """Create a Wayback engine."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        return WaybackSearchEngine()

    def test_extract_url_from_http_url(self, engine):
        """Test extraction of full HTTP URL (domain portion)."""
        urls = engine._extract_urls_from_query("https://example.com/page")
        # The regex extracts the domain portion of the URL
        assert len(urls) >= 1
        assert any("example.com" in url for url in urls)

    def test_extract_url_from_multiple_urls(self, engine):
        """Test extraction of multiple URLs."""
        query = "Check https://example.com and http://test.org"
        urls = engine._extract_urls_from_query(query)
        assert len(urls) == 2
        assert "https://example.com" in urls
        assert "http://test.org" in urls

    def test_extract_url_from_domain_only(self, engine):
        """Test extraction from domain without protocol."""
        urls = engine._extract_urls_from_query("example.com")
        assert "http://example.com" in urls

    def test_extract_url_from_partial_path(self, engine):
        """Test extraction from partial URL path."""
        urls = engine._extract_urls_from_query("example.com/page/test")
        assert "http://example.com/page/test" in urls


class TestWaybackTimestampFormatting:
    """Tests for Wayback timestamp formatting."""

    @pytest.fixture
    def engine(self):
        """Create a Wayback engine."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        return WaybackSearchEngine()

    def test_format_timestamp_full(self, engine):
        """Test formatting a full timestamp."""
        result = engine._format_timestamp("20240115103045")
        assert result == "2024-01-15 10:30:45"

    def test_format_timestamp_short(self, engine):
        """Test formatting a short timestamp returns as-is."""
        result = engine._format_timestamp("2024")
        assert result == "2024"

    def test_format_timestamp_invalid(self, engine):
        """Test formatting an invalid timestamp returns as-is."""
        result = engine._format_timestamp("invalid")
        assert result == "invalid"


class TestWaybackSnapshotRetrieval:
    """Tests for Wayback snapshot retrieval."""

    @pytest.fixture
    def engine(self):
        """Create a Wayback engine."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        return WaybackSearchEngine()

    def test_get_snapshots_closest_only(self, engine, monkeypatch):
        """Test snapshot retrieval in closest only mode."""
        engine.closest_only = True

        mock_response = Mock()
        mock_response.json = Mock(
            return_value={
                "archived_snapshots": {
                    "closest": {
                        "url": "https://web.archive.org/web/20240115/https://example.com",
                        "timestamp": "20240115000000",
                        "available": True,
                        "status": "200",
                    }
                }
            }
        )

        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(return_value=mock_response),
        )

        snapshots = engine._get_wayback_snapshots("https://example.com")

        assert len(snapshots) == 1
        assert "20240115" in snapshots[0]["timestamp"]

    def test_get_snapshots_cdx_api(self, engine, monkeypatch):
        """Test snapshot retrieval using CDX API."""
        engine.closest_only = False

        mock_response = Mock()
        mock_response.json = Mock(
            return_value=[
                ["timestamp", "original", "statuscode", "mimetype"],
                ["20240115000000", "https://example.com", "200", "text/html"],
                ["20230615000000", "https://example.com", "200", "text/html"],
            ]
        )

        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(return_value=mock_response),
        )

        snapshots = engine._get_wayback_snapshots("https://example.com")

        assert len(snapshots) == 2
        assert snapshots[0]["timestamp"] == "20240115000000"
        assert snapshots[1]["timestamp"] == "20230615000000"

    def test_get_snapshots_no_results(self, engine, monkeypatch):
        """Test snapshot retrieval with no archived versions."""
        engine.closest_only = True

        mock_response = Mock()
        mock_response.json = Mock(return_value={"archived_snapshots": {}})

        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(return_value=mock_response),
        )

        snapshots = engine._get_wayback_snapshots("https://example.com")

        assert snapshots == []

    def test_get_snapshots_handles_exception(self, engine, monkeypatch):
        """Test that exceptions are handled gracefully."""
        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(side_effect=Exception("Network error")),
        )

        snapshots = engine._get_wayback_snapshots("https://example.com")
        assert snapshots == []

    def test_get_snapshots_rate_limit_error(self, engine, monkeypatch):
        """Test that 429 errors raise RateLimitError."""
        from local_deep_research.web_search_engines.rate_limiting import (
            RateLimitError,
        )

        mock_response = Mock()
        mock_response.status_code = 429

        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(return_value=mock_response),
        )

        engine.closest_only = True
        with pytest.raises(RateLimitError):
            engine._get_wayback_snapshots("https://example.com")

    def test_get_snapshots_cdx_rate_limit_error(self, engine, monkeypatch):
        """Test that CDX API 429 errors raise RateLimitError."""
        from local_deep_research.web_search_engines.rate_limiting import (
            RateLimitError,
        )

        mock_response = Mock()
        mock_response.status_code = 429

        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(return_value=mock_response),
        )

        engine.closest_only = False
        with pytest.raises(RateLimitError):
            engine._get_wayback_snapshots("https://example.com")


class TestWaybackPreviewRetrieval:
    """Tests for Wayback preview retrieval."""

    @pytest.fixture
    def engine(self):
        """Create a Wayback engine."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        return WaybackSearchEngine(closest_only=True)

    def test_get_previews_with_url_query(self, engine, monkeypatch):
        """Test preview retrieval with URL query."""
        mock_response = Mock()
        mock_response.json = Mock(
            return_value={
                "archived_snapshots": {
                    "closest": {
                        "url": "https://web.archive.org/web/20240115/https://example.com",
                        "timestamp": "20240115000000",
                        "available": True,
                        "status": "200",
                    }
                }
            }
        )

        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(return_value=mock_response),
        )

        previews = engine._get_previews("https://example.com")

        assert len(previews) == 1
        assert "example.com" in previews[0]["title"]
        assert previews[0]["original_url"] == "https://example.com"

    def test_get_previews_no_urls_found(self, engine, monkeypatch):
        """Test preview retrieval when no URLs can be extracted."""
        # Ensure DuckDuckGo fallback doesn't work either
        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.WaybackSearchEngine._extract_urls_from_query",
            Mock(return_value=[]),
        )

        previews = engine._get_previews("random text without urls")

        assert previews == []


class TestWaybackContentRetrieval:
    """Tests for Wayback content retrieval."""

    @pytest.fixture
    def engine(self):
        """Create a Wayback engine."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        return WaybackSearchEngine()

    def test_get_wayback_content_success(self, engine, monkeypatch):
        """Test successful content retrieval."""
        mock_response = Mock()
        mock_response.text = "<html><body><p>Test content</p></body></html>"

        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(return_value=mock_response),
        )

        # Mock extract_content to avoid complex HTML processing
        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.extract_content",
            Mock(return_value=""),
        )

        raw_html, cleaned = engine._get_wayback_content(
            "https://web.archive.org/web/20240115/https://example.com"
        )

        assert "<html>" in raw_html

    def test_get_wayback_content_handles_error(self, engine, monkeypatch):
        """Test that content retrieval errors are handled."""
        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(side_effect=Exception("Network error")),
        )

        raw_html, cleaned = engine._get_wayback_content(
            "https://web.archive.org/web/20240115/https://example.com"
        )

        assert raw_html == ""
        assert "Error" in cleaned

    def test_remove_boilerplate_empty_html(self, engine):
        """Test boilerplate removal with empty HTML."""
        result = engine._remove_boilerplate("")
        assert result == ""

    def test_remove_boilerplate_whitespace_only(self, engine):
        """Test boilerplate removal with whitespace only."""
        result = engine._remove_boilerplate("   ")
        assert result == ""


class TestWaybackSpecialMethods:
    """Tests for Wayback special methods."""

    @pytest.fixture
    def engine(self):
        """Create a Wayback engine."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        return WaybackSearchEngine()

    def test_search_by_url_method_exists(self, engine):
        """Test that search_by_url method exists."""
        assert hasattr(engine, "search_by_url")
        assert callable(engine.search_by_url)

    def test_search_by_date_range_method_exists(self, engine):
        """Test that search_by_date_range method exists."""
        assert hasattr(engine, "search_by_date_range")
        assert callable(engine.search_by_date_range)

    def test_get_latest_snapshot_method_exists(self, engine):
        """Test that get_latest_snapshot method exists."""
        assert hasattr(engine, "get_latest_snapshot")
        assert callable(engine.get_latest_snapshot)

    def test_search_by_url_with_custom_max(self, engine, monkeypatch):
        """Test search_by_url with custom max snapshots."""
        mock_response = Mock()
        mock_response.json = Mock(
            return_value={
                "archived_snapshots": {
                    "closest": {
                        "url": "https://web.archive.org/web/20240115/https://example.com",
                        "timestamp": "20240115000000",
                        "available": True,
                        "status": "200",
                    }
                }
            }
        )

        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(return_value=mock_response),
        )

        # Mock _get_wayback_content to avoid network calls
        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.WaybackSearchEngine._get_wayback_content",
            Mock(return_value=("", "")),
        )

        engine.closest_only = True
        results = engine.search_by_url("https://example.com", max_snapshots=5)

        assert len(results) >= 0  # May be 0 or 1 depending on mock

    def test_get_latest_snapshot_success(self, engine, monkeypatch):
        """Test successful latest snapshot retrieval."""
        mock_response = Mock()
        mock_response.json = Mock(
            return_value={
                "archived_snapshots": {
                    "closest": {
                        "url": "https://web.archive.org/web/20240115/https://example.com",
                        "timestamp": "20240115000000",
                        "available": True,
                        "status": "200",
                    }
                }
            }
        )

        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(return_value=mock_response),
        )

        # Mock _get_wayback_content to avoid network calls
        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.WaybackSearchEngine._get_wayback_content",
            Mock(return_value=("", "")),
        )

        result = engine.get_latest_snapshot("https://example.com")

        assert result is not None
        assert "example.com" in result["title"]

    def test_get_latest_snapshot_not_found(self, engine, monkeypatch):
        """Test latest snapshot retrieval when none exists."""
        mock_response = Mock()
        mock_response.json = Mock(return_value={"archived_snapshots": {}})

        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(return_value=mock_response),
        )

        result = engine.get_latest_snapshot("https://newsite-no-archive.com")

        assert result is None


class TestWaybackDateRangeSearch:
    """Tests for Wayback date range search."""

    @pytest.fixture
    def engine(self):
        """Create a Wayback engine."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        return WaybackSearchEngine()

    def test_search_by_date_range_success(self, engine, monkeypatch):
        """Test successful date range search."""
        mock_response = Mock()
        mock_response.json = Mock(
            return_value=[
                ["timestamp", "original", "statuscode", "mimetype"],
                ["20240115000000", "https://example.com", "200", "text/html"],
            ]
        )

        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(return_value=mock_response),
        )

        # Mock _get_wayback_content to avoid network calls
        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.WaybackSearchEngine._get_wayback_content",
            Mock(return_value=("", "")),
        )

        results = engine.search_by_date_range(
            "https://example.com", "20240101", "20240131"
        )

        assert len(results) == 1
        assert "20240115" in results[0]["timestamp"]

    def test_search_by_date_range_no_results(self, engine, monkeypatch):
        """Test date range search with no results."""
        mock_response = Mock()
        mock_response.json = Mock(
            return_value=[["timestamp", "original", "statuscode", "mimetype"]]
        )

        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(return_value=mock_response),
        )

        results = engine.search_by_date_range(
            "https://example.com", "19900101", "19901231"
        )

        assert results == []

    def test_search_by_date_range_handles_error(self, engine, monkeypatch):
        """Test that date range search errors are handled."""
        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(side_effect=Exception("Network error")),
        )

        results = engine.search_by_date_range(
            "https://example.com", "20240101", "20240131"
        )

        assert results == []


class TestWaybackFullContent:
    """Tests for Wayback full content retrieval."""

    @pytest.fixture
    def engine(self):
        """Create a Wayback engine."""
        from local_deep_research.web_search_engines.engines.search_engine_wayback import (
            WaybackSearchEngine,
        )

        return WaybackSearchEngine()

    def test_get_full_content_with_retrieval(self, engine, monkeypatch):
        """Test full content retrieval with actual fetching."""
        mock_response = Mock()
        mock_response.text = "<html><body><p>Full content</p></body></html>"

        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.safe_get",
            Mock(return_value=mock_response),
        )

        # Mock extract_content
        monkeypatch.setattr(
            "local_deep_research.web_search_engines.engines.search_engine_wayback.extract_content",
            Mock(return_value=""),
        )

        items = [
            {
                "title": "Test Archive",
                "link": "https://web.archive.org/web/20240115/https://example.com",
                "snippet": "Test snippet",
            }
        ]

        results = engine._get_full_content(items)

        assert len(results) == 1
        assert "raw_html" in results[0]