learningcircuit--local-deep-research/tests/utilities/test_url_utils.py

"""
Tests for utilities/url_utils.py

Tests cover:
- URL normalization
- Scheme handling
- Private IP detection
"""

import pytest
from unittest.mock import patch


class TestNormalizeUrl:
    """Tests for normalize_url function."""

    def test_normalize_url_with_http_scheme(self):
        """Test URL with http:// scheme is returned unchanged."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("http://example.com")
        assert result == "http://example.com"

    def test_normalize_url_with_https_scheme(self):
        """Test URL with https:// scheme is returned unchanged."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("https://example.com")
        assert result == "https://example.com"

    def test_normalize_url_with_http_scheme_and_port(self):
        """Test URL with http:// scheme and port is returned unchanged."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("http://example.com:8080")
        assert result == "http://example.com:8080"

    def test_normalize_url_empty_raises_error(self):
        """Test that empty URL raises ValueError."""
        from local_deep_research.utilities.url_utils import normalize_url

        with pytest.raises(ValueError) as exc_info:
            normalize_url("")
        assert "empty" in str(exc_info.value).lower()

    def test_normalize_url_strips_whitespace(self):
        """Test that whitespace is stripped."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("  http://example.com  ")
        assert result == "http://example.com"

    def test_normalize_url_malformed_http_colon(self):
        """Test URL with malformed http: (missing //) is fixed."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("http:example.com")
        assert result == "http://example.com"

    def test_normalize_url_malformed_https_colon(self):
        """Test URL with malformed https: (missing //) is fixed."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("https:example.com")
        assert result == "https://example.com"

    def test_normalize_url_localhost_gets_http(self):
        """Test that localhost gets http:// scheme."""
        from local_deep_research.utilities.url_utils import normalize_url

        with patch(
            "local_deep_research.utilities.url_utils.is_private_ip",
            return_value=True,
        ):
            result = normalize_url("localhost:8080")
            assert result == "http://localhost:8080"

    def test_normalize_url_external_gets_https(self):
        """Test that external hosts get https:// scheme."""
        from local_deep_research.utilities.url_utils import normalize_url

        with patch(
            "local_deep_research.utilities.url_utils.is_private_ip",
            return_value=False,
        ):
            result = normalize_url("example.com:443")
            assert result == "https://example.com:443"

    def test_normalize_url_double_slash_prefix(self):
        """Test URL starting with // has prefix removed."""
        from local_deep_research.utilities.url_utils import normalize_url

        with patch(
            "local_deep_research.utilities.url_utils.is_private_ip",
            return_value=False,
        ):
            result = normalize_url("//example.com")
            assert result == "https://example.com"

    def test_normalize_url_127_0_0_1(self):
        """Test that 127.0.0.1 gets http:// scheme."""
        from local_deep_research.utilities.url_utils import normalize_url

        with patch(
            "local_deep_research.utilities.url_utils.is_private_ip",
            return_value=True,
        ):
            result = normalize_url("127.0.0.1:11434")
            assert result == "http://127.0.0.1:11434"

    def test_normalize_url_preserves_path(self):
        """Test that URL path is preserved."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("http://example.com/api/v1/search")
        assert result == "http://example.com/api/v1/search"

    def test_normalize_url_preserves_query_string(self):
        """Test that query string is preserved."""
        from local_deep_research.utilities.url_utils import normalize_url

        result = normalize_url("http://example.com?q=test&page=1")
        assert result == "http://example.com?q=test&page=1"


class TestCanonicalUrlKey:
    """Tests for canonical_url_key — dedup key generation, NOT for display."""

    def setup_method(self):
        # lru_cache persists across tests; clear to keep each assertion
        # independent of prior test inputs.
        from local_deep_research.utilities.url_utils import canonical_url_key

        canonical_url_key.cache_clear()

    def test_empty_returns_empty(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert canonical_url_key("") == ""

    def test_lowercases_scheme_and_host_preserves_path_case(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert (
            canonical_url_key("HTTPS://EXAMPLE.COM/Foo/Bar")
            == "https://example.com/Foo/Bar"
        )

    def test_strips_fragment(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert (
            canonical_url_key("https://example.com/page#section")
            == "https://example.com/page"
        )

    def test_trailing_slash_normalized(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert canonical_url_key("https://example.com/p") == canonical_url_key(
            "https://example.com/p/"
        )
        # Root path '/' is preserved (not stripped to empty).
        assert canonical_url_key("https://example.com/").endswith("/")

    def test_strips_utm_and_common_trackers(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        url = (
            "https://example.com/p?"
            "utm_source=x&UTM_Medium=y&utm_campaign=z&"
            "fbclid=a&gclid=b&msclkid=c&yclid=d&dclid=e&gad_source=f&"
            "mc_eid=g&mc_cid=h&ref_src=i&igshid=j&_ga=k&_gl=l"
        )
        assert canonical_url_key(url) == "https://example.com/p"

    def test_keeps_non_tracking_query_params(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        # q, ref (GitHub branch), v (YouTube id), page, id are content-bearing.
        assert (
            canonical_url_key("https://github.com/o/r?ref=main&utm_source=x")
            == "https://github.com/o/r?ref=main"
        )
        assert (
            canonical_url_key(
                "https://www.youtube.com/watch?v=abc123&utm_source=x"
            )
            == "https://www.youtube.com/watch?v=abc123"
        )
        assert (
            canonical_url_key("https://example.com/p?q=hello&page=2")
            == "https://example.com/p?q=hello&page=2"
        )

    def test_strips_userinfo(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert (
            canonical_url_key("https://user:pass@example.com/p")
            == "https://example.com/p"
        )
        # Userinfo without colon.
        assert (
            canonical_url_key("https://user@example.com/p")
            == "https://example.com/p"
        )

    def test_strips_default_ports(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert (
            canonical_url_key("https://example.com:443/p")
            == "https://example.com/p"
        )
        assert (
            canonical_url_key("http://example.com:80/p")
            == "http://example.com/p"
        )

    def test_preserves_nondefault_port(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert (
            canonical_url_key("https://example.com:8443/p")
            == "https://example.com:8443/p"
        )

    def test_ipv6_host_preserved(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert (
            canonical_url_key("https://[::1]:8443/page")
            == "https://[::1]:8443/page"
        )

    def test_invalid_url_returned_stripped(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert canonical_url_key("  not a url  ") == "not a url"

    def test_mailto_returns_stripped(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert canonical_url_key("mailto:foo@bar.com") == "mailto:foo@bar.com"

    def test_protocol_relative_returns_stripped(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        # Protocol-relative URL has no scheme — canonicalization is ambiguous,
        # so we fall back to the stripped input.
        assert canonical_url_key("//example.com/path") == "//example.com/path"

    def test_combined_normalization(self):
        from local_deep_research.utilities.url_utils import canonical_url_key

        assert (
            canonical_url_key(
                "HTTPS://User:Pass@Example.COM:443/Path/?utm_source=x&q=1#frag"
            )
            == "https://example.com/Path?q=1"
        )