learningcircuit--local-deep-research/tests/utilities/test_url_utils_coverage.py

"""
Comprehensive coverage tests for url_utils.normalize_url.

Focuses on exhaustive path coverage including:
- http/https scheme passthrough
- Malformed scheme correction (http: / https: without //)
- Double-slash prefix stripping
- Hostname extraction for private vs public classification
- IPv6 bracket handling
- Edge cases: empty, whitespace, None, special characters
"""

import pytest
from unittest.mock import patch


# ---------------------------------------------------------------------------
# Helper – import once per test to keep tests independent of import order
# ---------------------------------------------------------------------------
def _normalize(url):
    from local_deep_research.utilities.url_utils import normalize_url

    return normalize_url(url)


def _is_private(hostname):
    from local_deep_research.security.network_utils import is_private_ip

    return is_private_ip(hostname)


# ===================================================================
# 1. Proper scheme passthrough (early return on line 34)
# ===================================================================
class TestSchemePassthrough:
    """URLs that already have http:// or https:// are returned verbatim."""

    @pytest.mark.parametrize(
        "url",
        [
            "http://localhost",
            "http://localhost:11434",
            "http://127.0.0.1:8080",
            "http://192.168.1.1/api/v1",
            "http://[::1]:11434",
            "http://example.com?q=1&r=2#frag",
            "http://user:pass@host:9200/path",
        ],
    )
    def test_http_urls_returned_unchanged(self, url):
        assert _normalize(url) == url

    @pytest.mark.parametrize(
        "url",
        [
            "https://example.com",
            "https://api.openai.com/v1/chat",
            "https://example.com:8443",
            "https://example.com/search?q=hello",
            "https://[2001:db8::1]:443/path",
        ],
    )
    def test_https_urls_returned_unchanged(self, url):
        assert _normalize(url) == url

    def test_http_triple_slash_passthrough(self):
        """http:/// still starts with http:// so the early return fires."""
        assert _normalize("http:///some/path") == "http:///some/path"

    def test_https_triple_slash_passthrough(self):
        assert _normalize("https:///some/path") == "https:///some/path"


# ===================================================================
# 2. Malformed scheme correction (lines 38-43)
# ===================================================================
class TestMalformedScheme:
    """URLs like http:host or https:host get slashes inserted."""

    def test_http_colon_hostname(self):
        assert _normalize("http:example.com") == "http://example.com"

    def test_https_colon_hostname(self):
        assert _normalize("https:example.com") == "https://example.com"

    def test_http_colon_hostname_with_port(self):
        assert _normalize("http:example.com:8080") == "http://example.com:8080"

    def test_https_colon_hostname_with_path(self):
        assert (
            _normalize("https:example.com/api/v1")
            == "https://example.com/api/v1"
        )

    def test_http_colon_localhost(self):
        assert _normalize("http:localhost:11434") == "http://localhost:11434"

    def test_https_colon_ip(self):
        assert _normalize("https:8.8.8.8") == "https://8.8.8.8"

    def test_http_colon_private_ip(self):
        assert _normalize("http:192.168.1.1:3000") == "http://192.168.1.1:3000"

    def test_malformed_preserves_query(self):
        assert (
            _normalize("http:example.com/p?a=1") == "http://example.com/p?a=1"
        )


# ===================================================================
# 3. Double-slash prefix (lines 46-48)
# ===================================================================
class TestDoubleSlashPrefix:
    """Protocol-relative URLs starting with // are stripped then re-schemed."""

    def test_double_slash_public_host(self):
        result = _normalize("//example.com")
        assert result == "https://example.com"

    def test_double_slash_public_with_port(self):
        result = _normalize("//example.com:443")
        assert result == "https://example.com:443"

    def test_double_slash_localhost(self):
        result = _normalize("//localhost:8080")
        assert result == "http://localhost:8080"

    def test_double_slash_private_ip(self):
        result = _normalize("//192.168.1.1:9200")
        assert result == "http://192.168.1.1:9200"

    def test_double_slash_with_path(self):
        result = _normalize("//example.com/api/v1")
        assert result == "https://example.com/api/v1"

    def test_double_slash_127(self):
        result = _normalize("//127.0.0.1:11434")
        assert result == "http://127.0.0.1:11434"


# ===================================================================
# 4. Localhost / loopback addresses
# ===================================================================
class TestLocalhost:
    """localhost and 127.0.0.1 should always get http://."""

    def test_localhost_bare(self):
        assert _normalize("localhost") == "http://localhost"

    def test_localhost_with_port(self):
        assert _normalize("localhost:11434") == "http://localhost:11434"

    def test_localhost_with_path(self):
        assert _normalize("localhost:8080/api") == "http://localhost:8080/api"

    def test_127_0_0_1_bare(self):
        assert _normalize("127.0.0.1") == "http://127.0.0.1"

    def test_127_0_0_1_with_port(self):
        assert _normalize("127.0.0.1:11434") == "http://127.0.0.1:11434"

    def test_0_0_0_0(self):
        assert _normalize("0.0.0.0") == "http://0.0.0.0"

    def test_0_0_0_0_with_port(self):
        assert _normalize("0.0.0.0:5000") == "http://0.0.0.0:5000"


# ===================================================================
# 5. RFC 1918 private IP ranges
# ===================================================================
class TestPrivateIPs:
    """Private IPs (10.x, 172.16-31.x, 192.168.x) should get http://."""

    @pytest.mark.parametrize(
        "ip",
        [
            "10.0.0.1",
            "10.255.255.255",
            "172.16.0.1",
            "172.31.255.255",
            "192.168.0.1",
            "192.168.255.254",
        ],
    )
    def test_private_ip_gets_http(self, ip):
        result = _normalize(ip)
        assert result == f"http://{ip}"

    @pytest.mark.parametrize(
        "ip",
        [
            "10.0.0.1:8080",
            "172.16.0.50:3000",
            "192.168.1.100:8888",
        ],
    )
    def test_private_ip_with_port_gets_http(self, ip):
        result = _normalize(ip)
        assert result == f"http://{ip}"


# ===================================================================
# 6. Public IPs and hostnames -> https
# ===================================================================
class TestPublicHosts:
    """Public IPs and hostnames should get https://."""

    @pytest.mark.parametrize(
        "host",
        [
            "8.8.8.8",
            "1.1.1.1",
            "example.com",
            "api.openai.com",
            "sub.domain.example.org",
        ],
    )
    def test_public_host_gets_https(self, host):
        result = _normalize(host)
        assert result == f"https://{host}"

    def test_public_ip_with_port(self):
        assert _normalize("8.8.8.8:53") == "https://8.8.8.8:53"

    def test_public_hostname_with_port(self):
        assert _normalize("example.com:443") == "https://example.com:443"


# ===================================================================
# 7. IPv6 addresses
# ===================================================================
class TestIPv6:
    """IPv6 addresses in brackets."""

    def test_ipv6_loopback_gets_http(self):
        assert _normalize("[::1]") == "http://[::1]"

    def test_ipv6_loopback_with_port(self):
        assert _normalize("[::1]:8080") == "http://[::1]:8080"

    def test_ipv6_link_local_gets_http(self):
        result = _normalize("[fe80::1]:8080")
        assert result.startswith("http://")

    def test_ipv6_private_fc00_gets_http(self):
        result = _normalize("[fc00::1]:9200")
        assert result.startswith("http://")

    def test_ipv6_public_gets_https(self):
        result = _normalize("[2001:4860:4860::8888]")
        assert result.startswith("https://")

    def test_ipv6_public_with_port_gets_https(self):
        result = _normalize("[2607:f8b0::1]:443")
        assert result.startswith("https://")
        assert "[2607:f8b0::1]" in result

    def test_ipv6_with_existing_http_scheme(self):
        assert _normalize("http://[::1]:11434") == "http://[::1]:11434"

    def test_ipv6_with_existing_https_scheme(self):
        url = "https://[2001:db8::1]:443"
        assert _normalize(url) == url


# ===================================================================
# 8. .local mDNS domains
# ===================================================================
class TestLocalDomains:
    """mDNS .local domains should be treated as private."""

    def test_dot_local_gets_http(self):
        assert _normalize("myserver.local") == "http://myserver.local"

    def test_dot_local_with_port(self):
        assert _normalize("myserver.local:8080") == "http://myserver.local:8080"

    def test_dot_local_with_path(self):
        assert (
            _normalize("printer.local/status") == "http://printer.local/status"
        )

    def test_not_local_suffix(self):
        """localnet.com is NOT a .local domain."""
        assert _normalize("localnet.com") == "https://localnet.com"


# ===================================================================
# 9. Link-local addresses (169.254.x.x)
# ===================================================================
class TestLinkLocal:
    def test_link_local_ipv4(self):
        assert _normalize("169.254.0.1") == "http://169.254.0.1"

    def test_link_local_ipv4_with_port(self):
        assert _normalize("169.254.1.100:3000") == "http://169.254.1.100:3000"


# ===================================================================
# 10. Private/public boundary cases
# ===================================================================
class TestBoundary:
    """Boundary values at the edges of private ranges."""

    def test_172_15_is_public(self):
        """172.15.x.x is NOT in 172.16.0.0/12 private range."""
        assert _normalize("172.15.255.255") == "https://172.15.255.255"

    def test_172_16_is_private(self):
        assert _normalize("172.16.0.0") == "http://172.16.0.0"

    def test_172_31_is_private(self):
        assert _normalize("172.31.255.255") == "http://172.31.255.255"

    def test_172_32_is_public(self):
        """172.32.x.x is beyond the 172.16.0.0/12 range."""
        assert _normalize("172.32.0.0") == "https://172.32.0.0"

    def test_11_0_0_1_is_public(self):
        """11.0.0.1 is outside the 10.0.0.0/8 range."""
        assert _normalize("11.0.0.1") == "https://11.0.0.1"

    def test_192_167_is_public(self):
        assert _normalize("192.167.1.1") == "https://192.167.1.1"


# ===================================================================
# 11. Whitespace handling
# ===================================================================
class TestWhitespace:
    def test_leading_whitespace(self):
        assert _normalize("  https://example.com") == "https://example.com"

    def test_trailing_whitespace(self):
        assert _normalize("https://example.com  ") == "https://example.com"

    def test_both_whitespace(self):
        assert _normalize("  https://example.com  ") == "https://example.com"

    def test_whitespace_around_bare_host(self):
        assert _normalize("  localhost:8080  ") == "http://localhost:8080"

    def test_tab_whitespace(self):
        assert _normalize("\thttps://example.com\t") == "https://example.com"

    def test_whitespace_only_returns_bare_scheme(self):
        """Whitespace-only passes initial truthy check, strips to empty."""
        # After strip, raw_url is "" which is falsy but the check is before strip
        # Actually: " " is truthy, strip makes it "", then no early return,
        # hostname becomes "" which is_private_ip returns False -> https://
        result = _normalize("   ")
        assert result == "https://"


# ===================================================================
# 12. Empty / None / invalid input
# ===================================================================
class TestInvalidInput:
    def test_empty_string_raises(self):
        with pytest.raises(ValueError, match="(?i)empty"):
            _normalize("")

    def test_none_raises(self):
        with pytest.raises((ValueError, TypeError, AttributeError)):
            _normalize(None)


# ===================================================================
# 13. URL preservation (paths, query strings, fragments, userinfo)
# ===================================================================
class TestUrlPreservation:
    """Everything after the host should be preserved intact."""

    def test_path_preserved(self):
        assert (
            _normalize("http://example.com/a/b/c") == "http://example.com/a/b/c"
        )

    def test_query_preserved(self):
        assert (
            _normalize("http://example.com?a=1&b=2")
            == "http://example.com?a=1&b=2"
        )

    def test_fragment_preserved(self):
        assert (
            _normalize("http://example.com/p#sec") == "http://example.com/p#sec"
        )

    def test_encoded_chars_preserved(self):
        url = "http://example.com/path%20with%20spaces"
        assert _normalize(url) == url

    def test_userinfo_preserved(self):
        url = "http://user:pass@example.com:9200/path"
        assert _normalize(url) == url

    def test_complex_query_preserved(self):
        url = "https://example.com/search?q=hello+world&lang=en&page=2#results"
        assert _normalize(url) == url

    def test_path_added_to_bare_host(self):
        result = _normalize("example.com/api/v1/data")
        assert result == "https://example.com/api/v1/data"


# ===================================================================
# 14. is_private_ip direct tests (re-exported from url_utils)
# ===================================================================
class TestIsPrivateIpReExport:
    """Verify is_private_ip is accessible from url_utils and works."""

    def test_reexport_available(self):
        from local_deep_research.utilities.url_utils import is_private_ip

        assert callable(is_private_ip)

    def test_reexport_matches_original(self):
        from local_deep_research.utilities.url_utils import (
            is_private_ip as via_utils,
        )
        from local_deep_research.security.network_utils import (
            is_private_ip as via_security,
        )

        for host in [
            "localhost",
            "127.0.0.1",
            "10.0.0.1",
            "8.8.8.8",
            "example.com",
        ]:
            assert via_utils(host) == via_security(host), f"Mismatch for {host}"


# ===================================================================
# 15. is_private_ip detailed tests
# ===================================================================
class TestIsPrivateIp:
    """Directly test the is_private_ip function."""

    @pytest.mark.parametrize(
        "hostname",
        ["localhost", "127.0.0.1", "[::1]", "0.0.0.0"],
    )
    def test_known_localhost_values(self, hostname):
        assert _is_private(hostname) is True

    @pytest.mark.parametrize(
        "hostname",
        [
            "10.0.0.1",
            "10.255.255.255",
            "172.16.0.1",
            "172.31.255.255",
            "192.168.0.1",
            "192.168.255.254",
        ],
    )
    def test_rfc1918_private(self, hostname):
        assert _is_private(hostname) is True

    @pytest.mark.parametrize(
        "hostname",
        ["8.8.8.8", "1.1.1.1", "172.15.0.1", "172.32.0.1"],
    )
    def test_public_ips(self, hostname):
        assert _is_private(hostname) is False

    def test_ipv6_loopback_bracketed(self):
        assert _is_private("[::1]") is True

    def test_ipv6_link_local(self):
        assert _is_private("fe80::1") is True

    def test_ipv6_private_fc00(self):
        assert _is_private("fc00::1") is True

    def test_ipv6_public(self):
        assert _is_private("2001:4860:4860::8888") is False

    def test_dot_local_domain(self):
        assert _is_private("printer.local") is True

    def test_non_local_domain(self):
        assert _is_private("example.com") is False

    def test_empty_string(self):
        assert _is_private("") is False

    def test_bracketed_ipv6_private(self):
        """Bracketed fc00:: should be private."""
        assert _is_private("[fc00::1]") is True

    def test_169_254_link_local(self):
        assert _is_private("169.254.1.1") is True


# ===================================================================
# 16. Real-world usage patterns
# ===================================================================
class TestRealWorldPatterns:
    """URLs commonly seen in local-deep-research configuration."""

    def test_ollama_default(self):
        assert _normalize("localhost:11434") == "http://localhost:11434"

    def test_ollama_with_path(self):
        assert (
            _normalize("localhost:11434/api/generate")
            == "http://localhost:11434/api/generate"
        )

    def test_searxng_local(self):
        assert _normalize("192.168.1.100:8888") == "http://192.168.1.100:8888"

    def test_docker_internal(self):
        assert _normalize("172.17.0.2:5000") == "http://172.17.0.2:5000"

    def test_openai_api(self):
        assert _normalize("api.openai.com") == "https://api.openai.com"

    def test_already_correct_ollama(self):
        assert _normalize("http://localhost:11434") == "http://localhost:11434"

    def test_already_correct_openai(self):
        url = "https://api.openai.com/v1/chat/completions"
        assert _normalize(url) == url

    def test_elasticsearch_local(self):
        assert _normalize("localhost:9200") == "http://localhost:9200"

    def test_vllm_local(self):
        assert _normalize("0.0.0.0:8000") == "http://0.0.0.0:8000"


# ===================================================================
# 17. Mock-based tests for explicit branch coverage
# ===================================================================
class TestWithMockedPrivateIp:
    """Use mocks to guarantee we hit both branches of the scheme decision."""

    def test_private_branch_returns_http(self):
        with patch(
            "local_deep_research.utilities.url_utils.is_private_ip",
            return_value=True,
        ):
            assert _normalize("anyhost:1234") == "http://anyhost:1234"

    def test_public_branch_returns_https(self):
        with patch(
            "local_deep_research.utilities.url_utils.is_private_ip",
            return_value=False,
        ):
            assert _normalize("anyhost:1234") == "https://anyhost:1234"

    def test_double_slash_private_branch(self):
        with patch(
            "local_deep_research.utilities.url_utils.is_private_ip",
            return_value=True,
        ):
            assert _normalize("//somehost") == "http://somehost"

    def test_double_slash_public_branch(self):
        with patch(
            "local_deep_research.utilities.url_utils.is_private_ip",
            return_value=False,
        ):
            assert _normalize("//somehost") == "https://somehost"

    def test_ipv6_bracket_extraction_calls_is_private_with_brackets(self):
        """When a bracketed IPv6 is found, the hostname includes brackets."""
        with patch(
            "local_deep_research.utilities.url_utils.is_private_ip",
            return_value=True,
        ) as mock_ip:
            _normalize("[::1]:8080")
            mock_ip.assert_called_once_with("[::1]")

    def test_hostname_extraction_splits_on_colon(self):
        """Hostname is extracted before : for non-IPv6."""
        with patch(
            "local_deep_research.utilities.url_utils.is_private_ip",
            return_value=False,
        ) as mock_ip:
            _normalize("myhost:9999")
            mock_ip.assert_called_once_with("myhost")

    def test_hostname_extraction_splits_on_slash(self):
        """Hostname is extracted before / when no port."""
        with patch(
            "local_deep_research.utilities.url_utils.is_private_ip",
            return_value=False,
        ) as mock_ip:
            _normalize("myhost/some/path")
            mock_ip.assert_called_once_with("myhost")