""" Comprehensive coverage tests for url_utils.normalize_url. Focuses on exhaustive path coverage including: - http/https scheme passthrough - Malformed scheme correction (http: / https: without //) - Double-slash prefix stripping - Hostname extraction for private vs public classification - IPv6 bracket handling - Edge cases: empty, whitespace, None, special characters """ import pytest from unittest.mock import patch # --------------------------------------------------------------------------- # Helper – import once per test to keep tests independent of import order # --------------------------------------------------------------------------- def _normalize(url): from local_deep_research.utilities.url_utils import normalize_url return normalize_url(url) def _is_private(hostname): from local_deep_research.security.network_utils import is_private_ip return is_private_ip(hostname) # =================================================================== # 1. Proper scheme passthrough (early return on line 34) # =================================================================== class TestSchemePassthrough: """URLs that already have http:// or https:// are returned verbatim.""" @pytest.mark.parametrize( "url", [ "http://localhost", "http://localhost:11434", "http://127.0.0.1:8080", "http://192.168.1.1/api/v1", "http://[::1]:11434", "http://example.com?q=1&r=2#frag", "http://user:pass@host:9200/path", ], ) def test_http_urls_returned_unchanged(self, url): assert _normalize(url) == url @pytest.mark.parametrize( "url", [ "https://example.com", "https://api.openai.com/v1/chat", "https://example.com:8443", "https://example.com/search?q=hello", "https://[2001:db8::1]:443/path", ], ) def test_https_urls_returned_unchanged(self, url): assert _normalize(url) == url def test_http_triple_slash_passthrough(self): """http:/// still starts with http:// so the early return fires.""" assert _normalize("http:///some/path") == "http:///some/path" def test_https_triple_slash_passthrough(self): assert _normalize("https:///some/path") == "https:///some/path" # =================================================================== # 2. Malformed scheme correction (lines 38-43) # =================================================================== class TestMalformedScheme: """URLs like http:host or https:host get slashes inserted.""" def test_http_colon_hostname(self): assert _normalize("http:example.com") == "http://example.com" def test_https_colon_hostname(self): assert _normalize("https:example.com") == "https://example.com" def test_http_colon_hostname_with_port(self): assert _normalize("http:example.com:8080") == "http://example.com:8080" def test_https_colon_hostname_with_path(self): assert ( _normalize("https:example.com/api/v1") == "https://example.com/api/v1" ) def test_http_colon_localhost(self): assert _normalize("http:localhost:11434") == "http://localhost:11434" def test_https_colon_ip(self): assert _normalize("https:8.8.8.8") == "https://8.8.8.8" def test_http_colon_private_ip(self): assert _normalize("http:192.168.1.1:3000") == "http://192.168.1.1:3000" def test_malformed_preserves_query(self): assert ( _normalize("http:example.com/p?a=1") == "http://example.com/p?a=1" ) # =================================================================== # 3. Double-slash prefix (lines 46-48) # =================================================================== class TestDoubleSlashPrefix: """Protocol-relative URLs starting with // are stripped then re-schemed.""" def test_double_slash_public_host(self): result = _normalize("//example.com") assert result == "https://example.com" def test_double_slash_public_with_port(self): result = _normalize("//example.com:443") assert result == "https://example.com:443" def test_double_slash_localhost(self): result = _normalize("//localhost:8080") assert result == "http://localhost:8080" def test_double_slash_private_ip(self): result = _normalize("//192.168.1.1:9200") assert result == "http://192.168.1.1:9200" def test_double_slash_with_path(self): result = _normalize("//example.com/api/v1") assert result == "https://example.com/api/v1" def test_double_slash_127(self): result = _normalize("//127.0.0.1:11434") assert result == "http://127.0.0.1:11434" # =================================================================== # 4. Localhost / loopback addresses # =================================================================== class TestLocalhost: """localhost and 127.0.0.1 should always get http://.""" def test_localhost_bare(self): assert _normalize("localhost") == "http://localhost" def test_localhost_with_port(self): assert _normalize("localhost:11434") == "http://localhost:11434" def test_localhost_with_path(self): assert _normalize("localhost:8080/api") == "http://localhost:8080/api" def test_127_0_0_1_bare(self): assert _normalize("127.0.0.1") == "http://127.0.0.1" def test_127_0_0_1_with_port(self): assert _normalize("127.0.0.1:11434") == "http://127.0.0.1:11434" def test_0_0_0_0(self): assert _normalize("0.0.0.0") == "http://0.0.0.0" def test_0_0_0_0_with_port(self): assert _normalize("0.0.0.0:5000") == "http://0.0.0.0:5000" # =================================================================== # 5. RFC 1918 private IP ranges # =================================================================== class TestPrivateIPs: """Private IPs (10.x, 172.16-31.x, 192.168.x) should get http://.""" @pytest.mark.parametrize( "ip", [ "10.0.0.1", "10.255.255.255", "172.16.0.1", "172.31.255.255", "192.168.0.1", "192.168.255.254", ], ) def test_private_ip_gets_http(self, ip): result = _normalize(ip) assert result == f"http://{ip}" @pytest.mark.parametrize( "ip", [ "10.0.0.1:8080", "172.16.0.50:3000", "192.168.1.100:8888", ], ) def test_private_ip_with_port_gets_http(self, ip): result = _normalize(ip) assert result == f"http://{ip}" # =================================================================== # 6. Public IPs and hostnames -> https # =================================================================== class TestPublicHosts: """Public IPs and hostnames should get https://.""" @pytest.mark.parametrize( "host", [ "8.8.8.8", "1.1.1.1", "example.com", "api.openai.com", "sub.domain.example.org", ], ) def test_public_host_gets_https(self, host): result = _normalize(host) assert result == f"https://{host}" def test_public_ip_with_port(self): assert _normalize("8.8.8.8:53") == "https://8.8.8.8:53" def test_public_hostname_with_port(self): assert _normalize("example.com:443") == "https://example.com:443" # =================================================================== # 7. IPv6 addresses # =================================================================== class TestIPv6: """IPv6 addresses in brackets.""" def test_ipv6_loopback_gets_http(self): assert _normalize("[::1]") == "http://[::1]" def test_ipv6_loopback_with_port(self): assert _normalize("[::1]:8080") == "http://[::1]:8080" def test_ipv6_link_local_gets_http(self): result = _normalize("[fe80::1]:8080") assert result.startswith("http://") def test_ipv6_private_fc00_gets_http(self): result = _normalize("[fc00::1]:9200") assert result.startswith("http://") def test_ipv6_public_gets_https(self): result = _normalize("[2001:4860:4860::8888]") assert result.startswith("https://") def test_ipv6_public_with_port_gets_https(self): result = _normalize("[2607:f8b0::1]:443") assert result.startswith("https://") assert "[2607:f8b0::1]" in result def test_ipv6_with_existing_http_scheme(self): assert _normalize("http://[::1]:11434") == "http://[::1]:11434" def test_ipv6_with_existing_https_scheme(self): url = "https://[2001:db8::1]:443" assert _normalize(url) == url # =================================================================== # 8. .local mDNS domains # =================================================================== class TestLocalDomains: """mDNS .local domains should be treated as private.""" def test_dot_local_gets_http(self): assert _normalize("myserver.local") == "http://myserver.local" def test_dot_local_with_port(self): assert _normalize("myserver.local:8080") == "http://myserver.local:8080" def test_dot_local_with_path(self): assert ( _normalize("printer.local/status") == "http://printer.local/status" ) def test_not_local_suffix(self): """localnet.com is NOT a .local domain.""" assert _normalize("localnet.com") == "https://localnet.com" # =================================================================== # 9. Link-local addresses (169.254.x.x) # =================================================================== class TestLinkLocal: def test_link_local_ipv4(self): assert _normalize("169.254.0.1") == "http://169.254.0.1" def test_link_local_ipv4_with_port(self): assert _normalize("169.254.1.100:3000") == "http://169.254.1.100:3000" # =================================================================== # 10. Private/public boundary cases # =================================================================== class TestBoundary: """Boundary values at the edges of private ranges.""" def test_172_15_is_public(self): """172.15.x.x is NOT in 172.16.0.0/12 private range.""" assert _normalize("172.15.255.255") == "https://172.15.255.255" def test_172_16_is_private(self): assert _normalize("172.16.0.0") == "http://172.16.0.0" def test_172_31_is_private(self): assert _normalize("172.31.255.255") == "http://172.31.255.255" def test_172_32_is_public(self): """172.32.x.x is beyond the 172.16.0.0/12 range.""" assert _normalize("172.32.0.0") == "https://172.32.0.0" def test_11_0_0_1_is_public(self): """11.0.0.1 is outside the 10.0.0.0/8 range.""" assert _normalize("11.0.0.1") == "https://11.0.0.1" def test_192_167_is_public(self): assert _normalize("192.167.1.1") == "https://192.167.1.1" # =================================================================== # 11. Whitespace handling # =================================================================== class TestWhitespace: def test_leading_whitespace(self): assert _normalize(" https://example.com") == "https://example.com" def test_trailing_whitespace(self): assert _normalize("https://example.com ") == "https://example.com" def test_both_whitespace(self): assert _normalize(" https://example.com ") == "https://example.com" def test_whitespace_around_bare_host(self): assert _normalize(" localhost:8080 ") == "http://localhost:8080" def test_tab_whitespace(self): assert _normalize("\thttps://example.com\t") == "https://example.com" def test_whitespace_only_returns_bare_scheme(self): """Whitespace-only passes initial truthy check, strips to empty.""" # After strip, raw_url is "" which is falsy but the check is before strip # Actually: " " is truthy, strip makes it "", then no early return, # hostname becomes "" which is_private_ip returns False -> https:// result = _normalize(" ") assert result == "https://" # =================================================================== # 12. Empty / None / invalid input # =================================================================== class TestInvalidInput: def test_empty_string_raises(self): with pytest.raises(ValueError, match="(?i)empty"): _normalize("") def test_none_raises(self): with pytest.raises((ValueError, TypeError, AttributeError)): _normalize(None) # =================================================================== # 13. URL preservation (paths, query strings, fragments, userinfo) # =================================================================== class TestUrlPreservation: """Everything after the host should be preserved intact.""" def test_path_preserved(self): assert ( _normalize("http://example.com/a/b/c") == "http://example.com/a/b/c" ) def test_query_preserved(self): assert ( _normalize("http://example.com?a=1&b=2") == "http://example.com?a=1&b=2" ) def test_fragment_preserved(self): assert ( _normalize("http://example.com/p#sec") == "http://example.com/p#sec" ) def test_encoded_chars_preserved(self): url = "http://example.com/path%20with%20spaces" assert _normalize(url) == url def test_userinfo_preserved(self): url = "http://user:pass@example.com:9200/path" assert _normalize(url) == url def test_complex_query_preserved(self): url = "https://example.com/search?q=hello+world&lang=en&page=2#results" assert _normalize(url) == url def test_path_added_to_bare_host(self): result = _normalize("example.com/api/v1/data") assert result == "https://example.com/api/v1/data" # =================================================================== # 14. is_private_ip direct tests (re-exported from url_utils) # =================================================================== class TestIsPrivateIpReExport: """Verify is_private_ip is accessible from url_utils and works.""" def test_reexport_available(self): from local_deep_research.utilities.url_utils import is_private_ip assert callable(is_private_ip) def test_reexport_matches_original(self): from local_deep_research.utilities.url_utils import ( is_private_ip as via_utils, ) from local_deep_research.security.network_utils import ( is_private_ip as via_security, ) for host in [ "localhost", "127.0.0.1", "10.0.0.1", "8.8.8.8", "example.com", ]: assert via_utils(host) == via_security(host), f"Mismatch for {host}" # =================================================================== # 15. is_private_ip detailed tests # =================================================================== class TestIsPrivateIp: """Directly test the is_private_ip function.""" @pytest.mark.parametrize( "hostname", ["localhost", "127.0.0.1", "[::1]", "0.0.0.0"], ) def test_known_localhost_values(self, hostname): assert _is_private(hostname) is True @pytest.mark.parametrize( "hostname", [ "10.0.0.1", "10.255.255.255", "172.16.0.1", "172.31.255.255", "192.168.0.1", "192.168.255.254", ], ) def test_rfc1918_private(self, hostname): assert _is_private(hostname) is True @pytest.mark.parametrize( "hostname", ["8.8.8.8", "1.1.1.1", "172.15.0.1", "172.32.0.1"], ) def test_public_ips(self, hostname): assert _is_private(hostname) is False def test_ipv6_loopback_bracketed(self): assert _is_private("[::1]") is True def test_ipv6_link_local(self): assert _is_private("fe80::1") is True def test_ipv6_private_fc00(self): assert _is_private("fc00::1") is True def test_ipv6_public(self): assert _is_private("2001:4860:4860::8888") is False def test_dot_local_domain(self): assert _is_private("printer.local") is True def test_non_local_domain(self): assert _is_private("example.com") is False def test_empty_string(self): assert _is_private("") is False def test_bracketed_ipv6_private(self): """Bracketed fc00:: should be private.""" assert _is_private("[fc00::1]") is True def test_169_254_link_local(self): assert _is_private("169.254.1.1") is True # =================================================================== # 16. Real-world usage patterns # =================================================================== class TestRealWorldPatterns: """URLs commonly seen in local-deep-research configuration.""" def test_ollama_default(self): assert _normalize("localhost:11434") == "http://localhost:11434" def test_ollama_with_path(self): assert ( _normalize("localhost:11434/api/generate") == "http://localhost:11434/api/generate" ) def test_searxng_local(self): assert _normalize("192.168.1.100:8888") == "http://192.168.1.100:8888" def test_docker_internal(self): assert _normalize("172.17.0.2:5000") == "http://172.17.0.2:5000" def test_openai_api(self): assert _normalize("api.openai.com") == "https://api.openai.com" def test_already_correct_ollama(self): assert _normalize("http://localhost:11434") == "http://localhost:11434" def test_already_correct_openai(self): url = "https://api.openai.com/v1/chat/completions" assert _normalize(url) == url def test_elasticsearch_local(self): assert _normalize("localhost:9200") == "http://localhost:9200" def test_vllm_local(self): assert _normalize("0.0.0.0:8000") == "http://0.0.0.0:8000" # =================================================================== # 17. Mock-based tests for explicit branch coverage # =================================================================== class TestWithMockedPrivateIp: """Use mocks to guarantee we hit both branches of the scheme decision.""" def test_private_branch_returns_http(self): with patch( "local_deep_research.utilities.url_utils.is_private_ip", return_value=True, ): assert _normalize("anyhost:1234") == "http://anyhost:1234" def test_public_branch_returns_https(self): with patch( "local_deep_research.utilities.url_utils.is_private_ip", return_value=False, ): assert _normalize("anyhost:1234") == "https://anyhost:1234" def test_double_slash_private_branch(self): with patch( "local_deep_research.utilities.url_utils.is_private_ip", return_value=True, ): assert _normalize("//somehost") == "http://somehost" def test_double_slash_public_branch(self): with patch( "local_deep_research.utilities.url_utils.is_private_ip", return_value=False, ): assert _normalize("//somehost") == "https://somehost" def test_ipv6_bracket_extraction_calls_is_private_with_brackets(self): """When a bracketed IPv6 is found, the hostname includes brackets.""" with patch( "local_deep_research.utilities.url_utils.is_private_ip", return_value=True, ) as mock_ip: _normalize("[::1]:8080") mock_ip.assert_called_once_with("[::1]") def test_hostname_extraction_splits_on_colon(self): """Hostname is extracted before : for non-IPv6.""" with patch( "local_deep_research.utilities.url_utils.is_private_ip", return_value=False, ) as mock_ip: _normalize("myhost:9999") mock_ip.assert_called_once_with("myhost") def test_hostname_extraction_splits_on_slash(self): """Hostname is extracted before / when no port.""" with patch( "local_deep_research.utilities.url_utils.is_private_ip", return_value=False, ) as mock_ip: _normalize("myhost/some/path") mock_ip.assert_called_once_with("myhost")