chore: import upstream snapshot with attribution

2026-07-13 12:10:23 +08:00
commit fbab2c6005
567 changed files with 114434 additions and 0 deletions
@@ -0,0 +1,136 @@
+"""
+URL validation shared by the credentials API and the AI provisioning layer.
+
+Lives in open_notebook.utils (not api/) so it can be imported both by the API
+layer (credential create/update, source-URL ingestion) and by open_notebook's
+own AI layer (connection_tester.py, ModelManager) to re-validate a
+provider-configured URL immediately before it is actually used - closing most
+of the DNS-rebinding TOCTOU window left by validating only once, at save time.
+"""
+
+import asyncio
+import ipaddress
+import socket
+from urllib.parse import urlparse
+
+# AWS's IMDSv6 metadata endpoint. Unlike the IPv4 metadata address
+# (169.254.169.254), this is a Unique Local Address, not link-local, so it is
+# not caught by ip.is_link_local and must be checked explicitly.
+_AWS_IMDS_V6_ADDRESS = ipaddress.ip_address("fd00:ec2::254")
+
+
+async def validate_url(url: str, provider: str) -> None:
+    """
+    Validate URL format for API endpoints.
+
+    This is a self-hosted application, so we allow:
+    - Private IPs (10.x, 172.16-31.x, 192.168.x) for self-hosted services
+    - Localhost for local services (Ollama, LM Studio, etc.)
+
+    We only block:
+    - Invalid schemes (must be http or https)
+    - Malformed URLs
+    - Link-local addresses (169.254.x.x) - used for cloud metadata endpoints
+    - AWS's IPv6 metadata address (fd00:ec2::254)
+    - Hostnames that resolve to any of the above
+
+    Args:
+        url: The URL to validate
+        provider: The provider name (for logging/context)
+
+    Raises:
+        ValueError: If the URL is invalid
+    """
+    if not url or not url.strip():
+        return  # Empty URLs handled elsewhere
+
+    try:
+        parsed = urlparse(url.strip())
+
+        # Validate scheme - only http/https allowed
+        if parsed.scheme not in ("http", "https"):
+            raise ValueError(
+                f"Invalid URL scheme: '{parsed.scheme}'. Only http and https are allowed."
+            )
+
+        # Extract hostname
+        hostname = parsed.hostname
+        if not hostname:
+            raise ValueError("Invalid URL: hostname could not be determined.")
+
+        # Try to parse as IP address to check for dangerous addresses
+        try:
+            ip = ipaddress.ip_address(hostname)
+            _reject_dangerous_ip(ip, hostname)
+
+        except ValueError as ve:
+            # Re-raise our own ValueErrors
+            if "Link-local" in str(ve) or "Invalid URL" in str(ve) or "metadata" in str(ve):
+                raise
+            # Not an IP address, it's a hostname - need to resolve and check
+            try:
+                # Resolve hostname to IP address. This is a blocking call -
+                # run it off the event loop so a slow/hanging DNS lookup
+                # doesn't stall every other concurrent request (this is
+                # called on the hot path of model provisioning, potentially
+                # once per chat message/transformation).
+                resolved_ips = await asyncio.to_thread(socket.getaddrinfo, hostname, None)
+                for family, _, _, _, sockaddr in resolved_ips:
+                    ip_addr = sockaddr[0]
+                    try:
+                        parsed_ip = ipaddress.ip_address(ip_addr)
+                        _reject_dangerous_ip(parsed_ip, hostname, resolved=True)
+                    except ValueError as inner_ve:
+                        if "link-local" in str(inner_ve).lower() or "metadata" in str(inner_ve).lower():
+                            raise
+                        # Skip non-IP addresses (e.g., IPv6 zones)
+                        continue
+            except socket.gaierror:
+                # Could not resolve hostname - allow it since the URL may be
+                # valid in the deployment environment (e.g., Azure endpoints,
+                # internal DNS names). We only block link-local addresses.
+                pass
+
+    except ValueError:
+        raise
+    except Exception:
+        raise ValueError("Invalid URL format. Check server logs for details.")
+
+
+def _reject_dangerous_ip(
+    ip: "ipaddress.IPv4Address | ipaddress.IPv6Address",
+    hostname: str,
+    resolved: bool = False,
+) -> None:
+    """Raise ValueError if `ip` is a link-local or cloud-metadata address."""
+    is_ipv4_mapped_link_local = (
+        hasattr(ip, "ipv4_mapped") and ip.ipv4_mapped and ip.ipv4_mapped.is_link_local
+    )
+
+    # Block link-local addresses (169.254.x.x / fe80::/10) - used for cloud
+    # metadata - including IPv4-mapped IPv6 addresses pointing to link-local
+    # (e.g. ::ffff:169.254.169.254 bypasses IPv6 is_link_local check).
+    if ip.is_link_local or is_ipv4_mapped_link_local:
+        if resolved:
+            raise ValueError(
+                f"Hostname '{hostname}' resolves to a link-local address (169.254.x.x) "
+                "which is not allowed for security reasons. These addresses are used "
+                "for cloud metadata endpoints."
+            )
+        raise ValueError(
+            "Link-local addresses (169.254.x.x) are not allowed for security reasons. "
+            "These addresses are used for cloud metadata endpoints."
+        )
+
+    # Block AWS's IMDSv6 metadata address - a Unique Local Address, not
+    # link-local, so it needs its own explicit check.
+    if ip == _AWS_IMDS_V6_ADDRESS:
+        if resolved:
+            raise ValueError(
+                f"Hostname '{hostname}' resolves to the AWS IMDSv6 metadata address "
+                "(fd00:ec2::254), which is not allowed for security reasons."
+            )
+        raise ValueError(
+            "The AWS IMDSv6 metadata address (fd00:ec2::254) is not allowed for "
+            "security reasons."
+        )