wehub/unclecode--crawl4ai
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:12:13 +08:00
commit 0446c45d8e
898 changed files with 328024 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
deploy/docker/tests/conftest.py
+168
View File
@@ -0,0 +1,168 @@
"""
Shared fixtures for the Crawl4AI Docker server *behavioral* security tests.
Unlike the legacy `test_security_*.py` suites (which grep the source text and
pass even when the running server is wide open), these fixtures boot the real
FastAPI application via Starlette's TestClient and exercise it as a client
would. That makes the tests behavioral: they assert what the server actually
*does*, not what the source happens to contain.
Design notes
------------
* The app is imported once per session. `deploy/docker` is put on `sys.path`
first, because `server.py` does `from crawler_pool import ...` at module top,
before its own `sys.path.append`.
* The TestClient is created *without* the `with` context-manager form on
purpose: that skips the FastAPI lifespan, so no Chromium is launched and no
Redis connection is opened at startup. Authentication is decided by a
dependency/middleware that runs before any route handler, so auth-posture
assertions resolve before the app ever needs a browser or Redis.
* `offline_dns` / `rebinding_dns` monkeypatch name resolution so the SSRF /
egress behavioral tests (R3) run fully offline and can model DNS-rebinding
(resolve-then-reconnect TOCTOU) deterministically.
"""
import importlib
import socket
import sys
from pathlib import Path
import pytest
# deploy/docker (the dir that holds server.py, auth.py, ...) must be importable
# before we import `server`, since server.py imports its siblings by bare name.
DOCKER_DIR = Path(__file__).resolve().parents[1]
if str(DOCKER_DIR) not in sys.path:
sys.path.insert(0, str(DOCKER_DIR))
def pytest_configure(config):
config.addinivalue_line(
"markers",
"posture: the secure-by-default acceptance gate (expected RED until R1-R7 land)",
)
config.addinivalue_line(
"markers", "cve: regression test for a specific reported vulnerability"
)
@pytest.fixture(scope="session")
def server_module():
"""Import the Docker server app once for the whole test session.
Returns the imported `server` module so tests can introspect the loaded
config, the redis-url builder, feature flags, etc.
"""
# Import fresh; if a previous test mutated it we still want the real module.
if "server" in sys.modules:
return sys.modules["server"]
return importlib.import_module("server")
@pytest.fixture
def stock_client(server_module):
"""A TestClient bound to the app as shipped (config.yml defaults).
No lifespan => no real browser, no Redis connect. Auth/headers/routing are
all exercised normally because they sit in front of the route handlers.
"""
from starlette.testclient import TestClient
# raise_server_exceptions=False so an un-gated handler that blows up on a
# missing pool/redis surfaces as a 500 response (which our posture test
# treats as "not 401" => still a finding) instead of bubbling out of the
# test client as an exception.
return TestClient(server_module.app, raise_server_exceptions=False)
@pytest.fixture
def effective_config(server_module):
"""The configuration dict the running app actually loaded."""
return server_module.config
@pytest.fixture
def effective_browser_args(effective_config):
"""The Chromium launch flags the app will pass to every browser."""
return list(effective_config["crawler"]["browser"].get("extra_args", []))
@pytest.fixture
def effective_redis_url(server_module):
"""The Redis URL the app builds from config + environment."""
return server_module._build_redis_url(server_module.config)
# ─────────────────────── offline DNS control ────────────────────────
# These let SSRF/egress tests run without touching the network and let us
# model DNS rebinding precisely.
class _FakeResolver:
"""Drop-in for socket.getaddrinfo with a controllable host->IP map.
Unknown hosts resolve to a stable public-ish default so accidental real
lookups never escape the test process.
"""
DEFAULT_IP = "93.184.216.34" # documentation/example address space
def __init__(self):
self.map = {} # host -> ip (single answer)
def set(self, host, ip):
self.map[host] = ip
def getaddrinfo(self, host, port, *args, **kwargs):
ip = self.map.get(host, self.DEFAULT_IP)
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, port or 0))]
@pytest.fixture
def offline_dns(monkeypatch):
"""Patch socket.getaddrinfo with a controllable, offline resolver.
Usage:
offline_dns.set("evil.example", "169.254.169.254")
"""
resolver = _FakeResolver()
monkeypatch.setattr(socket, "getaddrinfo", resolver.getaddrinfo)
return resolver
class _RebindingResolver:
"""Returns a *different* answer on the second+ lookup of the same host.
Models the resolve-then-discard TOCTOU: validation sees a public IP, the
later real connection re-resolves to an internal IP. A correct egress
broker resolves once and pins, so the second (internal) answer is never
dialed.
"""
def __init__(self, host, first_ip, second_ip):
self.host = host
self.first_ip = first_ip
self.second_ip = second_ip
self.calls = 0
def getaddrinfo(self, host, port, *args, **kwargs):
if host == self.host:
self.calls += 1
ip = self.first_ip if self.calls == 1 else self.second_ip
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, port or 0))]
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", port or 0))]
@pytest.fixture
def rebinding_dns(monkeypatch):
"""Factory that installs a DNS-rebinding resolver.
Usage:
r = rebinding_dns("rebind.example", "93.184.216.34", "169.254.169.254")
... # first lookup public, every later lookup internal
"""
def _install(host, first_ip, second_ip):
resolver = _RebindingResolver(host, first_ip, second_ip)
monkeypatch.setattr(socket, "getaddrinfo", resolver.getaddrinfo)
return resolver
return _install
deploy/docker/tests/demo_monitor_dashboard.py
Executable
+164
View File
@@ -0,0 +1,164 @@
#!/usr/bin/env python3
"""
Monitor Dashboard Demo Script
Generates varied activity to showcase all monitoring features for video recording.
"""
import httpx
import asyncio
import time
from datetime import datetime
BASE_URL = "http://localhost:11235"
async def demo_dashboard():
print("🎬 Monitor Dashboard Demo - Starting...\n")
print(f"📊 Dashboard: {BASE_URL}/dashboard")
print("=" * 60)
async with httpx.AsyncClient(timeout=60.0) as client:
# Phase 1: Simple requests (permanent browser)
print("\n🔷 Phase 1: Testing permanent browser pool")
print("-" * 60)
for i in range(5):
print(f" {i+1}/5 Request to /crawl (default config)...")
try:
r = await client.post(
f"{BASE_URL}/crawl",
json={"urls": [f"https://httpbin.org/html?req={i}"], "crawler_config": {}}
)
print(f" ✅ Status: {r.status_code}, Time: {r.elapsed.total_seconds():.2f}s")
except Exception as e:
print(f" ❌ Error: {e}")
await asyncio.sleep(1) # Small delay between requests
# Phase 2: Create variant browsers (different configs)
print("\n🔶 Phase 2: Testing cold→hot pool promotion")
print("-" * 60)
viewports = [
{"width": 1920, "height": 1080},
{"width": 1280, "height": 720},
{"width": 800, "height": 600}
]
for idx, viewport in enumerate(viewports):
print(f" Viewport {viewport['width']}x{viewport['height']}:")
for i in range(4): # 4 requests each to trigger promotion at 3
try:
r = await client.post(
f"{BASE_URL}/crawl",
json={
"urls": [f"https://httpbin.org/json?v={idx}&r={i}"],
"browser_config": {"viewport": viewport},
"crawler_config": {}
}
)
print(f" {i+1}/4 ✅ {r.status_code} - Should see cold→hot after 3 uses")
except Exception as e:
print(f" {i+1}/4 ❌ {e}")
await asyncio.sleep(0.5)
# Phase 3: Concurrent burst (stress pool)
print("\n🔷 Phase 3: Concurrent burst (10 parallel)")
print("-" * 60)
tasks = []
for i in range(10):
tasks.append(
client.post(
f"{BASE_URL}/crawl",
json={"urls": [f"https://httpbin.org/delay/2?burst={i}"], "crawler_config": {}}
)
)
print(" Sending 10 concurrent requests...")
start = time.time()
results = await asyncio.gather(*tasks, return_exceptions=True)
elapsed = time.time() - start
successes = sum(1 for r in results if not isinstance(r, Exception) and r.status_code == 200)
print(f" ✅ {successes}/10 succeeded in {elapsed:.2f}s")
# Phase 4: Multi-endpoint coverage
print("\n🔶 Phase 4: Testing multiple endpoints")
print("-" * 60)
endpoints = [
("/md", {"url": "https://httpbin.org/html", "f": "fit", "c": "0"}),
("/screenshot", {"url": "https://httpbin.org/html"}),
("/pdf", {"url": "https://httpbin.org/html"}),
]
for endpoint, payload in endpoints:
print(f" Testing {endpoint}...")
try:
if endpoint == "/md":
r = await client.post(f"{BASE_URL}{endpoint}", json=payload)
else:
r = await client.post(f"{BASE_URL}{endpoint}", json=payload)
print(f" ✅ {r.status_code}")
except Exception as e:
print(f" ❌ {e}")
await asyncio.sleep(1)
# Phase 5: Intentional error (to populate errors tab)
print("\n🔷 Phase 5: Generating error examples")
print("-" * 60)
print(" Triggering invalid URL error...")
try:
r = await client.post(
f"{BASE_URL}/crawl",
json={"urls": ["invalid://bad-url"], "crawler_config": {}}
)
print(f" Response: {r.status_code}")
except Exception as e:
print(f" ✅ Error captured: {type(e).__name__}")
# Phase 6: Wait for janitor activity
print("\n🔶 Phase 6: Waiting for janitor cleanup...")
print("-" * 60)
print(" Idle for 40s to allow janitor to clean cold pool browsers...")
for i in range(40, 0, -10):
print(f" {i}s remaining... (Check dashboard for cleanup events)")
await asyncio.sleep(10)
# Phase 7: Final stats check
print("\n🔷 Phase 7: Final dashboard state")
print("-" * 60)
r = await client.get(f"{BASE_URL}/monitor/health")
health = r.json()
print(f" Memory: {health['container']['memory_percent']:.1f}%")
print(f" Browsers: Perm={health['pool']['permanent']['active']}, "
f"Hot={health['pool']['hot']['count']}, Cold={health['pool']['cold']['count']}")
r = await client.get(f"{BASE_URL}/monitor/endpoints/stats")
stats = r.json()
print(f"\n Endpoint Stats:")
for endpoint, data in stats.items():
print(f" {endpoint}: {data['count']} req, "
f"{data['avg_latency_ms']:.0f}ms avg, "
f"{data['success_rate_percent']:.1f}% success")
r = await client.get(f"{BASE_URL}/monitor/browsers")
browsers = r.json()
print(f"\n Pool Efficiency:")
print(f" Total browsers: {browsers['summary']['total_count']}")
print(f" Memory usage: {browsers['summary']['total_memory_mb']} MB")
print(f" Reuse rate: {browsers['summary']['reuse_rate_percent']:.1f}%")
print("\n" + "=" * 60)
print("✅ Demo complete! Dashboard is now populated with rich data.")
print(f"\n📹 Recording tip: Refresh {BASE_URL}/dashboard")
print(" You should see:")
print(" • Active & completed requests")
print(" • Browser pool (permanent + hot/cold)")
print(" • Janitor cleanup events")
print(" • Endpoint analytics")
print(" • Memory timeline")
if __name__ == "__main__":
try:
asyncio.run(demo_dashboard())
except KeyboardInterrupt:
print("\n\n⚠️ Demo interrupted by user")
except Exception as e:
print(f"\n\n❌ Demo failed: {e}")
deploy/docker/tests/requirements.txt
+2
View File
@@ -0,0 +1,2 @@
httpx>=0.25.0
docker>=7.0.0
deploy/docker/tests/run_security_tests.py
Executable
+196
View File
@@ -0,0 +1,196 @@
#!/usr/bin/env python3
"""
Security Integration Tests for Crawl4AI Docker API.
Tests that security fixes are working correctly against a running server.
Usage:
python run_security_tests.py [base_url]
Example:
python run_security_tests.py http://localhost:11235
"""
import subprocess
import sys
import re
# Colors for terminal output
GREEN = '\033[0;32m'
RED = '\033[0;31m'
YELLOW = '\033[1;33m'
NC = '\033[0m' # No Color
PASSED = 0
FAILED = 0
def run_curl(args: list) -> str:
"""Run curl command and return output."""
try:
result = subprocess.run(
['curl', '-s'] + args,
capture_output=True,
text=True,
timeout=30
)
return result.stdout + result.stderr
except subprocess.TimeoutExpired:
return "TIMEOUT"
except Exception as e:
return str(e)
def test_expect(name: str, expect_pattern: str, curl_args: list) -> bool:
"""Run a test and check if output matches expected pattern."""
global PASSED, FAILED
result = run_curl(curl_args)
if re.search(expect_pattern, result, re.IGNORECASE):
print(f"{GREEN}✓{NC} {name}")
PASSED += 1
return True
else:
print(f"{RED}✗{NC} {name}")
print(f" Expected pattern: {expect_pattern}")
print(f" Got: {result[:200]}")
FAILED += 1
return False
def main():
global PASSED, FAILED
base_url = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:11235"
print("=" * 60)
print("Crawl4AI Security Integration Tests")
print(f"Target: {base_url}")
print("=" * 60)
print()
# Check server availability
print("Checking server availability...")
result = run_curl(['-o', '/dev/null', '-w', '%{http_code}', f'{base_url}/health'])
if '200' not in result:
print(f"{RED}ERROR: Server not reachable at {base_url}{NC}")
print("Please start the server first.")
sys.exit(1)
print(f"{GREEN}Server is running{NC}")
print()
# === Part A: Security Tests ===
print("=== Part A: Security Tests ===")
print("(Vulnerabilities must be BLOCKED)")
print()
test_expect(
"A1: Hooks disabled by default (403)",
r"403|disabled|Hooks are disabled",
['-X', 'POST', f'{base_url}/crawl',
'-H', 'Content-Type: application/json',
'-d', '{"urls":["https://example.com"],"hooks":{"code":{"on_page_context_created":"async def hook(page, context, **kwargs): return page"}}}']
)
test_expect(
"A2: file:// blocked on /execute_js (400)",
r"400|must start with",
['-X', 'POST', f'{base_url}/execute_js',
'-H', 'Content-Type: application/json',
'-d', '{"url":"file:///etc/passwd","scripts":["1"]}']
)
test_expect(
"A3: file:// blocked on /screenshot (400)",
r"400|must start with",
['-X', 'POST', f'{base_url}/screenshot',
'-H', 'Content-Type: application/json',
'-d', '{"url":"file:///etc/passwd"}']
)
test_expect(
"A4: file:// blocked on /pdf (400)",
r"400|must start with",
['-X', 'POST', f'{base_url}/pdf',
'-H', 'Content-Type: application/json',
'-d', '{"url":"file:///etc/passwd"}']
)
test_expect(
"A5: file:// blocked on /html (400)",
r"400|must start with",
['-X', 'POST', f'{base_url}/html',
'-H', 'Content-Type: application/json',
'-d', '{"url":"file:///etc/passwd"}']
)
print()
# === Part B: Functionality Tests ===
print("=== Part B: Functionality Tests ===")
print("(Normal operations must WORK)")
print()
test_expect(
"B1: Basic crawl works",
r"success.*true|results",
['-X', 'POST', f'{base_url}/crawl',
'-H', 'Content-Type: application/json',
'-d', '{"urls":["https://example.com"]}']
)
test_expect(
"B2: /md works with https://",
r"success.*true|markdown",
['-X', 'POST', f'{base_url}/md',
'-H', 'Content-Type: application/json',
'-d', '{"url":"https://example.com"}']
)
test_expect(
"B3: Health endpoint works",
r"ok",
[f'{base_url}/health']
)
print()
# === Part C: Edge Cases ===
print("=== Part C: Edge Cases ===")
print("(Malformed input must be REJECTED)")
print()
test_expect(
"C1: javascript: URL rejected (400)",
r"400|must start with",
['-X', 'POST', f'{base_url}/execute_js',
'-H', 'Content-Type: application/json',
'-d', '{"url":"javascript:alert(1)","scripts":["1"]}']
)
test_expect(
"C2: data: URL rejected (400)",
r"400|must start with",
['-X', 'POST', f'{base_url}/execute_js',
'-H', 'Content-Type: application/json',
'-d', '{"url":"data:text/html,<h1>test</h1>","scripts":["1"]}']
)
print()
print("=" * 60)
print("Results")
print("=" * 60)
print(f"Passed: {GREEN}{PASSED}{NC}")
print(f"Failed: {RED}{FAILED}{NC}")
print()
if FAILED > 0:
print(f"{RED}SOME TESTS FAILED{NC}")
sys.exit(1)
else:
print(f"{GREEN}ALL TESTS PASSED{NC}")
sys.exit(0)
if __name__ == '__main__':
main()
deploy/docker/tests/test_1_basic.py
Executable
+138
View File
@@ -0,0 +1,138 @@
#!/usr/bin/env python3
"""
Test 1: Basic Container Health + Single Endpoint
- Starts container
- Hits /health endpoint 10 times
- Reports success rate and basic latency
"""
import asyncio
import time
import docker
import httpx
# Config
IMAGE = "crawl4ai-local:latest"
CONTAINER_NAME = "crawl4ai-test"
PORT = 11235
REQUESTS = 10
async def test_endpoint(url: str, count: int):
"""Hit endpoint multiple times, return stats."""
results = []
async with httpx.AsyncClient(timeout=30.0) as client:
for i in range(count):
start = time.time()
try:
resp = await client.get(url)
elapsed = (time.time() - start) * 1000 # ms
results.append({
"success": resp.status_code == 200,
"latency_ms": elapsed,
"status": resp.status_code
})
print(f" [{i+1}/{count}] ✓ {resp.status_code} - {elapsed:.0f}ms")
except Exception as e:
results.append({
"success": False,
"latency_ms": None,
"error": str(e)
})
print(f" [{i+1}/{count}] ✗ Error: {e}")
return results
def start_container(client, image: str, name: str, port: int):
"""Start container, return container object."""
# Clean up existing
try:
old = client.containers.get(name)
print(f"🧹 Stopping existing container '{name}'...")
old.stop()
old.remove()
except docker.errors.NotFound:
pass
print(f"🚀 Starting container '{name}' from image '{image}'...")
container = client.containers.run(
image,
name=name,
ports={f"{port}/tcp": port},
detach=True,
shm_size="1g",
environment={"PYTHON_ENV": "production"}
)
# Wait for health
print(f"⏳ Waiting for container to be healthy...")
for _ in range(30): # 30s timeout
time.sleep(1)
container.reload()
if container.status == "running":
try:
# Quick health check
import requests
resp = requests.get(f"http://localhost:{port}/health", timeout=2)
if resp.status_code == 200:
print(f"✅ Container healthy!")
return container
except:
pass
raise TimeoutError("Container failed to start")
def stop_container(container):
"""Stop and remove container."""
print(f"🛑 Stopping container...")
container.stop()
container.remove()
print(f"✅ Container removed")
async def main():
print("="*60)
print("TEST 1: Basic Container Health + Single Endpoint")
print("="*60)
client = docker.from_env()
container = None
try:
# Start container
container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
# Test /health endpoint
print(f"\n📊 Testing /health endpoint ({REQUESTS} requests)...")
url = f"http://localhost:{PORT}/health"
results = await test_endpoint(url, REQUESTS)
# Calculate stats
successes = sum(1 for r in results if r["success"])
success_rate = (successes / len(results)) * 100
latencies = [r["latency_ms"] for r in results if r["latency_ms"] is not None]
avg_latency = sum(latencies) / len(latencies) if latencies else 0
# Print results
print(f"\n{'='*60}")
print(f"RESULTS:")
print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(results)})")
print(f" Avg Latency: {avg_latency:.0f}ms")
if latencies:
print(f" Min Latency: {min(latencies):.0f}ms")
print(f" Max Latency: {max(latencies):.0f}ms")
print(f"{'='*60}")
# Pass/Fail
if success_rate >= 100:
print(f"✅ TEST PASSED")
return 0
else:
print(f"❌ TEST FAILED (expected 100% success rate)")
return 1
except Exception as e:
print(f"\n❌ TEST ERROR: {e}")
return 1
finally:
if container:
stop_container(container)
if __name__ == "__main__":
exit_code = asyncio.run(main())
exit(exit_code)
deploy/docker/tests/test_2_memory.py
Executable
+205
View File
@@ -0,0 +1,205 @@
#!/usr/bin/env python3
"""
Test 2: Docker Stats Monitoring
- Extends Test 1 with real-time container stats
- Monitors memory % and CPU during requests
- Reports baseline, peak, and final memory
"""
import asyncio
import time
import docker
import httpx
from threading import Thread, Event
# Config
IMAGE = "crawl4ai-local:latest"
CONTAINER_NAME = "crawl4ai-test"
PORT = 11235
REQUESTS = 20 # More requests to see memory usage
# Stats tracking
stats_history = []
stop_monitoring = Event()
def monitor_stats(container):
"""Background thread to collect container stats."""
for stat in container.stats(decode=True, stream=True):
if stop_monitoring.is_set():
break
try:
# Extract memory stats
mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024) # MB
mem_limit = stat['memory_stats'].get('limit', 1) / (1024 * 1024)
mem_percent = (mem_usage / mem_limit * 100) if mem_limit > 0 else 0
# Extract CPU stats (handle missing fields on Mac)
cpu_percent = 0
try:
cpu_delta = stat['cpu_stats']['cpu_usage']['total_usage'] - \
stat['precpu_stats']['cpu_usage']['total_usage']
system_delta = stat['cpu_stats'].get('system_cpu_usage', 0) - \
stat['precpu_stats'].get('system_cpu_usage', 0)
if system_delta > 0:
num_cpus = stat['cpu_stats'].get('online_cpus', 1)
cpu_percent = (cpu_delta / system_delta * num_cpus * 100.0)
except (KeyError, ZeroDivisionError):
pass
stats_history.append({
'timestamp': time.time(),
'memory_mb': mem_usage,
'memory_percent': mem_percent,
'cpu_percent': cpu_percent
})
except Exception as e:
# Skip malformed stats
pass
time.sleep(0.5) # Sample every 500ms
async def test_endpoint(url: str, count: int):
"""Hit endpoint, return stats."""
results = []
async with httpx.AsyncClient(timeout=30.0) as client:
for i in range(count):
start = time.time()
try:
resp = await client.get(url)
elapsed = (time.time() - start) * 1000
results.append({
"success": resp.status_code == 200,
"latency_ms": elapsed,
})
if (i + 1) % 5 == 0: # Print every 5 requests
print(f" [{i+1}/{count}] ✓ {resp.status_code} - {elapsed:.0f}ms")
except Exception as e:
results.append({"success": False, "error": str(e)})
print(f" [{i+1}/{count}] ✗ Error: {e}")
return results
def start_container(client, image: str, name: str, port: int):
"""Start container."""
try:
old = client.containers.get(name)
print(f"🧹 Stopping existing container '{name}'...")
old.stop()
old.remove()
except docker.errors.NotFound:
pass
print(f"🚀 Starting container '{name}'...")
container = client.containers.run(
image,
name=name,
ports={f"{port}/tcp": port},
detach=True,
shm_size="1g",
mem_limit="4g", # Set explicit memory limit
)
print(f"⏳ Waiting for health...")
for _ in range(30):
time.sleep(1)
container.reload()
if container.status == "running":
try:
import requests
resp = requests.get(f"http://localhost:{port}/health", timeout=2)
if resp.status_code == 200:
print(f"✅ Container healthy!")
return container
except:
pass
raise TimeoutError("Container failed to start")
def stop_container(container):
"""Stop container."""
print(f"🛑 Stopping container...")
container.stop()
container.remove()
async def main():
print("="*60)
print("TEST 2: Docker Stats Monitoring")
print("="*60)
client = docker.from_env()
container = None
monitor_thread = None
try:
# Start container
container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
# Start stats monitoring in background
print(f"\n📊 Starting stats monitor...")
stop_monitoring.clear()
stats_history.clear()
monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
monitor_thread.start()
# Wait a bit for baseline
await asyncio.sleep(2)
baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
print(f"📏 Baseline memory: {baseline_mem:.1f} MB")
# Test /health endpoint
print(f"\n🔄 Running {REQUESTS} requests to /health...")
url = f"http://localhost:{PORT}/health"
results = await test_endpoint(url, REQUESTS)
# Wait a bit to capture peak
await asyncio.sleep(1)
# Stop monitoring
stop_monitoring.set()
if monitor_thread:
monitor_thread.join(timeout=2)
# Calculate stats
successes = sum(1 for r in results if r.get("success"))
success_rate = (successes / len(results)) * 100
latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
avg_latency = sum(latencies) / len(latencies) if latencies else 0
# Memory stats
memory_samples = [s['memory_mb'] for s in stats_history]
peak_mem = max(memory_samples) if memory_samples else 0
final_mem = memory_samples[-1] if memory_samples else 0
mem_delta = final_mem - baseline_mem
# Print results
print(f"\n{'='*60}")
print(f"RESULTS:")
print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(results)})")
print(f" Avg Latency: {avg_latency:.0f}ms")
print(f"\n Memory Stats:")
print(f" Baseline: {baseline_mem:.1f} MB")
print(f" Peak: {peak_mem:.1f} MB")
print(f" Final: {final_mem:.1f} MB")
print(f" Delta: {mem_delta:+.1f} MB")
print(f"{'='*60}")
# Pass/Fail
if success_rate >= 100 and mem_delta < 100: # No significant memory growth
print(f"✅ TEST PASSED")
return 0
else:
if success_rate < 100:
print(f"❌ TEST FAILED (success rate < 100%)")
if mem_delta >= 100:
print(f"⚠️ WARNING: Memory grew by {mem_delta:.1f} MB")
return 1
except Exception as e:
print(f"\n❌ TEST ERROR: {e}")
return 1
finally:
stop_monitoring.set()
if container:
stop_container(container)
if __name__ == "__main__":
exit_code = asyncio.run(main())
exit(exit_code)
deploy/docker/tests/test_3_pool.py
Executable
+229
View File
@@ -0,0 +1,229 @@
#!/usr/bin/env python3
"""
Test 3: Pool Validation - Permanent Browser Reuse
- Tests /html endpoint (should use permanent browser)
- Monitors container logs for pool hit markers
- Validates browser reuse rate
- Checks memory after browser creation
"""
import asyncio
import time
import docker
import httpx
from threading import Thread, Event
# Config
IMAGE = "crawl4ai-local:latest"
CONTAINER_NAME = "crawl4ai-test"
PORT = 11235
REQUESTS = 30
# Stats tracking
stats_history = []
stop_monitoring = Event()
def monitor_stats(container):
"""Background stats collector."""
for stat in container.stats(decode=True, stream=True):
if stop_monitoring.is_set():
break
try:
mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
stats_history.append({
'timestamp': time.time(),
'memory_mb': mem_usage,
})
except:
pass
time.sleep(0.5)
def count_log_markers(container):
"""Extract pool usage markers from logs."""
logs = container.logs().decode('utf-8')
permanent_hits = logs.count("🔥 Using permanent browser")
hot_hits = logs.count("♨️ Using hot pool browser")
cold_hits = logs.count("❄️ Using cold pool browser")
new_created = logs.count("🆕 Creating new browser")
return {
'permanent_hits': permanent_hits,
'hot_hits': hot_hits,
'cold_hits': cold_hits,
'new_created': new_created,
'total_hits': permanent_hits + hot_hits + cold_hits
}
async def test_endpoint(url: str, count: int):
"""Hit endpoint multiple times."""
results = []
async with httpx.AsyncClient(timeout=60.0) as client:
for i in range(count):
start = time.time()
try:
resp = await client.post(url, json={"url": "https://httpbin.org/html"})
elapsed = (time.time() - start) * 1000
results.append({
"success": resp.status_code == 200,
"latency_ms": elapsed,
})
if (i + 1) % 10 == 0:
print(f" [{i+1}/{count}] ✓ {resp.status_code} - {elapsed:.0f}ms")
except Exception as e:
results.append({"success": False, "error": str(e)})
print(f" [{i+1}/{count}] ✗ Error: {e}")
return results
def start_container(client, image: str, name: str, port: int):
"""Start container."""
try:
old = client.containers.get(name)
print(f"🧹 Stopping existing container...")
old.stop()
old.remove()
except docker.errors.NotFound:
pass
print(f"🚀 Starting container...")
container = client.containers.run(
image,
name=name,
ports={f"{port}/tcp": port},
detach=True,
shm_size="1g",
mem_limit="4g",
)
print(f"⏳ Waiting for health...")
for _ in range(30):
time.sleep(1)
container.reload()
if container.status == "running":
try:
import requests
resp = requests.get(f"http://localhost:{port}/health", timeout=2)
if resp.status_code == 200:
print(f"✅ Container healthy!")
return container
except:
pass
raise TimeoutError("Container failed to start")
def stop_container(container):
"""Stop container."""
print(f"🛑 Stopping container...")
container.stop()
container.remove()
async def main():
print("="*60)
print("TEST 3: Pool Validation - Permanent Browser Reuse")
print("="*60)
client = docker.from_env()
container = None
monitor_thread = None
try:
# Start container
container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
# Wait for permanent browser initialization
print(f"\n⏳ Waiting for permanent browser init (3s)...")
await asyncio.sleep(3)
# Start stats monitoring
print(f"📊 Starting stats monitor...")
stop_monitoring.clear()
stats_history.clear()
monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
monitor_thread.start()
await asyncio.sleep(1)
baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
print(f"📏 Baseline (with permanent browser): {baseline_mem:.1f} MB")
# Test /html endpoint (uses permanent browser for default config)
print(f"\n🔄 Running {REQUESTS} requests to /html...")
url = f"http://localhost:{PORT}/html"
results = await test_endpoint(url, REQUESTS)
# Wait a bit
await asyncio.sleep(1)
# Stop monitoring
stop_monitoring.set()
if monitor_thread:
monitor_thread.join(timeout=2)
# Analyze logs for pool markers
print(f"\n📋 Analyzing pool usage...")
pool_stats = count_log_markers(container)
# Calculate request stats
successes = sum(1 for r in results if r.get("success"))
success_rate = (successes / len(results)) * 100
latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
avg_latency = sum(latencies) / len(latencies) if latencies else 0
# Memory stats
memory_samples = [s['memory_mb'] for s in stats_history]
peak_mem = max(memory_samples) if memory_samples else 0
final_mem = memory_samples[-1] if memory_samples else 0
mem_delta = final_mem - baseline_mem
# Calculate reuse rate
total_requests = len(results)
total_pool_hits = pool_stats['total_hits']
reuse_rate = (total_pool_hits / total_requests * 100) if total_requests > 0 else 0
# Print results
print(f"\n{'='*60}")
print(f"RESULTS:")
print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(results)})")
print(f" Avg Latency: {avg_latency:.0f}ms")
print(f"\n Pool Stats:")
print(f" 🔥 Permanent Hits: {pool_stats['permanent_hits']}")
print(f" ♨️ Hot Pool Hits: {pool_stats['hot_hits']}")
print(f" ❄️ Cold Pool Hits: {pool_stats['cold_hits']}")
print(f" 🆕 New Created: {pool_stats['new_created']}")
print(f" 📊 Reuse Rate: {reuse_rate:.1f}%")
print(f"\n Memory Stats:")
print(f" Baseline: {baseline_mem:.1f} MB")
print(f" Peak: {peak_mem:.1f} MB")
print(f" Final: {final_mem:.1f} MB")
print(f" Delta: {mem_delta:+.1f} MB")
print(f"{'='*60}")
# Pass/Fail
passed = True
if success_rate < 100:
print(f"❌ FAIL: Success rate {success_rate:.1f}% < 100%")
passed = False
if reuse_rate < 80:
print(f"❌ FAIL: Reuse rate {reuse_rate:.1f}% < 80% (expected high permanent browser usage)")
passed = False
if pool_stats['permanent_hits'] < (total_requests * 0.8):
print(f"⚠️ WARNING: Only {pool_stats['permanent_hits']} permanent hits out of {total_requests} requests")
if mem_delta > 200:
print(f"⚠️ WARNING: Memory grew by {mem_delta:.1f} MB (possible browser leak)")
if passed:
print(f"✅ TEST PASSED")
return 0
else:
return 1
except Exception as e:
print(f"\n❌ TEST ERROR: {e}")
import traceback
traceback.print_exc()
return 1
finally:
stop_monitoring.set()
if container:
stop_container(container)
if __name__ == "__main__":
exit_code = asyncio.run(main())
exit(exit_code)
deploy/docker/tests/test_4_concurrent.py
Executable
+236
View File
@@ -0,0 +1,236 @@
#!/usr/bin/env python3
"""
Test 4: Concurrent Load Testing
- Tests pool under concurrent load
- Escalates: 10 → 50 → 100 concurrent requests
- Validates latency distribution (P50, P95, P99)
- Monitors memory stability
"""
import asyncio
import time
import docker
import httpx
from threading import Thread, Event
from collections import defaultdict
# Config
IMAGE = "crawl4ai-local:latest"
CONTAINER_NAME = "crawl4ai-test"
PORT = 11235
LOAD_LEVELS = [
{"name": "Light", "concurrent": 10, "requests": 20},
{"name": "Medium", "concurrent": 50, "requests": 100},
{"name": "Heavy", "concurrent": 100, "requests": 200},
]
# Stats
stats_history = []
stop_monitoring = Event()
def monitor_stats(container):
"""Background stats collector."""
for stat in container.stats(decode=True, stream=True):
if stop_monitoring.is_set():
break
try:
mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
except:
pass
time.sleep(0.5)
def count_log_markers(container):
"""Extract pool markers."""
logs = container.logs().decode('utf-8')
return {
'permanent': logs.count("🔥 Using permanent browser"),
'hot': logs.count("♨️ Using hot pool browser"),
'cold': logs.count("❄️ Using cold pool browser"),
'new': logs.count("🆕 Creating new browser"),
}
async def hit_endpoint(client, url, payload, semaphore):
"""Single request with concurrency control."""
async with semaphore:
start = time.time()
try:
resp = await client.post(url, json=payload, timeout=60.0)
elapsed = (time.time() - start) * 1000
return {"success": resp.status_code == 200, "latency_ms": elapsed}
except Exception as e:
return {"success": False, "error": str(e)}
async def run_concurrent_test(url, payload, concurrent, total_requests):
"""Run concurrent requests."""
semaphore = asyncio.Semaphore(concurrent)
async with httpx.AsyncClient() as client:
tasks = [hit_endpoint(client, url, payload, semaphore) for _ in range(total_requests)]
results = await asyncio.gather(*tasks)
return results
def calculate_percentiles(latencies):
"""Calculate P50, P95, P99."""
if not latencies:
return 0, 0, 0
sorted_lat = sorted(latencies)
n = len(sorted_lat)
return (
sorted_lat[int(n * 0.50)],
sorted_lat[int(n * 0.95)],
sorted_lat[int(n * 0.99)],
)
def start_container(client, image, name, port):
"""Start container."""
try:
old = client.containers.get(name)
print(f"🧹 Stopping existing container...")
old.stop()
old.remove()
except docker.errors.NotFound:
pass
print(f"🚀 Starting container...")
container = client.containers.run(
image, name=name, ports={f"{port}/tcp": port},
detach=True, shm_size="1g", mem_limit="4g",
)
print(f"⏳ Waiting for health...")
for _ in range(30):
time.sleep(1)
container.reload()
if container.status == "running":
try:
import requests
if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
print(f"✅ Container healthy!")
return container
except:
pass
raise TimeoutError("Container failed to start")
async def main():
print("="*60)
print("TEST 4: Concurrent Load Testing")
print("="*60)
client = docker.from_env()
container = None
monitor_thread = None
try:
container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
print(f"\n⏳ Waiting for permanent browser init (3s)...")
await asyncio.sleep(3)
# Start monitoring
stop_monitoring.clear()
stats_history.clear()
monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
monitor_thread.start()
await asyncio.sleep(1)
baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
url = f"http://localhost:{PORT}/html"
payload = {"url": "https://httpbin.org/html"}
all_results = []
level_stats = []
# Run load levels
for level in LOAD_LEVELS:
print(f"{'='*60}")
print(f"🔄 {level['name']} Load: {level['concurrent']} concurrent, {level['requests']} total")
print(f"{'='*60}")
start_time = time.time()
results = await run_concurrent_test(url, payload, level['concurrent'], level['requests'])
duration = time.time() - start_time
successes = sum(1 for r in results if r.get("success"))
success_rate = (successes / len(results)) * 100
latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
p50, p95, p99 = calculate_percentiles(latencies)
avg_lat = sum(latencies) / len(latencies) if latencies else 0
print(f" Duration: {duration:.1f}s")
print(f" Success: {success_rate:.1f}% ({successes}/{len(results)})")
print(f" Avg Latency: {avg_lat:.0f}ms")
print(f" P50/P95/P99: {p50:.0f}ms / {p95:.0f}ms / {p99:.0f}ms")
level_stats.append({
'name': level['name'],
'concurrent': level['concurrent'],
'success_rate': success_rate,
'avg_latency': avg_lat,
'p50': p50, 'p95': p95, 'p99': p99,
})
all_results.extend(results)
await asyncio.sleep(2) # Cool down between levels
# Stop monitoring
await asyncio.sleep(1)
stop_monitoring.set()
if monitor_thread:
monitor_thread.join(timeout=2)
# Final stats
pool_stats = count_log_markers(container)
memory_samples = [s['memory_mb'] for s in stats_history]
peak_mem = max(memory_samples) if memory_samples else 0
final_mem = memory_samples[-1] if memory_samples else 0
print(f"\n{'='*60}")
print(f"FINAL RESULTS:")
print(f"{'='*60}")
print(f" Total Requests: {len(all_results)}")
print(f"\n Pool Utilization:")
print(f" 🔥 Permanent: {pool_stats['permanent']}")
print(f" ♨️ Hot: {pool_stats['hot']}")
print(f" ❄️ Cold: {pool_stats['cold']}")
print(f" 🆕 New: {pool_stats['new']}")
print(f"\n Memory:")
print(f" Baseline: {baseline_mem:.1f} MB")
print(f" Peak: {peak_mem:.1f} MB")
print(f" Final: {final_mem:.1f} MB")
print(f" Delta: {final_mem - baseline_mem:+.1f} MB")
print(f"{'='*60}")
# Pass/Fail
passed = True
for ls in level_stats:
if ls['success_rate'] < 99:
print(f"❌ FAIL: {ls['name']} success rate {ls['success_rate']:.1f}% < 99%")
passed = False
if ls['p99'] > 10000: # 10s threshold
print(f"⚠️ WARNING: {ls['name']} P99 latency {ls['p99']:.0f}ms very high")
if final_mem - baseline_mem > 300:
print(f"⚠️ WARNING: Memory grew {final_mem - baseline_mem:.1f} MB")
if passed:
print(f"✅ TEST PASSED")
return 0
else:
return 1
except Exception as e:
print(f"\n❌ TEST ERROR: {e}")
import traceback
traceback.print_exc()
return 1
finally:
stop_monitoring.set()
if container:
print(f"🛑 Stopping container...")
container.stop()
container.remove()
if __name__ == "__main__":
exit_code = asyncio.run(main())
exit(exit_code)
deploy/docker/tests/test_5_pool_stress.py
Executable
+267
View File
@@ -0,0 +1,267 @@
#!/usr/bin/env python3
"""
Test 5: Pool Stress - Mixed Configs
- Tests hot/cold pool with different browser configs
- Uses different viewports to create config variants
- Validates cold → hot promotion after 3 uses
- Monitors pool tier distribution
"""
import asyncio
import time
import docker
import httpx
from threading import Thread, Event
import random
# Config
IMAGE = "crawl4ai-local:latest"
CONTAINER_NAME = "crawl4ai-test"
PORT = 11235
REQUESTS_PER_CONFIG = 5 # 5 requests per config variant
# Different viewport configs to test pool tiers
VIEWPORT_CONFIGS = [
None, # Default (permanent browser)
{"width": 1920, "height": 1080}, # Desktop
{"width": 1024, "height": 768}, # Tablet
{"width": 375, "height": 667}, # Mobile
]
# Stats
stats_history = []
stop_monitoring = Event()
def monitor_stats(container):
"""Background stats collector."""
for stat in container.stats(decode=True, stream=True):
if stop_monitoring.is_set():
break
try:
mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
except:
pass
time.sleep(0.5)
def analyze_pool_logs(container):
"""Extract detailed pool stats from logs."""
logs = container.logs().decode('utf-8')
permanent = logs.count("🔥 Using permanent browser")
hot = logs.count("♨️ Using hot pool browser")
cold = logs.count("❄️ Using cold pool browser")
new = logs.count("🆕 Creating new browser")
promotions = logs.count("⬆️ Promoting to hot pool")
return {
'permanent': permanent,
'hot': hot,
'cold': cold,
'new': new,
'promotions': promotions,
'total': permanent + hot + cold
}
async def crawl_with_viewport(client, url, viewport):
"""Single request with specific viewport."""
payload = {
"urls": ["https://httpbin.org/html"],
"browser_config": {},
"crawler_config": {}
}
# Add viewport if specified
if viewport:
payload["browser_config"] = {
"type": "BrowserConfig",
"params": {
"viewport": {"type": "dict", "value": viewport},
"headless": True,
"text_mode": True,
"extra_args": [
"--no-sandbox",
"--disable-dev-shm-usage",
"--disable-gpu",
"--disable-software-rasterizer",
"--disable-web-security",
"--allow-insecure-localhost",
"--ignore-certificate-errors"
]
}
}
start = time.time()
try:
resp = await client.post(url, json=payload, timeout=60.0)
elapsed = (time.time() - start) * 1000
return {"success": resp.status_code == 200, "latency_ms": elapsed, "viewport": viewport}
except Exception as e:
return {"success": False, "error": str(e), "viewport": viewport}
def start_container(client, image, name, port):
"""Start container."""
try:
old = client.containers.get(name)
print(f"🧹 Stopping existing container...")
old.stop()
old.remove()
except docker.errors.NotFound:
pass
print(f"🚀 Starting container...")
container = client.containers.run(
image, name=name, ports={f"{port}/tcp": port},
detach=True, shm_size="1g", mem_limit="4g",
)
print(f"⏳ Waiting for health...")
for _ in range(30):
time.sleep(1)
container.reload()
if container.status == "running":
try:
import requests
if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
print(f"✅ Container healthy!")
return container
except:
pass
raise TimeoutError("Container failed to start")
async def main():
print("="*60)
print("TEST 5: Pool Stress - Mixed Configs")
print("="*60)
client = docker.from_env()
container = None
monitor_thread = None
try:
container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
print(f"\n⏳ Waiting for permanent browser init (3s)...")
await asyncio.sleep(3)
# Start monitoring
stop_monitoring.clear()
stats_history.clear()
monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
monitor_thread.start()
await asyncio.sleep(1)
baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
url = f"http://localhost:{PORT}/crawl"
print(f"Testing {len(VIEWPORT_CONFIGS)} different configs:")
for i, vp in enumerate(VIEWPORT_CONFIGS):
vp_str = "Default" if vp is None else f"{vp['width']}x{vp['height']}"
print(f" {i+1}. {vp_str}")
print()
# Run requests: repeat each config REQUESTS_PER_CONFIG times
all_results = []
config_sequence = []
for _ in range(REQUESTS_PER_CONFIG):
for viewport in VIEWPORT_CONFIGS:
config_sequence.append(viewport)
# Shuffle to mix configs
random.shuffle(config_sequence)
print(f"🔄 Running {len(config_sequence)} requests with mixed configs...")
async with httpx.AsyncClient() as http_client:
for i, viewport in enumerate(config_sequence):
result = await crawl_with_viewport(http_client, url, viewport)
all_results.append(result)
if (i + 1) % 5 == 0:
vp_str = "default" if result['viewport'] is None else f"{result['viewport']['width']}x{result['viewport']['height']}"
status = "✓" if result.get('success') else "✗"
lat = f"{result.get('latency_ms', 0):.0f}ms" if 'latency_ms' in result else "error"
print(f" [{i+1}/{len(config_sequence)}] {status} {vp_str} - {lat}")
# Stop monitoring
await asyncio.sleep(2)
stop_monitoring.set()
if monitor_thread:
monitor_thread.join(timeout=2)
# Analyze results
pool_stats = analyze_pool_logs(container)
successes = sum(1 for r in all_results if r.get("success"))
success_rate = (successes / len(all_results)) * 100
latencies = [r["latency_ms"] for r in all_results if "latency_ms" in r]
avg_lat = sum(latencies) / len(latencies) if latencies else 0
memory_samples = [s['memory_mb'] for s in stats_history]
peak_mem = max(memory_samples) if memory_samples else 0
final_mem = memory_samples[-1] if memory_samples else 0
print(f"\n{'='*60}")
print(f"RESULTS:")
print(f"{'='*60}")
print(f" Requests: {len(all_results)}")
print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(all_results)})")
print(f" Avg Latency: {avg_lat:.0f}ms")
print(f"\n Pool Statistics:")
print(f" 🔥 Permanent: {pool_stats['permanent']}")
print(f" ♨️ Hot: {pool_stats['hot']}")
print(f" ❄️ Cold: {pool_stats['cold']}")
print(f" 🆕 New: {pool_stats['new']}")
print(f" ⬆️ Promotions: {pool_stats['promotions']}")
print(f" 📊 Reuse: {(pool_stats['total'] / len(all_results) * 100):.1f}%")
print(f"\n Memory:")
print(f" Baseline: {baseline_mem:.1f} MB")
print(f" Peak: {peak_mem:.1f} MB")
print(f" Final: {final_mem:.1f} MB")
print(f" Delta: {final_mem - baseline_mem:+.1f} MB")
print(f"{'='*60}")
# Pass/Fail
passed = True
if success_rate < 99:
print(f"❌ FAIL: Success rate {success_rate:.1f}% < 99%")
passed = False
# Should see promotions since we repeat each config 5 times
if pool_stats['promotions'] < (len(VIEWPORT_CONFIGS) - 1): # -1 for default
print(f"⚠️ WARNING: Only {pool_stats['promotions']} promotions (expected ~{len(VIEWPORT_CONFIGS)-1})")
# Should have created some browsers for different configs
if pool_stats['new'] == 0:
print(f"⚠️ NOTE: No new browsers created (all used default?)")
if pool_stats['permanent'] == len(all_results):
print(f"⚠️ NOTE: All requests used permanent browser (configs not varying enough?)")
if final_mem - baseline_mem > 500:
print(f"⚠️ WARNING: Memory grew {final_mem - baseline_mem:.1f} MB")
if passed:
print(f"✅ TEST PASSED")
return 0
else:
return 1
except Exception as e:
print(f"\n❌ TEST ERROR: {e}")
import traceback
traceback.print_exc()
return 1
finally:
stop_monitoring.set()
if container:
print(f"🛑 Stopping container...")
container.stop()
container.remove()
if __name__ == "__main__":
exit_code = asyncio.run(main())
exit(exit_code)
deploy/docker/tests/test_6_multi_endpoint.py
Executable
+234
View File
@@ -0,0 +1,234 @@
#!/usr/bin/env python3
"""
Test 6: Multi-Endpoint Testing
- Tests multiple endpoints together: /html, /screenshot, /pdf, /crawl
- Validates each endpoint works correctly
- Monitors success rates per endpoint
"""
import asyncio
import time
import docker
import httpx
from threading import Thread, Event
# Config
IMAGE = "crawl4ai-local:latest"
CONTAINER_NAME = "crawl4ai-test"
PORT = 11235
REQUESTS_PER_ENDPOINT = 10
# Stats
stats_history = []
stop_monitoring = Event()
def monitor_stats(container):
"""Background stats collector."""
for stat in container.stats(decode=True, stream=True):
if stop_monitoring.is_set():
break
try:
mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
except:
pass
time.sleep(0.5)
async def test_html(client, base_url, count):
"""Test /html endpoint."""
url = f"{base_url}/html"
results = []
for _ in range(count):
start = time.time()
try:
resp = await client.post(url, json={"url": "https://httpbin.org/html"}, timeout=30.0)
elapsed = (time.time() - start) * 1000
results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
except Exception as e:
results.append({"success": False, "error": str(e)})
return results
async def test_screenshot(client, base_url, count):
"""Test /screenshot endpoint."""
url = f"{base_url}/screenshot"
results = []
for _ in range(count):
start = time.time()
try:
resp = await client.post(url, json={"url": "https://httpbin.org/html"}, timeout=30.0)
elapsed = (time.time() - start) * 1000
results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
except Exception as e:
results.append({"success": False, "error": str(e)})
return results
async def test_pdf(client, base_url, count):
"""Test /pdf endpoint."""
url = f"{base_url}/pdf"
results = []
for _ in range(count):
start = time.time()
try:
resp = await client.post(url, json={"url": "https://httpbin.org/html"}, timeout=30.0)
elapsed = (time.time() - start) * 1000
results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
except Exception as e:
results.append({"success": False, "error": str(e)})
return results
async def test_crawl(client, base_url, count):
"""Test /crawl endpoint."""
url = f"{base_url}/crawl"
results = []
payload = {
"urls": ["https://httpbin.org/html"],
"browser_config": {},
"crawler_config": {}
}
for _ in range(count):
start = time.time()
try:
resp = await client.post(url, json=payload, timeout=30.0)
elapsed = (time.time() - start) * 1000
results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
except Exception as e:
results.append({"success": False, "error": str(e)})
return results
def start_container(client, image, name, port):
"""Start container."""
try:
old = client.containers.get(name)
print(f"🧹 Stopping existing container...")
old.stop()
old.remove()
except docker.errors.NotFound:
pass
print(f"🚀 Starting container...")
container = client.containers.run(
image, name=name, ports={f"{port}/tcp": port},
detach=True, shm_size="1g", mem_limit="4g",
)
print(f"⏳ Waiting for health...")
for _ in range(30):
time.sleep(1)
container.reload()
if container.status == "running":
try:
import requests
if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
print(f"✅ Container healthy!")
return container
except:
pass
raise TimeoutError("Container failed to start")
async def main():
print("="*60)
print("TEST 6: Multi-Endpoint Testing")
print("="*60)
client = docker.from_env()
container = None
monitor_thread = None
try:
container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
print(f"\n⏳ Waiting for permanent browser init (3s)...")
await asyncio.sleep(3)
# Start monitoring
stop_monitoring.clear()
stats_history.clear()
monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
monitor_thread.start()
await asyncio.sleep(1)
baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
base_url = f"http://localhost:{PORT}"
# Test each endpoint
endpoints = {
"/html": test_html,
"/screenshot": test_screenshot,
"/pdf": test_pdf,
"/crawl": test_crawl,
}
all_endpoint_stats = {}
async with httpx.AsyncClient() as http_client:
for endpoint_name, test_func in endpoints.items():
print(f"🔄 Testing {endpoint_name} ({REQUESTS_PER_ENDPOINT} requests)...")
results = await test_func(http_client, base_url, REQUESTS_PER_ENDPOINT)
successes = sum(1 for r in results if r.get("success"))
success_rate = (successes / len(results)) * 100
latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
avg_lat = sum(latencies) / len(latencies) if latencies else 0
all_endpoint_stats[endpoint_name] = {
'success_rate': success_rate,
'avg_latency': avg_lat,
'total': len(results),
'successes': successes
}
print(f" ✓ Success: {success_rate:.1f}% ({successes}/{len(results)}), Avg: {avg_lat:.0f}ms")
# Stop monitoring
await asyncio.sleep(1)
stop_monitoring.set()
if monitor_thread:
monitor_thread.join(timeout=2)
# Final stats
memory_samples = [s['memory_mb'] for s in stats_history]
peak_mem = max(memory_samples) if memory_samples else 0
final_mem = memory_samples[-1] if memory_samples else 0
print(f"\n{'='*60}")
print(f"RESULTS:")
print(f"{'='*60}")
for endpoint, stats in all_endpoint_stats.items():
print(f" {endpoint:12} Success: {stats['success_rate']:5.1f}% Avg: {stats['avg_latency']:6.0f}ms")
print(f"\n Memory:")
print(f" Baseline: {baseline_mem:.1f} MB")
print(f" Peak: {peak_mem:.1f} MB")
print(f" Final: {final_mem:.1f} MB")
print(f" Delta: {final_mem - baseline_mem:+.1f} MB")
print(f"{'='*60}")
# Pass/Fail
passed = True
for endpoint, stats in all_endpoint_stats.items():
if stats['success_rate'] < 100:
print(f"❌ FAIL: {endpoint} success rate {stats['success_rate']:.1f}% < 100%")
passed = False
if passed:
print(f"✅ TEST PASSED")
return 0
else:
return 1
except Exception as e:
print(f"\n❌ TEST ERROR: {e}")
import traceback
traceback.print_exc()
return 1
finally:
stop_monitoring.set()
if container:
print(f"🛑 Stopping container...")
container.stop()
container.remove()
if __name__ == "__main__":
exit_code = asyncio.run(main())
exit(exit_code)
deploy/docker/tests/test_7_cleanup.py
Executable
+199
View File
@@ -0,0 +1,199 @@
#!/usr/bin/env python3
"""
Test 7: Cleanup Verification (Janitor)
- Creates load spike then goes idle
- Verifies memory returns to near baseline
- Tests janitor cleanup of idle browsers
- Monitors memory recovery time
"""
import asyncio
import time
import docker
import httpx
from threading import Thread, Event
# Config
IMAGE = "crawl4ai-local:latest"
CONTAINER_NAME = "crawl4ai-test"
PORT = 11235
SPIKE_REQUESTS = 20 # Create some browsers
IDLE_TIME = 90 # Wait 90s for janitor (runs every 60s)
# Stats
stats_history = []
stop_monitoring = Event()
def monitor_stats(container):
"""Background stats collector."""
for stat in container.stats(decode=True, stream=True):
if stop_monitoring.is_set():
break
try:
mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
except:
pass
time.sleep(1) # Sample every 1s for this test
def start_container(client, image, name, port):
"""Start container."""
try:
old = client.containers.get(name)
print(f"🧹 Stopping existing container...")
old.stop()
old.remove()
except docker.errors.NotFound:
pass
print(f"🚀 Starting container...")
container = client.containers.run(
image, name=name, ports={f"{port}/tcp": port},
detach=True, shm_size="1g", mem_limit="4g",
)
print(f"⏳ Waiting for health...")
for _ in range(30):
time.sleep(1)
container.reload()
if container.status == "running":
try:
import requests
if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
print(f"✅ Container healthy!")
return container
except:
pass
raise TimeoutError("Container failed to start")
async def main():
print("="*60)
print("TEST 7: Cleanup Verification (Janitor)")
print("="*60)
client = docker.from_env()
container = None
monitor_thread = None
try:
container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
print(f"\n⏳ Waiting for permanent browser init (3s)...")
await asyncio.sleep(3)
# Start monitoring
stop_monitoring.clear()
stats_history.clear()
monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
monitor_thread.start()
await asyncio.sleep(2)
baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
# Create load spike with different configs to populate pool
print(f"🔥 Creating load spike ({SPIKE_REQUESTS} requests with varied configs)...")
url = f"http://localhost:{PORT}/crawl"
viewports = [
{"width": 1920, "height": 1080},
{"width": 1024, "height": 768},
{"width": 375, "height": 667},
]
async with httpx.AsyncClient(timeout=60.0) as http_client:
tasks = []
for i in range(SPIKE_REQUESTS):
vp = viewports[i % len(viewports)]
payload = {
"urls": ["https://httpbin.org/html"],
"browser_config": {
"type": "BrowserConfig",
"params": {
"viewport": {"type": "dict", "value": vp},
"headless": True,
"text_mode": True,
"extra_args": [
"--no-sandbox", "--disable-dev-shm-usage",
"--disable-gpu", "--disable-software-rasterizer",
"--disable-web-security", "--allow-insecure-localhost",
"--ignore-certificate-errors"
]
}
},
"crawler_config": {}
}
tasks.append(http_client.post(url, json=payload))
results = await asyncio.gather(*tasks, return_exceptions=True)
successes = sum(1 for r in results if hasattr(r, 'status_code') and r.status_code == 200)
print(f" ✓ Spike completed: {successes}/{len(results)} successful")
# Measure peak
await asyncio.sleep(2)
peak_mem = max([s['memory_mb'] for s in stats_history]) if stats_history else baseline_mem
print(f" 📊 Peak memory: {peak_mem:.1f} MB (+{peak_mem - baseline_mem:.1f} MB)")
# Now go idle and wait for janitor
print(f"\n⏸️ Going idle for {IDLE_TIME}s (janitor cleanup)...")
print(f" (Janitor runs every 60s, checking for idle browsers)")
for elapsed in range(0, IDLE_TIME, 10):
await asyncio.sleep(10)
current_mem = stats_history[-1]['memory_mb'] if stats_history else 0
print(f" [{elapsed+10:3d}s] Memory: {current_mem:.1f} MB")
# Stop monitoring
stop_monitoring.set()
if monitor_thread:
monitor_thread.join(timeout=2)
# Analyze memory recovery
final_mem = stats_history[-1]['memory_mb'] if stats_history else 0
recovery_mb = peak_mem - final_mem
recovery_pct = (recovery_mb / (peak_mem - baseline_mem) * 100) if (peak_mem - baseline_mem) > 0 else 0
print(f"\n{'='*60}")
print(f"RESULTS:")
print(f"{'='*60}")
print(f" Memory Journey:")
print(f" Baseline: {baseline_mem:.1f} MB")
print(f" Peak: {peak_mem:.1f} MB (+{peak_mem - baseline_mem:.1f} MB)")
print(f" Final: {final_mem:.1f} MB (+{final_mem - baseline_mem:.1f} MB)")
print(f" Recovered: {recovery_mb:.1f} MB ({recovery_pct:.1f}%)")
print(f"{'='*60}")
# Pass/Fail
passed = True
# Should have created some memory pressure
if peak_mem - baseline_mem < 100:
print(f"⚠️ WARNING: Peak increase only {peak_mem - baseline_mem:.1f} MB (expected more browsers)")
# Should recover most memory (within 100MB of baseline)
if final_mem - baseline_mem > 100:
print(f"⚠️ WARNING: Memory didn't recover well (still +{final_mem - baseline_mem:.1f} MB above baseline)")
else:
print(f"✅ Good memory recovery!")
# Baseline + 50MB tolerance
if final_mem - baseline_mem < 50:
print(f"✅ Excellent cleanup (within 50MB of baseline)")
print(f"✅ TEST PASSED")
return 0
except Exception as e:
print(f"\n❌ TEST ERROR: {e}")
import traceback
traceback.print_exc()
return 1
finally:
stop_monitoring.set()
if container:
print(f"🛑 Stopping container...")
container.stop()
container.remove()
if __name__ == "__main__":
exit_code = asyncio.run(main())
exit(exit_code)
deploy/docker/tests/test_monitor_demo.py
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
"""Quick test to generate monitor dashboard activity"""
import httpx
import asyncio
async def test_dashboard():
async with httpx.AsyncClient(timeout=30.0) as client:
print("📊 Generating dashboard activity...")
# Test 1: Simple crawl
print("\n1️⃣ Running simple crawl...")
r1 = await client.post(
"http://localhost:11235/crawl",
json={"urls": ["https://httpbin.org/html"], "crawler_config": {}}
)
print(f" Status: {r1.status_code}")
# Test 2: Multiple URLs
print("\n2️⃣ Running multi-URL crawl...")
r2 = await client.post(
"http://localhost:11235/crawl",
json={
"urls": [
"https://httpbin.org/html",
"https://httpbin.org/json"
],
"crawler_config": {}
}
)
print(f" Status: {r2.status_code}")
# Test 3: Check monitor health
print("\n3️⃣ Checking monitor health...")
r3 = await client.get("http://localhost:11235/monitor/health")
health = r3.json()
print(f" Memory: {health['container']['memory_percent']}%")
print(f" Browsers: {health['pool']['permanent']['active']}")
# Test 4: Check requests
print("\n4️⃣ Checking request log...")
r4 = await client.get("http://localhost:11235/monitor/requests")
reqs = r4.json()
print(f" Active: {len(reqs['active'])}")
print(f" Completed: {len(reqs['completed'])}")
# Test 5: Check endpoint stats
print("\n5️⃣ Checking endpoint stats...")
r5 = await client.get("http://localhost:11235/monitor/endpoints/stats")
stats = r5.json()
for endpoint, data in stats.items():
print(f" {endpoint}: {data['count']} requests, {data['avg_latency_ms']}ms avg")
print("\n✅ Dashboard should now show activity!")
print(f"\n🌐 Open: http://localhost:11235/dashboard")
if __name__ == "__main__":
asyncio.run(test_dashboard())
deploy/docker/tests/test_security_2026_04.py
+312
View File
@@ -0,0 +1,312 @@
#!/usr/bin/env python3
"""
Adversarial security tests for the 4 vulns reported 2026-04-13.
Self-contained: copies validation logic to avoid import issues with dns.resolver etc.
Vuln 1: Arbitrary File Write via /screenshot + /pdf output_path
Vuln 2: Monitor Auth Bypass (structural source check)
Vuln 3: Stored XSS in Monitor Dashboard (server-side + client-side)
Vuln 4: SSRF via Webhook URL
"""
import os
import sys
import unittest
import ipaddress
import socket
from urllib.parse import urlparse
# ============================================================================
# Local copies of security utilities (to avoid dns.resolver import in utils.py)
# ============================================================================
ALLOWED_OUTPUT_DIR = os.environ.get("CRAWL4AI_OUTPUT_DIR", "/tmp/crawl4ai-outputs")
def validate_output_path(user_path):
safe_path = os.path.normpath(user_path).lstrip(os.sep)
abs_path = os.path.abspath(os.path.join(ALLOWED_OUTPUT_DIR, safe_path))
abs_allowed = os.path.abspath(ALLOWED_OUTPUT_DIR) + os.sep
if not abs_path.startswith(abs_allowed):
raise ValueError(f"output_path must resolve within {ALLOWED_OUTPUT_DIR}")
return abs_path
_BLOCKED_NETWORKS = [
ipaddress.ip_network("0.0.0.0/8"),
ipaddress.ip_network("10.0.0.0/8"),
ipaddress.ip_network("100.64.0.0/10"),
ipaddress.ip_network("127.0.0.0/8"),
ipaddress.ip_network("169.254.0.0/16"),
ipaddress.ip_network("172.16.0.0/12"),
ipaddress.ip_network("192.0.0.0/24"),
ipaddress.ip_network("192.168.0.0/16"),
ipaddress.ip_network("198.18.0.0/15"),
ipaddress.ip_network("::1/128"),
ipaddress.ip_network("fc00::/7"),
ipaddress.ip_network("fe80::/10"),
]
_BLOCKED_HOSTNAMES = {
"localhost", "metadata.google.internal", "metadata",
"kubernetes.default", "kubernetes.default.svc",
}
def validate_webhook_url(url):
parsed = urlparse(str(url))
hostname = parsed.hostname
if not hostname:
raise ValueError("Webhook URL must have a valid hostname")
hostname_lower = hostname.lower()
if hostname_lower in _BLOCKED_HOSTNAMES:
raise ValueError(f"Webhook URL hostname '{hostname}' is blocked")
if hostname_lower.startswith("host.docker.internal"):
raise ValueError(f"Webhook URL hostname '{hostname}' is blocked")
try:
resolved = socket.getaddrinfo(hostname, None)
except socket.gaierror:
raise ValueError(f"Cannot resolve webhook hostname '{hostname}'")
for _, _, _, _, sockaddr in resolved:
ip = ipaddress.ip_address(sockaddr[0])
for network in _BLOCKED_NETWORKS:
if ip in network:
raise ValueError(f"Webhook URL resolves to blocked address: {ip}")
DEPLOY_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
# ============================================================================
# VULN 1: Arbitrary File Write - Path Traversal
# ============================================================================
class TestOutputPathRemoved(unittest.TestCase):
"""output_path is gone; the server owns paths via the artifact store.
The old string-only validate_output_path was bypassable (symlink/TOCTOU,
sibling-prefix '...-evil') -> arbitrary write -> RCE. The fix is to never
accept a caller path at all. Behavioral artifact-store coverage (O_NOFOLLOW,
O_EXCL, hex id, TTL, quota) lives in test_security_artifact_store.py.
"""
def test_screenshot_request_has_no_output_path(self):
sys.path.insert(0, DEPLOY_DIR)
from schemas import ScreenshotRequest
self.assertNotIn("output_path", ScreenshotRequest.model_fields)
def test_pdf_request_has_no_output_path(self):
sys.path.insert(0, DEPLOY_DIR)
from schemas import PDFRequest
self.assertNotIn("output_path", PDFRequest.model_fields)
def test_validate_output_path_deleted(self):
sys.path.insert(0, DEPLOY_DIR)
import utils
self.assertFalse(
hasattr(utils, "validate_output_path"),
"validate_output_path must be deleted (caller paths no longer accepted)",
)
class TestPydanticPathValidator(unittest.TestCase):
"""output_path (and its traversal validator) are gone entirely.
Traversal rejection used to be the mitigation; the real fix is that no
caller path is accepted at all, so there is nothing to traverse. The
sandboxed artifact store owns all paths now.
"""
def test_no_output_path_field_on_request_models(self):
sys.path.insert(0, DEPLOY_DIR)
from schemas import ScreenshotRequest, PDFRequest
self.assertNotIn("output_path", ScreenshotRequest.model_fields)
self.assertNotIn("output_path", PDFRequest.model_fields)
def test_traversal_validator_removed(self):
# No reject_traversal validator should remain registered on the models.
sys.path.insert(0, DEPLOY_DIR)
from schemas import ScreenshotRequest, PDFRequest
for model in (ScreenshotRequest, PDFRequest):
names = set(getattr(model, "__pydantic_decorators__").field_validators)
self.assertNotIn("reject_traversal", names)
# ============================================================================
# VULN 2: Monitor Auth Bypass (structural check)
# ============================================================================
class TestMonitorAuthStructural(unittest.TestCase):
def test_monitor_router_has_auth(self):
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
source = f.read()
# Find the line with monitor_router
for line in source.splitlines():
if "monitor_router" in line and "include_router" in line:
self.assertIn("dependencies=", line,
"Monitor router must have dependencies=[Depends(token_dep)]")
return
self.fail("Could not find monitor_router include_router line")
def test_websocket_auth_enforced_by_gate(self):
"""The monitor WS is gated by the AuthGateMiddleware (behavioral).
Auth moved out of the route's bespoke, timing-unsafe env-token check
and into the outermost ASGI gate, which closes an unauthenticated
WebSocket before it is accepted.
"""
import sys
sys.path.insert(0, DEPLOY_DIR)
import server
from starlette.testclient import TestClient
client = TestClient(server.app)
# Unauthenticated connect must be rejected by the gate.
with self.assertRaises(Exception):
with client.websocket_connect("/monitor/ws") as ws:
ws.receive_text()
# The old in-route check must be gone (it read a different secret than
# the REST path and compared with a timing-unsafe `!=`).
with open(os.path.join(DEPLOY_DIR, "monitor_routes.py")) as f:
source = f.read()
self.assertNotIn('os.environ.get("CRAWL4AI_API_TOKEN"', source,
"bespoke WS token check must be removed; the gate owns WS auth")
# ============================================================================
# VULN 3: Stored XSS (server-side + client-side)
# ============================================================================
class TestXSSPrevention(unittest.TestCase):
def test_html_escape_blocks_script_tags(self):
import html
payload = '<script>alert(document.cookie)</script>'
escaped = html.escape(payload)
self.assertNotIn("<script>", escaped)
self.assertIn("&lt;script&gt;", escaped)
def test_html_escape_blocks_img_tag(self):
import html
payload = '"><img src=x onerror=alert(1)>'
escaped = html.escape(payload)
self.assertNotIn("<img", escaped)
self.assertIn("&lt;img", escaped)
def test_monitor_py_uses_html_escape(self):
with open(os.path.join(DEPLOY_DIR, "monitor.py")) as f:
source = f.read()
self.assertIn("import html", source)
self.assertIn("html.escape(url", source)
self.assertIn("html.escape(str(error)", source)
def test_index_html_has_escape_function(self):
html_path = os.path.join(DEPLOY_DIR, "static", "monitor", "index.html")
with open(html_path) as f:
source = f.read()
self.assertIn("function escapeHtml(", source)
def test_index_html_no_raw_url_injection(self):
html_path = os.path.join(DEPLOY_DIR, "static", "monitor", "index.html")
with open(html_path) as f:
source = f.read()
self.assertNotIn("${req.url}", source, "Raw ${req.url} found")
self.assertNotIn("${err.url}", source, "Raw ${err.url} found")
self.assertNotIn("${err.error}", source, "Raw ${err.error} found")
# ============================================================================
# VULN 4: SSRF via Webhook URL
# ============================================================================
class TestSSRFBlocked(unittest.TestCase):
def test_localhost_ip(self):
with self.assertRaises(ValueError):
validate_webhook_url("http://127.0.0.1:9999/steal")
def test_localhost_hostname(self):
with self.assertRaises(ValueError):
validate_webhook_url("http://localhost:9999/steal")
def test_loopback_127_x(self):
with self.assertRaises(ValueError):
validate_webhook_url("http://127.0.0.2:9999/steal")
def test_10_network(self):
with self.assertRaises(ValueError):
validate_webhook_url("http://10.0.0.1:8080/admin")
def test_172_16_network(self):
with self.assertRaises(ValueError):
validate_webhook_url("http://172.16.0.1:8080/admin")
def test_192_168_network(self):
with self.assertRaises(ValueError):
validate_webhook_url("http://192.168.1.1:8080/admin")
def test_aws_metadata_ip(self):
with self.assertRaises(ValueError):
validate_webhook_url("http://169.254.169.254/latest/meta-data/")
def test_gcp_metadata_hostname(self):
with self.assertRaises(ValueError):
validate_webhook_url("http://metadata.google.internal/computeMetadata/v1/")
def test_docker_internal(self):
with self.assertRaises(ValueError):
validate_webhook_url("http://host.docker.internal:9998/steal")
def test_kubernetes_default(self):
with self.assertRaises(ValueError):
validate_webhook_url("http://kubernetes.default/api/v1/secrets")
def test_empty_hostname(self):
with self.assertRaises(ValueError):
validate_webhook_url("http:///steal")
def test_valid_external_url(self):
validate_webhook_url("https://webhook.site/test-uuid")
validate_webhook_url("https://hooks.slack.com/services/T00/B00/xxx")
def test_valid_external_with_port(self):
"""External URL with port -- may fail DNS in CI, that's expected."""
try:
validate_webhook_url("https://google.com:443/webhook")
except ValueError as e:
if "Cannot resolve" in str(e):
self.skipTest("DNS resolution unavailable in this environment")
raise
class TestWebhookSourceChecks(unittest.TestCase):
def test_redirects_not_auto_followed(self):
# Redirects are followed manually and re-validated per hop, never
# auto-followed (which would skip the SSRF re-check).
with open(os.path.join(DEPLOY_DIR, "webhook.py")) as f:
source = f.read()
self.assertIn("allow_redirects=False", source)
self.assertIn("check_redirect", source) # each hop re-validated
def test_webhook_validates_at_send_time(self):
# Validation happens at send time via the egress broker (resolve_and_pin
# rejects non-global targets); the connection is also pinned to that IP.
with open(os.path.join(DEPLOY_DIR, "webhook.py")) as f:
source = f.read()
self.assertIn("resolve_and_pin", source)
def test_job_validates_webhook(self):
with open(os.path.join(DEPLOY_DIR, "job.py")) as f:
source = f.read()
self.assertIn("validate_webhook_url", source)
if __name__ == "__main__":
print("=" * 70)
print("Crawl4AI Security Tests - 2026-04-13 Vulnerabilities")
print("=" * 70)
print()
unittest.main(verbosity=2)
deploy/docker/tests/test_security_2026_04_b2.py
+247
View File
@@ -0,0 +1,247 @@
#!/usr/bin/env python3
"""
Adversarial security tests for Batch 2 vulns reported 2026-04-14 (by111/August829).
Self-contained tests that verify fixes at the code/source level.
B2-V1: /execute_js disabled by default + SSRF block
B2-V2: Hardcoded JWT secret removed
B2-V3: eval() in /config/dump replaced with JSON
B2-V4: Hook sandbox __builtins__ escape fixed
"""
import os
import sys
import ast
import unittest
import builtins
import types
from unittest import mock
DEPLOY_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
# ============================================================================
# B2-V2: Hardcoded JWT Secret
# ============================================================================
class TestJWTSecretHardened(unittest.TestCase):
"""Verify the hardcoded 'mysecret' default is gone from auth.py."""
def test_no_mysecret_as_default(self):
"""auth.py must not use 'mysecret' as a fallback default for SECRET_KEY."""
with open(os.path.join(DEPLOY_DIR, "auth.py")) as f:
source = f.read()
# The old dangerous pattern: os.environ.get("SECRET_KEY", "mysecret")
self.assertNotIn('get("SECRET_KEY", "mysecret")', source,
"auth.py must not use 'mysecret' as env var default")
def test_weak_secret_validation_exists(self):
"""auth.py must reject weak and too-short SECRET_KEY values (behavioral)."""
import importlib
import sys
sys.path.insert(0, DEPLOY_DIR)
auth = importlib.import_module("auth")
# A known-weak secret must be rejected.
with mock.patch.dict(os.environ, {"SECRET_KEY": "mysecret"}):
with self.assertRaises(RuntimeError):
auth.resolve_secret_key(required=True)
# A too-short secret (< 32 chars) must be rejected.
with mock.patch.dict(os.environ, {"SECRET_KEY": "short"}):
with self.assertRaises(RuntimeError):
auth.resolve_secret_key(required=True)
# A strong secret must be accepted and returned unchanged.
strong = "a" * 32
with mock.patch.dict(os.environ, {"SECRET_KEY": strong}):
self.assertEqual(auth.resolve_secret_key(required=True), strong)
def test_mysecret_in_weak_list(self):
"""'mysecret' must be in the weak secrets blocklist."""
with open(os.path.join(DEPLOY_DIR, "auth.py")) as f:
source = f.read()
# Parse the source to find _WEAK_SECRETS set
self.assertIn("mysecret", source,
"'mysecret' must be listed in _WEAK_SECRETS blocklist")
def test_auto_generation_exists(self):
"""auth.py must auto-generate key when none is set."""
with open(os.path.join(DEPLOY_DIR, "auth.py")) as f:
source = f.read()
self.assertIn("token_hex", source,
"auth.py must use secrets.token_hex for auto-generation")
# ============================================================================
# B2-V3: eval() removed from /config/dump
# ============================================================================
class TestConfigDumpNoEval(unittest.TestCase):
"""Verify eval() is completely removed from the /config/dump path."""
def test_no_safe_eval_config(self):
"""_safe_eval_config function must be removed from server.py."""
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
source = f.read()
self.assertNotIn("def _safe_eval_config", source,
"_safe_eval_config must be deleted (replaced with JSON input)")
def test_config_from_json_exists(self):
"""_config_from_json function must exist."""
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
source = f.read()
self.assertIn("def _config_from_json", source,
"_config_from_json must replace _safe_eval_config")
def test_config_dump_has_auth(self):
"""config_dump endpoint must require authentication."""
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
source = f.read()
# Find the config_dump function and check it has token_dep
idx = source.index("config_dump")
# Look backwards for the decorator/function definition area
nearby = source[max(0, idx-200):idx+200]
self.assertIn("token_dep", nearby,
"/config/dump must require token_dep authentication")
def test_no_eval_in_config_path(self):
"""No eval() call should exist in the config dump code path."""
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
source = f.read()
# The old allowlist constants should be gone
self.assertNotIn("_SAFE_CONFIG_ALLOWED_NAMES", source,
"Old eval allowlist constants should be removed")
self.assertNotIn("_SAFE_CONFIG_ALLOWED_ATTRS", source,
"Old eval allowlist constants should be removed")
# ============================================================================
# B2-V1: /execute_js disabled by default
# ============================================================================
class TestExecuteJsDisabled(unittest.TestCase):
"""Verify /execute_js is disabled by default with proper guards."""
def test_execute_js_flag_exists(self):
"""EXECUTE_JS_ENABLED flag must exist in server.py."""
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
source = f.read()
self.assertIn("EXECUTE_JS_ENABLED", source)
def test_execute_js_disabled_by_default(self):
"""EXECUTE_JS_ENABLED must default to false."""
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
source = f.read()
# Find the line that sets EXECUTE_JS_ENABLED
for line in source.splitlines():
if "EXECUTE_JS_ENABLED" in line and "os.environ" in line:
self.assertIn('"false"', line,
"EXECUTE_JS_ENABLED must default to 'false'")
return
self.fail("Could not find EXECUTE_JS_ENABLED env var line")
def test_execute_js_checks_flag(self):
"""execute_js endpoint must check EXECUTE_JS_ENABLED."""
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
source = f.read()
idx = source.index("async def execute_js")
func_body = source[idx:idx+3000]
self.assertIn("EXECUTE_JS_ENABLED", func_body,
"execute_js must check EXECUTE_JS_ENABLED flag")
def test_execute_js_has_ssrf_check(self):
"""execute_js must validate URL against SSRF blocklist."""
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
source = f.read()
idx = source.index("async def execute_js")
func_body = source[idx:idx+3000]
self.assertIn("validate_webhook_url", func_body,
"execute_js must validate URL against SSRF blocklist")
def test_disable_web_security_removed_from_defaults(self):
"""--disable-web-security must not be in default browser args."""
with open(os.path.join(DEPLOY_DIR, "utils.py")) as f:
source = f.read()
# Find the DEFAULT_CONFIG extra_args
tree = ast.parse(source)
for node in ast.walk(tree):
if isinstance(node, ast.Constant) and node.value == "--disable-web-security":
self.fail("--disable-web-security must not be in DEFAULT_CONFIG extra_args")
def test_disable_web_security_removed_from_config_yml(self):
"""--disable-web-security must not be active in config.yml."""
with open(os.path.join(DEPLOY_DIR, "config.yml")) as f:
for line in f:
stripped = line.strip()
if stripped == '- "--disable-web-security"':
self.fail("--disable-web-security must not be an active entry in config.yml")
# ============================================================================
# B2-V4: exec-based hook system removed (replaced by declarative registry)
# ============================================================================
class TestHookExecPathRemoved(unittest.TestCase):
"""The exec()/compile() hook system is gone; hooks are now declarative.
No sandbox survives attacker Python in-process, so hook_manager.py was
deleted and replaced by hook_registry.py (a fixed set of server-authored
actions selected by name with schema-validated scalar params).
"""
def test_hook_manager_module_deleted(self):
self.assertFalse(
os.path.exists(os.path.join(DEPLOY_DIR, "hook_manager.py")),
"hook_manager.py (the exec-based hook system) must be deleted",
)
def test_no_exec_compile_eval_in_docker_layer(self):
"""The RCE primitives must be absent from the docker server code.
Uses AST so it flags only bare-builtin calls (exec/eval/compile), not
attribute calls like re.compile(...) or the words in comments/docstrings.
"""
import ast
import glob
offenders = []
for path in glob.glob(os.path.join(DEPLOY_DIR, "*.py")):
with open(path) as f:
tree = ast.parse(f.read(), filename=path)
for node in ast.walk(tree):
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id in {"exec", "eval", "compile"}
):
offenders.append((os.path.basename(path), node.func.id, node.lineno))
self.assertEqual(offenders, [], f"bare exec/eval/compile call found: {offenders}")
def test_registry_rejects_unknown_action(self):
sys.path.insert(0, DEPLOY_DIR)
from hook_registry import build_declarative_hooks, HookValidationError
with self.assertRaises(HookValidationError):
build_declarative_hooks([{"action": "run_python", "params": {"code": "x"}}])
def test_registry_rejects_bad_params(self):
sys.path.insert(0, DEPLOY_DIR)
from hook_registry import build_declarative_hooks, HookValidationError
with self.assertRaises(HookValidationError):
build_declarative_hooks([{"action": "block_resources", "params": {"resource_types": ["script"]}}])
def test_registry_builds_valid_hooks(self):
sys.path.insert(0, DEPLOY_DIR)
from hook_registry import build_declarative_hooks
hooks = build_declarative_hooks([
{"action": "block_resources", "params": {"resource_types": ["image"]}},
{"action": "scroll_to_bottom", "params": {"max_steps": 5}},
])
# block_resources -> on_page_context_created, scroll -> before_retrieve_html
self.assertIn("on_page_context_created", hooks)
self.assertIn("before_retrieve_html", hooks)
for fn in hooks.values():
self.assertTrue(callable(fn))
if __name__ == "__main__":
unittest.main()
deploy/docker/tests/test_security_artifact_store.py
+108
View File
@@ -0,0 +1,108 @@
"""
R4 artifact-store behavioral tests: the server owns the directory, names and
bytes. Writes are O_EXCL|O_NOFOLLOW 0600; retrieval requires a 32-hex id,
refuses symlinks/non-regular files, and enforces a TTL.
"""
import os
import time
import pytest
@pytest.fixture
def store(tmp_path, monkeypatch):
"""Point the artifact store at an isolated temp dir and reload it."""
monkeypatch.setenv("CRAWL4AI_ARTIFACT_DIR", str(tmp_path / "art"))
monkeypatch.setenv("CRAWL4AI_MAX_ARTIFACT_BYTES", "1024")
monkeypatch.setenv("CRAWL4AI_ARTIFACT_QUOTA_BYTES", "4096")
monkeypatch.setenv("CRAWL4AI_ARTIFACT_TTL_SECONDS", "3600")
import importlib
import artifacts
importlib.reload(artifacts)
artifacts.init_store()
yield artifacts
importlib.reload(artifacts) # restore defaults for other tests
pytestmark = pytest.mark.posture
class TestWrite:
def test_write_returns_hex_id_and_meta(self, store):
meta = store.write_artifact("png", b"\x89PNG data")
assert len(meta["artifact_id"]) == 32 and all(c in "0123456789abcdef" for c in meta["artifact_id"])
assert meta["mime"] == "image/png" and meta["size"] == len(b"\x89PNG data")
def test_dir_is_0700_and_file_0600(self, store):
meta = store.write_artifact("pdf", b"%PDF-1.4")
assert oct(os.stat(store.ARTIFACT_DIR).st_mode)[-3:] == "700"
path, _ = store.resolve_artifact(meta["artifact_id"])
assert oct(os.stat(path).st_mode)[-3:] == "600"
def test_oversize_rejected(self, store):
with pytest.raises(store.ArtifactTooLarge):
store.write_artifact("png", b"x" * 2048) # cap is 1024
def test_quota_enforced(self, store):
for _ in range(4):
store.write_artifact("png", b"x" * 1000)
with pytest.raises(store.QuotaExceeded):
store.write_artifact("png", b"x" * 1000) # would exceed 4096
class TestResolve:
def test_roundtrip(self, store):
meta = store.write_artifact("png", b"hello")
path, mime = store.resolve_artifact(meta["artifact_id"])
assert mime == "image/png"
with open(path, "rb") as f:
assert f.read() == b"hello"
@pytest.mark.parametrize("bad", ["../etc/passwd", "..", "g" * 32, "abc", "/etc/passwd", "a" * 31])
def test_non_hex_or_traversal_404(self, store, bad):
with pytest.raises(store.ArtifactNotFound):
store.resolve_artifact(bad)
def test_symlink_not_followed(self, store):
# Plant a symlink named like a valid artifact pointing at a secret.
secret = os.path.join(store.ARTIFACT_DIR, "secret.txt")
with open(secret, "wb") as f:
f.write(b"TOPSECRET")
fake_id = "a" * 32
link = os.path.join(store.ARTIFACT_DIR, fake_id + ".png")
os.symlink(secret, link)
with pytest.raises(store.ArtifactNotFound):
store.resolve_artifact(fake_id) # lstat sees a symlink -> refuse
def test_ttl_expired_404_and_reaped(self, store):
meta = store.write_artifact("png", b"old")
path, _ = store.resolve_artifact(meta["artifact_id"])
old = time.time() - 7200 # 2h ago, TTL is 1h
os.utime(path, (old, old))
with pytest.raises(store.ArtifactNotFound):
store.resolve_artifact(meta["artifact_id"])
assert not os.path.exists(path) # resolve reaped it
class TestWriteIsExclusiveAndNoFollow:
def test_write_uses_oexcl_nofollow(self, store):
import inspect
src = inspect.getsource(store.write_artifact)
assert "O_EXCL" in src and "O_NOFOLLOW" in src
class TestJanitor:
def test_janitor_removes_symlinks_and_expired(self, store):
meta = store.write_artifact("png", b"keep")
# expired regular file
old_meta = store.write_artifact("pdf", b"old")
op, _ = store.resolve_artifact(old_meta["artifact_id"])
past = time.time() - 7200
os.utime(op, (past, past))
# a planted symlink
os.symlink("/etc/passwd", os.path.join(store.ARTIFACT_DIR, "deadbeef" * 4 + ".png"))
reaped = store.janitor()
assert reaped >= 2
# the fresh one survives
store.resolve_artifact(meta["artifact_id"])
deploy/docker/tests/test_security_authz.py
+116
View File
@@ -0,0 +1,116 @@
"""
Behavioral authorization tests for R1 (beyond the blanket default-deny gate):
* admin scope - monitor destructive actions require an admin principal;
a valid data-scope token is allowed past the gate but
rejected (403) by require_admin.
* MCP no-laundering - the MCP tool proxy authenticates its internal loopback
call (it no longer relies on the endpoints being open).
* job ownership - a task records its owner; a different requester gets 404
(not 403), and an admin can read any task.
These exercise the running app, not its source text.
"""
import pytest
pytestmark = pytest.mark.posture
from auth import create_access_token # noqa: E402
def _bearer(token: str) -> dict:
return {"Authorization": f"Bearer {token}"}
# ───────────────────────────── admin scope ─────────────────────────────
class TestAdminScope:
def test_data_scope_cannot_reset_stats(self, stock_client):
data_tok = create_access_token({"sub": "user@x.com"}, scope="data")
r = stock_client.post("/monitor/stats/reset", headers=_bearer(data_tok))
assert r.status_code == 403, f"data-scope reached admin action: {r.status_code}"
def test_data_scope_cannot_kill_browser(self, stock_client):
data_tok = create_access_token({"sub": "user@x.com"}, scope="data")
r = stock_client.post(
"/monitor/actions/kill_browser", json={"sig": "abc"}, headers=_bearer(data_tok)
)
assert r.status_code == 403
def test_admin_scope_passes_require_admin(self, stock_client):
admin_tok = create_access_token({"sub": "ops@x.com"}, scope="admin")
r = stock_client.post("/monitor/stats/reset", headers=_bearer(admin_tok))
# Past require_admin: not a 401/403. (500 acceptable here: no monitor
# singleton without a lifespan — the point is authz let it through.)
assert r.status_code not in (401, 403), f"admin blocked: {r.status_code}"
def test_unauthenticated_admin_action_is_401(self, stock_client):
r = stock_client.post("/monitor/stats/reset")
assert r.status_code == 401
# ────────────────────────── MCP no-laundering ──────────────────────────
class TestMcpNoLaundering:
def test_mcp_proxy_attaches_service_credential(self):
"""The internal loopback proxy must carry a valid, data-scope token."""
import mcp_bridge
from auth import decode_token
headers = mcp_bridge._service_auth_headers()
assert "Authorization" in headers
scheme, _, token = headers["Authorization"].partition(" ")
assert scheme == "Bearer" and token
claims = decode_token(token)
assert claims["scope"] == "data", "MCP service token must not be admin-scoped"
def test_mcp_base_url_is_loopback(self, server_module):
"""The MCP proxy must target loopback, never the 0.0.0.0 bind address."""
import inspect
src = inspect.getsource(server_module)
assert "http://127.0.0.1:" in src
assert 'base_url=f"http://{config' not in src
# ─────────────────────────── job ownership ─────────────────────────────
class _FakeRedis:
"""Minimal async Redis stub holding one task hash."""
def __init__(self, task):
# Real redis returns bytes keys/values; decode_redis_hash expects that.
self._task = {k.encode(): str(v).encode() for k, v in task.items()}
async def hgetall(self, key):
return dict(self._task)
async def delete(self, key):
self._task = {}
@pytest.mark.asyncio
class TestJobOwnership:
async def _status(self, task, **kw):
import api
redis = _FakeRedis(task)
return await api.handle_task_status(
redis, "crawl_abc", base_url="http://t/", keep=True, **kw
)
async def test_owner_can_read_own_task(self):
task = {"status": "completed", "created_at": "2026-01-01T00:00:00",
"url": "[]", "result": "{}", "owner": "alice@x.com"}
resp = await self._status(task, requester="alice@x.com", is_admin=False)
assert resp.status_code == 200
async def test_other_requester_gets_404_not_403(self):
from fastapi import HTTPException
task = {"status": "completed", "created_at": "2026-01-01T00:00:00",
"url": "[]", "result": "{}", "owner": "alice@x.com"}
with pytest.raises(HTTPException) as exc:
await self._status(task, requester="mallory@x.com", is_admin=False)
assert exc.value.status_code == 404 # not 403: don't reveal existence
async def test_admin_can_read_any_task(self):
task = {"status": "completed", "created_at": "2026-01-01T00:00:00",
"url": "[]", "result": "{}", "owner": "alice@x.com"}
resp = await self._status(task, requester="ops@x.com", is_admin=True)
assert resp.status_code == 200
deploy/docker/tests/test_security_container_posture.py
+124
View File
@@ -0,0 +1,124 @@
"""
R7 container/deployment posture - static parse of the deployment artifacts.
These assert the hardening is declared in the Dockerfile / docker-compose.yml /
supervisord.conf. The RUNTIME acceptance gate (the hardened image builds,
Chromium starts without --no-sandbox under userns/seccomp, the read-only rootfs
+ tmpfs work, redis truly requires auth) needs an actual `docker build` + boot
and is tracked as build-gated; it is out of scope for this offline suite.
"""
import os
import re
import pytest
pytestmark = pytest.mark.posture
REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
DOCKER_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
def _read(path):
with open(path) as f:
return f.read()
@pytest.fixture(scope="module")
def dockerfile():
return _read(os.path.join(REPO_ROOT, "Dockerfile"))
@pytest.fixture(scope="module")
def compose():
return _read(os.path.join(REPO_ROOT, "docker-compose.yml"))
@pytest.fixture(scope="module")
def supervisord():
return _read(os.path.join(DOCKER_DIR, "supervisord.conf"))
class TestDockerfile:
def test_no_redis_expose(self, dockerfile):
# No active EXPOSE 6379 line (commented references are fine).
for line in dockerfile.splitlines():
stripped = line.strip()
if stripped.startswith("#"):
continue
assert not re.match(r"EXPOSE\s+.*\b6379\b", stripped), \
"redis port 6379 must not be EXPOSEd"
def test_app_dir_root_owned_readonly(self, dockerfile):
assert "chown -R root:root ${APP_HOME}" in dockerfile
assert "chmod -R a-w ${APP_HOME}" in dockerfile
def test_artifact_dir_created_0700(self, dockerfile):
assert "/var/lib/crawl4ai/outputs" in dockerfile
assert "chmod 700 /var/lib/crawl4ai/outputs" in dockerfile
def test_runs_as_non_root(self, dockerfile):
assert re.search(r"^USER\s+appuser", dockerfile, re.MULTILINE)
class TestSupervisord:
def test_redis_requires_password(self, supervisord):
assert "--requirepass" in supervisord
def test_redis_bound_loopback(self, supervisord):
assert "--bind 127.0.0.1" in supervisord
def test_gunicorn_bind_is_env_driven(self, supervisord):
# entrypoint.sh resolves GUNICORN_BIND (loopback unless a credential).
assert "ENV_GUNICORN_BIND" in supervisord
def test_gunicorn_request_limits(self, supervisord):
assert "--limit-request-line" in supervisord
class TestCompose:
def test_cap_drop_all(self, compose):
assert "cap_drop" in compose and "ALL" in compose
def test_no_new_privileges(self, compose):
assert "no-new-privileges:true" in compose
def test_read_only_rootfs(self, compose):
assert re.search(r"read_only:\s*true", compose)
def test_no_host_dev_shm_bind(self, compose):
assert "/dev/shm:/dev/shm" not in compose
assert "shm_size" in compose
def test_pids_limit(self, compose):
assert "pids_limit" in compose
def test_read_only_runtime_tmpfs_are_appuser_owned(self, compose):
assert "/var/lib/redis:uid=999,gid=999,mode=0700" in compose
assert "/var/lib/crawl4ai/outputs:uid=999,gid=999,mode=0700" in compose
assert "/home/appuser/.crawl4ai:uid=999,gid=999,mode=0700" in compose
assert "/home/appuser/.gunicorn:uid=999,gid=999,mode=0700" in compose
def test_playwright_cache_is_not_shadowed(self, compose):
assert "/home/appuser/.cache\n" not in compose
assert "/home/appuser/.cache/url_seeder:uid=999,gid=999,mode=0700" in compose
class TestEntrypoint:
def test_entrypoint_exists_and_resolves_bind(self):
src = _read(os.path.join(DOCKER_DIR, "entrypoint.sh"))
assert "GUNICORN_BIND" in src and "REDIS_PASSWORD" in src
assert "127.0.0.1" in src # loopback default when no credential
class TestSandboxOptOut:
def test_default_keeps_no_sandbox(self, server_module, monkeypatch):
monkeypatch.setattr(server_module, "CHROMIUM_SANDBOX", False)
assert "--no-sandbox" in server_module._browser_extra_args()
def test_opt_in_drops_no_sandbox(self, server_module, monkeypatch):
# CRAWL4AI_CHROMIUM_SANDBOX=true -> run the renderer sandboxed.
monkeypatch.setattr(server_module, "CHROMIUM_SANDBOX", True)
assert "--no-sandbox" not in server_module._browser_extra_args()
# other flags are preserved
assert "--disable-dev-shm-usage" in server_module._browser_extra_args()
deploy/docker/tests/test_security_default_posture.py
+254
View File
@@ -0,0 +1,254 @@
"""
HEADLINE SECURITY POSTURE GATE — test_security_default_posture
This is the acceptance gate for the Docker-server security hardening
(root causes R1-R7). It boots the server *with the shipped config.yml* and
asserts the secure-by-default end state:
- every mutating + read endpoint, static mount, and MCP transport requires
auth out of the box (R1); only the health check and UI shell pages
(dashboard/playground) are public,
- the token endpoint will not freely mint credentials when no api_token /
secret is configured (R1),
- strong security headers + a strict CSP are present on every response even
when `security.enabled` is false (R6),
- the effective browser launch flags carry no `--no-sandbox` and no
TLS-verification-disabling flags (R3/R7),
- Redis requires a password and security is on by default (R7),
- dangerous request->code features are off by default (R2).
IT IS EXPECTED TO BE RED TODAY. Each failing assertion names one gap the
hardening must close. As R1-R7 land, methods flip green; when the whole class
is green, the default Docker deploy is safe and CI can gate on it.
Run: pytest deploy/docker/tests/test_security_default_posture.py -v
"""
import pytest
from redis_config import build_rate_limit_storage_uri, build_redis_url
pytestmark = pytest.mark.posture
HEALTH = "/health"
# (method, path, json-body-or-None) for every endpoint that must NOT be
# reachable without authentication. Bodies are intentionally minimal/invalid so
# that in the *current* (open) state the request fails fast at validation
# rather than launching a real crawl — the assertion (== 401) still fails,
# which is the RED signal we want, with no browser/network side effects.
PROTECTED_ENDPOINTS = [
# mutating / action endpoints
("post", "/crawl", {}),
("post", "/crawl/stream", {}),
("post", "/md", {}),
("post", "/html", {}),
("post", "/screenshot", {}),
("post", "/pdf", {}),
("post", "/execute_js", {}),
("post", "/config/dump", {}),
("post", "/crawl/job", {}),
("post", "/llm/job", {}),
# monitor mutating actions (must additionally require *admin* scope; here we
# only assert they are not anonymously reachable). Pools are empty without a
# lifespan, so probing these performs no destructive action.
("post", "/monitor/actions/cleanup", {}),
("post", "/monitor/actions/kill_browser", {}),
("post", "/monitor/actions/restart_browser", {}),
("post", "/monitor/stats/reset", {}),
# read / info endpoints that currently have NO auth dependency at all
("get", "/schema", None),
("get", "/hooks/info", None),
("get", "/ask", None),
("get", "/metrics", None),
("get", "/llm/example.com", None), # no `q` -> fast 400 today, must be 401
("get", "/monitor/health", None),
("get", "/monitor/requests", None),
("get", "/monitor/browsers", None),
("get", "/monitor/endpoints/stats", None),
("get", "/monitor/timeline", None),
# MCP transport (must be gated; today it is open AND launders credentials)
("get", "/mcp/schema", None),
]
# UI shell pages that are intentionally public — they serve only static
# HTML/CSS/JS and contain no data. All data routes behind them remain gated.
PUBLIC_UI_PATHS = ["/dashboard/", "/playground/"]
def _call(client, method, path, body):
fn = getattr(client, method)
if body is not None:
return fn(path, json=body)
return fn(path)
class TestDefaultDeployIsSafe:
"""The shipped Docker image must be secure with an empty/default config."""
# ───────────────────────── R1: authentication ─────────────────────────
def test_health_check_is_public(self, stock_client):
"""The health endpoint is the one intentionally public route."""
r = stock_client.get(HEALTH)
assert r.status_code == 200
@pytest.mark.parametrize("path", PUBLIC_UI_PATHS, ids=PUBLIC_UI_PATHS)
def test_ui_shell_is_public(self, stock_client, path):
"""UI shells serve static HTML only — no data — so they load without auth."""
r = stock_client.get(path)
assert r.status_code == 200, (
f"GET {path} returned {r.status_code}; expected 200. "
f"UI shell pages must be publicly accessible."
)
@pytest.mark.parametrize(
"method,path,body",
PROTECTED_ENDPOINTS,
ids=[f"{m.upper()} {p}" for m, p, _ in PROTECTED_ENDPOINTS],
)
def test_endpoint_requires_auth_by_default(self, stock_client, method, path, body):
"""Without a credential, every non-health route must refuse (401)."""
r = _call(stock_client, method, path, body)
assert r.status_code == 401, (
f"{method.upper()} {path} returned {r.status_code} unauthenticated; "
f"expected 401. Endpoint is reachable without a credential."
)
def test_token_endpoint_does_not_freely_mint(self, stock_client, server_module, monkeypatch):
"""With no api_token configured, /token must not hand a JWT to anyone.
Force the email-domain MX check to pass so the test exercises the real
vulnerability (anonymous token minting) rather than passing for the
wrong reason when offline.
"""
monkeypatch.setattr(server_module, "verify_email_domain", lambda email: True)
r = stock_client.post("/token", json={"email": "attacker@example.com"})
if r.status_code == 200:
assert "access_token" not in r.json(), (
"/token minted a credential for an anonymous caller with no "
"api_token configured — anyone can self-issue a valid JWT."
)
# ─────────────────────── R6: headers / CSP / framing ──────────────────
def test_security_headers_present_by_default(self, stock_client):
"""Baseline security headers must be emitted even when security.enabled is false."""
r = stock_client.get(HEALTH)
h = {k.lower(): v for k, v in r.headers.items()}
assert h.get("x-content-type-options") == "nosniff"
assert h.get("x-frame-options", "").upper() == "DENY"
assert "content-security-policy" in h, "no CSP header on a default response"
def test_csp_is_strict(self, stock_client):
"""CSP must lock script execution to self and forbid inline/unsafe + framing."""
r = stock_client.get(HEALTH)
csp = r.headers.get("content-security-policy", "")
assert "script-src 'self'" in csp, f"CSP missing strict script-src: {csp!r}"
assert "frame-ancestors 'none'" in csp, f"CSP allows framing: {csp!r}"
assert "unsafe-inline" not in csp, f"CSP permits unsafe-inline: {csp!r}"
# ─────────────────── R3 / R7: browser & TLS launch flags ───────────────
@pytest.mark.xfail(
reason="build-gated: --no-sandbox stays until the container runs with an "
"unprivileged userns or a verified seccomp profile and a docker build "
"confirms Chromium still starts. Tracked, not yet done.",
strict=False,
)
def test_no_no_sandbox_flag(self, effective_browser_args):
"""Chromium must not run with --no-sandbox by default (renderer escape)."""
assert "--no-sandbox" not in effective_browser_args, (
"default config launches Chromium with --no-sandbox"
)
def test_tls_verification_not_disabled(self, effective_browser_args):
"""TLS verification must not be disabled by default."""
assert "--ignore-certificate-errors" not in effective_browser_args
assert "--allow-insecure-localhost" not in effective_browser_args
# ───────────────────────── R7: deploy posture ─────────────────────────
def test_security_enabled_by_default(self, effective_config):
"""The security middleware block must be on out of the box."""
assert effective_config["security"]["enabled"] is True
def test_redis_is_not_network_exposed(self, effective_redis_url, effective_config):
"""Redis must not be open on the network: loopback host or a password.
The container runs redis loopback-only with --requirepass (supervisord)
and no EXPOSE (asserted in the container-posture suite); a password also
satisfies this for an external redis.
"""
rc = effective_config.get("redis", {})
host = str(rc.get("host", "localhost")).lower()
pw = rc.get("password", "") or ""
loopback = host in ("localhost", "127.0.0.1", "::1")
assert loopback or pw, (
"redis is neither loopback-bound nor password-protected"
)
def test_redis_url_includes_acl_username_and_encoded_password(self, monkeypatch):
"""Redis URLs must authenticate during HELLO when Redis is password-protected."""
monkeypatch.setenv("REDIS_PASSWORD", "p@ss/w:rd")
cfg = {
"redis": {
"host": "localhost",
"port": 6379,
"db": 0,
}
}
url = build_redis_url(cfg)
assert url.startswith("redis://default:")
assert "p%40ss%2Fw%3Ard" in url
assert url.endswith("@localhost:6379/0")
def test_rate_limit_redis_storage_reuses_redis_password(self, monkeypatch):
"""SlowAPI must not connect to protected Redis without credentials."""
monkeypatch.setenv("REDIS_PASSWORD", "abc123")
cfg = {
"redis": {"host": "localhost", "port": 6379, "db": 0},
"rate_limiting": {"storage_uri": "redis://localhost:6379/0"},
}
url = build_rate_limit_storage_uri(cfg)
assert url.startswith("redis://default:")
assert "abc123" in url
assert url.endswith("@localhost:6379/0")
def test_rate_limit_storage_preserves_explicit_auth(self, monkeypatch):
"""Do not rewrite operator-provided rate-limit Redis credentials."""
monkeypatch.setenv("REDIS_PASSWORD", "container-secret")
storage_uri = "redis://" + "alice" + ":" + "custom" + "@redis:6379/2"
cfg = {
"redis": {"host": "localhost", "port": 6379, "db": 0},
"rate_limiting": {"storage_uri": storage_uri},
}
assert build_rate_limit_storage_uri(cfg) == storage_uri
def test_rate_limit_storage_keeps_memory_backend(self, monkeypatch):
"""Non-Redis rate-limit backends must not be rewritten."""
monkeypatch.setenv("REDIS_PASSWORD", "abc123")
cfg = {
"redis": {"host": "localhost", "port": 6379, "db": 0},
"rate_limiting": {"storage_uri": "memory://"},
}
assert build_rate_limit_storage_uri(cfg) == "memory://"
def test_trusted_hosts_not_wildcard_when_exposed(self, effective_config):
"""A wildcard trusted_hosts on a non-loopback bind silently disables the host guard."""
host = effective_config["app"]["host"]
trusted = effective_config["security"]["trusted_hosts"]
exposed = host not in ("127.0.0.1", "localhost", "::1")
assert not (exposed and trusted == ["*"]), (
f"app binds {host} but trusted_hosts is {trusted} (host guard disabled)"
)
# ───────────────────── R2: dangerous features off by default ───────────
def test_hooks_disabled_by_default(self, server_module):
"""User-supplied hook code (exec path) must be off unless explicitly enabled."""
assert server_module.HOOKS_ENABLED is False
def test_execute_js_disabled_by_default(self, server_module):
"""Arbitrary JS execution endpoint must be off unless explicitly enabled."""
assert server_module.EXECUTE_JS_ENABLED is False
deploy/docker/tests/test_security_download_traversal.py
+100
View File
@@ -0,0 +1,100 @@
"""Path-traversal regression tests for crawler file downloads.
Y4tacker reported an arbitrary-file-write -> RCE: a download filename taken
from an attacker-controlled Content-Disposition header (HTTP strategy) or the
browser's suggested filename (Playwright strategy) was joined to the downloads
dir with no confinement, so an absolute path or ``../`` sequence escaped it.
Both call sites now route through ``_safe_download_filepath``, which reduces
the name to a bare basename and re-checks containment. These tests exercise
that helper directly (offline, no network/browser).
"""
import os
import tempfile
import pytest
from crawl4ai.async_crawler_strategy import _safe_download_filepath, _nofollow_opener
pytestmark = pytest.mark.posture
@pytest.fixture
def downloads_dir():
with tempfile.TemporaryDirectory() as d:
yield d
class TestDownloadPathConfinement:
def test_plain_filename_kept(self, downloads_dir):
p = _safe_download_filepath(downloads_dir, "report.pdf")
assert p == os.path.join(os.path.realpath(downloads_dir), "report.pdf")
@pytest.mark.parametrize(
"evil",
[
"/etc/cron.d/evil",
"/tmp/pwned_absolute.sh",
"../../../.bashrc",
"../../.ssh/authorized_keys",
"../../../../etc/passwd",
"..",
".",
"",
"foo/../../bar",
],
)
def test_escape_attempts_confined(self, downloads_dir, evil):
"""No input may produce a path outside the downloads root."""
root = os.path.realpath(downloads_dir)
p = _safe_download_filepath(downloads_dir, evil)
assert os.path.commonpath([root, p]) == root
# The resolved file sits directly inside the root (basename only).
assert os.path.dirname(p) == root
def test_absolute_path_does_not_win(self, downloads_dir):
p = _safe_download_filepath(downloads_dir, "/etc/cron.d/evil")
assert not p.startswith("/etc/")
assert p.startswith(os.path.realpath(downloads_dir))
def test_symlink_escape_rejected(self, downloads_dir):
"""If the basename is an existing symlink in the root pointing out,
realpath resolves outside the root and the write must be rejected."""
root = os.path.realpath(downloads_dir)
outside_dir = tempfile.mkdtemp()
outside_file = os.path.join(outside_dir, "target")
link = os.path.join(root, "evil")
os.symlink(outside_file, link)
# filename "evil" -> root/evil is a symlink to outside -> rejected.
with pytest.raises(ValueError):
_safe_download_filepath(downloads_dir, "evil")
class TestNoFollowOpener:
"""The write must refuse to follow a symlink swapped in after path
confinement (the TOCTOU window between realpath-check and open)."""
def test_open_refuses_symlink_target(self, downloads_dir):
import os
outside = tempfile.mkdtemp()
outside_file = os.path.join(outside, "secret")
with open(outside_file, "w") as f:
f.write("original")
# Attacker swaps the confined target for a symlink pointing outside.
target = os.path.join(downloads_dir, "report.pdf")
os.symlink(outside_file, target)
with pytest.raises(OSError): # ELOOP from O_NOFOLLOW
with open(target, "wb", opener=_nofollow_opener) as f:
f.write(b"pwned")
# The outside file was not overwritten through the symlink.
with open(outside_file) as f:
assert f.read() == "original"
def test_open_writes_normal_file(self, downloads_dir):
import os
target = os.path.join(downloads_dir, "ok.bin")
with open(target, "wb", opener=_nofollow_opener) as f:
f.write(b"data")
with open(target, "rb") as f:
assert f.read() == b"data"
deploy/docker/tests/test_security_egress_proxy.py
+132
View File
@@ -0,0 +1,132 @@
"""
R3 browser egress-proxy tests (real loopback sockets, fully offline).
The pinning proxy is what actually stops DNS rebinding on the browser path:
Chromium is pointed at it, so it asks the proxy to CONNECT host:port; the proxy
resolves-and-pins (egress_broker.resolve_and_pin) and dials only the pinned,
global IP. We drive it with a raw asyncio client + a fake upstream, and stub
resolve_and_pin so a "public" host pins to the loopback upstream while an
"internal" host is refused. (The not-is_global rule itself is covered in
test_security_ssrf_egress.py.)
"""
import asyncio
import pytest
import egress_proxy
from egress_broker import EgressBlocked, PinnedTarget
from egress_proxy import PinningProxy
pytestmark = pytest.mark.posture
async def _fake_upstream():
async def handle(reader, writer):
await reader.read(65536)
writer.write(b"UPSTREAM-OK")
await writer.drain()
writer.close()
server = await asyncio.start_server(handle, "127.0.0.1", 0)
return server, server.sockets[0].getsockname()[1]
@pytest.mark.asyncio
class TestPinningProxy:
async def test_connect_to_global_host_tunnels(self, monkeypatch):
up, up_port = await _fake_upstream()
# Pin "good.example" to the loopback upstream (stand-in for a global IP).
def fake_pin(url):
return PinnedTarget("https", "good.example", up_port, "127.0.0.1")
monkeypatch.setattr(egress_proxy, "resolve_and_pin", fake_pin)
proxy = PinningProxy()
await proxy.start()
try:
r, w = await asyncio.open_connection(proxy.bound_host, proxy.bound_port)
w.write(f"CONNECT good.example:{up_port} HTTP/1.1\r\n\r\n".encode())
await w.drain()
status = await asyncio.wait_for(r.readline(), timeout=5)
assert b"200" in status
await r.readline() # blank line after the 200
w.write(b"hello")
await w.drain()
body = await asyncio.wait_for(r.read(100), timeout=5)
assert b"UPSTREAM-OK" in body
w.close()
finally:
await proxy.stop()
up.close()
async def test_connect_to_internal_host_blocked(self, monkeypatch):
def fake_pin(url):
raise EgressBlocked()
monkeypatch.setattr(egress_proxy, "resolve_and_pin", fake_pin)
proxy = PinningProxy()
await proxy.start()
try:
r, w = await asyncio.open_connection(proxy.bound_host, proxy.bound_port)
w.write(b"CONNECT evil.example:443 HTTP/1.1\r\n\r\n")
await w.drain()
status = await asyncio.wait_for(r.readline(), timeout=5)
assert b"403" in status
w.close()
finally:
await proxy.stop()
async def test_proxy_dials_pinned_ip_not_requested_host(self, monkeypatch):
# resolve_and_pin returns a pinned ip distinct from the CONNECT host;
# assert the proxy dials the pinned ip.
dialed = {}
up, up_port = await _fake_upstream()
def fake_pin(url):
return PinnedTarget("https", "rebind.example", up_port, "127.0.0.1")
monkeypatch.setattr(egress_proxy, "resolve_and_pin", fake_pin)
real_open = asyncio.open_connection
async def spy_open(host, port, *a, **k):
dialed["host"], dialed["port"] = host, port
return await real_open(host, port, *a, **k)
# patch only the name the proxy module uses
monkeypatch.setattr(egress_proxy.asyncio, "open_connection", spy_open)
proxy = PinningProxy()
await proxy.start()
try:
r, w = await real_open(proxy.bound_host, proxy.bound_port)
w.write(f"CONNECT rebind.example:{up_port} HTTP/1.1\r\n\r\n".encode())
await w.drain()
await asyncio.wait_for(r.readline(), timeout=5)
assert dialed.get("host") == "127.0.0.1" # the pinned ip
w.close()
finally:
await proxy.stop()
up.close()
async def test_malformed_connect_400(self):
proxy = PinningProxy()
await proxy.start()
try:
r, w = await asyncio.open_connection(proxy.bound_host, proxy.bound_port)
w.write(b"CONNECT not-a-host-port HTTP/1.1\r\n\r\n")
await w.drain()
status = await asyncio.wait_for(r.readline(), timeout=5)
assert b"400" in status
w.close()
finally:
await proxy.stop()
class TestEnforceEgressWiring:
def test_enforce_egress_sets_proxy(self, monkeypatch):
import egress_broker
from crawl4ai import BrowserConfig
monkeypatch.setattr(egress_broker, "_EGRESS_PROXY_URL", "http://127.0.0.1:9999")
b = BrowserConfig()
egress_broker.enforce_egress(b)
assert b.proxy_config is not None
assert b.proxy_config.server == "http://127.0.0.1:9999"
deploy/docker/tests/test_security_fixes.py
+274
View File
@@ -0,0 +1,274 @@
#!/usr/bin/env python3
"""
Unit tests for security fixes.
These tests verify the security fixes at the code level without needing a running server.
"""
import sys
import os
# Add parent directory to path to import modules
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
import unittest
class TestURLValidation(unittest.TestCase):
"""Test URL scheme validation helper."""
def setUp(self):
"""Set up test fixtures."""
self.ALLOWED_URL_SCHEMES = ("http://", "https://")
self.ALLOWED_URL_SCHEMES_WITH_RAW = ("http://", "https://", "raw:", "raw://")
def validate_url_scheme(self, url: str, allow_raw: bool = False) -> bool:
"""Local version of validate_url_scheme for testing."""
allowed = self.ALLOWED_URL_SCHEMES_WITH_RAW if allow_raw else self.ALLOWED_URL_SCHEMES
return url.startswith(allowed)
# === SECURITY TESTS: These URLs must be BLOCKED ===
def test_file_url_blocked(self):
"""file:// URLs must be blocked (LFI vulnerability)."""
self.assertFalse(self.validate_url_scheme("file:///etc/passwd"))
self.assertFalse(self.validate_url_scheme("file:///etc/passwd", allow_raw=True))
def test_file_url_blocked_windows(self):
"""file:// URLs with Windows paths must be blocked."""
self.assertFalse(self.validate_url_scheme("file:///C:/Windows/System32/config/sam"))
def test_javascript_url_blocked(self):
"""javascript: URLs must be blocked (XSS)."""
self.assertFalse(self.validate_url_scheme("javascript:alert(1)"))
def test_data_url_blocked(self):
"""data: URLs must be blocked."""
self.assertFalse(self.validate_url_scheme("data:text/html,<script>alert(1)</script>"))
def test_ftp_url_blocked(self):
"""ftp: URLs must be blocked."""
self.assertFalse(self.validate_url_scheme("ftp://example.com/file"))
def test_empty_url_blocked(self):
"""Empty URLs must be blocked."""
self.assertFalse(self.validate_url_scheme(""))
def test_relative_url_blocked(self):
"""Relative URLs must be blocked."""
self.assertFalse(self.validate_url_scheme("/etc/passwd"))
self.assertFalse(self.validate_url_scheme("../../../etc/passwd"))
# === FUNCTIONALITY TESTS: These URLs must be ALLOWED ===
def test_http_url_allowed(self):
"""http:// URLs must be allowed."""
self.assertTrue(self.validate_url_scheme("http://example.com"))
self.assertTrue(self.validate_url_scheme("http://localhost:8080"))
def test_https_url_allowed(self):
"""https:// URLs must be allowed."""
self.assertTrue(self.validate_url_scheme("https://example.com"))
self.assertTrue(self.validate_url_scheme("https://example.com/path?query=1"))
def test_raw_url_allowed_when_enabled(self):
"""raw: URLs must be allowed when allow_raw=True."""
self.assertTrue(self.validate_url_scheme("raw:<html></html>", allow_raw=True))
self.assertTrue(self.validate_url_scheme("raw://<html></html>", allow_raw=True))
def test_raw_url_blocked_when_disabled(self):
"""raw: URLs must be blocked when allow_raw=False."""
self.assertFalse(self.validate_url_scheme("raw:<html></html>", allow_raw=False))
self.assertFalse(self.validate_url_scheme("raw://<html></html>", allow_raw=False))
class TestHookBuiltins(unittest.TestCase):
"""Hooks are declarative now - there is no builtins sandbox to harden.
The exec()-based hook system was removed (no in-process sandbox survives
attacker Python). Hooks are a fixed registry of server-authored actions, so
the relevant guarantee is simply that no action runs arbitrary code.
"""
def test_registry_actions_are_fixed_and_codeless(self):
from hook_registry import HOOK_REGISTRY
# No action accepts/executes raw code.
for action in HOOK_REGISTRY:
self.assertNotIn("code", action.lower())
self.assertNotIn("eval", action.lower())
self.assertNotIn("exec", action.lower())
# The documented safe actions are present.
self.assertTrue({"block_resources", "add_cookies", "set_headers",
"scroll_to_bottom", "wait_for_timeout"}.issubset(HOOK_REGISTRY))
class TestHooksEnabled(unittest.TestCase):
"""Test HOOKS_ENABLED environment variable logic."""
def test_hooks_disabled_by_default(self):
"""Hooks must be disabled by default."""
original = os.environ.pop("CRAWL4AI_HOOKS_ENABLED", None)
try:
hooks_enabled = os.environ.get("CRAWL4AI_HOOKS_ENABLED", "false").lower() == "true"
self.assertFalse(hooks_enabled)
finally:
if original is not None:
os.environ["CRAWL4AI_HOOKS_ENABLED"] = original
def test_hooks_enabled_when_true(self):
"""Hooks must be enabled when CRAWL4AI_HOOKS_ENABLED=true."""
original = os.environ.get("CRAWL4AI_HOOKS_ENABLED")
try:
os.environ["CRAWL4AI_HOOKS_ENABLED"] = "true"
hooks_enabled = os.environ.get("CRAWL4AI_HOOKS_ENABLED", "false").lower() == "true"
self.assertTrue(hooks_enabled)
finally:
if original is not None:
os.environ["CRAWL4AI_HOOKS_ENABLED"] = original
else:
os.environ.pop("CRAWL4AI_HOOKS_ENABLED", None)
def test_hooks_disabled_when_false(self):
"""Hooks must be disabled when CRAWL4AI_HOOKS_ENABLED=false."""
original = os.environ.get("CRAWL4AI_HOOKS_ENABLED")
try:
os.environ["CRAWL4AI_HOOKS_ENABLED"] = "false"
hooks_enabled = os.environ.get("CRAWL4AI_HOOKS_ENABLED", "false").lower() == "true"
self.assertFalse(hooks_enabled)
finally:
if original is not None:
os.environ["CRAWL4AI_HOOKS_ENABLED"] = original
else:
os.environ.pop("CRAWL4AI_HOOKS_ENABLED", None)
class TestComputedFieldExpressionDisabled(unittest.TestCase):
"""Test that computed field 'expression' key is completely disabled.
eval() on untrusted input is fundamentally unsafe - no sandbox survives.
The expression path is now disabled; only 'function' key works.
"""
def test_expression_returns_default(self):
"""expression key must return default value, not evaluate."""
import logging
# Import the actual class
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
schema = {
"baseSelector": "div",
"fields": [
{"name": "price", "selector": "span", "type": "text"},
{
"name": "computed",
"type": "computed",
"expression": "price * 2",
"default": "DISABLED",
},
],
}
strategy = JsonCssExtractionStrategy(schema)
result = strategy._compute_field({"price": 100}, schema["fields"][1])
self.assertEqual(result, "DISABLED")
def test_expression_does_not_execute_code(self):
"""expression must NEVER execute - even harmless code."""
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
schema = {
"baseSelector": "div",
"fields": [{"name": "x", "selector": "span", "type": "text"}],
}
strategy = JsonCssExtractionStrategy(schema)
# These should all return default, never execute
dangerous_expressions = [
"__import__('os').system('id')",
"open('/etc/passwd').read()",
"().__class__.__bases__[0].__subclasses__()",
]
for expr in dangerous_expressions:
field = {"name": "test", "type": "computed", "expression": expr, "default": None}
result = strategy._compute_field({}, field)
self.assertIsNone(result, f"Expression should not execute: {expr}")
def test_function_key_still_works(self):
"""function key with Python callable must still work."""
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
schema = {
"baseSelector": "div",
"fields": [{"name": "price", "selector": "span", "type": "text"}],
}
strategy = JsonCssExtractionStrategy(schema)
field = {
"name": "computed",
"type": "computed",
"function": lambda item: item["price"] * 2,
}
result = strategy._compute_field({"price": 100}, field)
self.assertEqual(result, 200)
class TestDeserializationAllowlist(unittest.TestCase):
"""Test that the deserialization allowlist blocks non-allowlisted types."""
def setUp(self):
self.allowed_types = {
"BrowserConfig", "CrawlerRunConfig", "HTTPCrawlerConfig",
"LLMConfig", "ProxyConfig", "GeolocationConfig",
"SeedingConfig", "VirtualScrollConfig", "LinkPreviewConfig",
"JsonCssExtractionStrategy", "JsonXPathExtractionStrategy",
"JsonLxmlExtractionStrategy", "LLMExtractionStrategy",
"CosineStrategy", "RegexExtractionStrategy",
"DefaultMarkdownGenerator",
"PruningContentFilter", "BM25ContentFilter", "LLMContentFilter",
"LXMLWebScrapingStrategy",
"RegexChunking",
"BFSDeepCrawlStrategy", "DFSDeepCrawlStrategy", "BestFirstCrawlingStrategy",
"FilterChain", "URLPatternFilter", "DomainFilter",
"ContentTypeFilter", "URLFilter", "SEOFilter", "ContentRelevanceFilter",
"KeywordRelevanceScorer", "URLScorer", "CompositeScorer",
"DomainAuthorityScorer", "FreshnessScorer", "PathDepthScorer",
"CacheMode", "MatchMode", "DisplayMode",
"MemoryAdaptiveDispatcher", "SemaphoreDispatcher",
"DefaultTableExtraction", "NoTableExtraction", "LLMTableExtraction",
"RoundRobinProxyStrategy",
}
def test_arbitrary_class_not_in_allowlist(self):
self.assertNotIn("AsyncWebCrawler", self.allowed_types)
def test_crawler_hub_not_in_allowlist(self):
self.assertNotIn("CrawlerHub", self.allowed_types)
def test_browser_profiler_not_in_allowlist(self):
self.assertNotIn("BrowserProfiler", self.allowed_types)
def test_docker_client_not_in_allowlist(self):
self.assertNotIn("Crawl4aiDockerClient", self.allowed_types)
def test_allowlist_has_core_config_types(self):
required = {"BrowserConfig", "CrawlerRunConfig", "LLMConfig", "ProxyConfig"}
self.assertTrue(required.issubset(self.allowed_types))
def test_allowlist_has_extraction_strategies(self):
required = {
"JsonCssExtractionStrategy", "LLMExtractionStrategy",
"RegexExtractionStrategy",
}
self.assertTrue(required.issubset(self.allowed_types))
def test_allowlist_has_enums(self):
required = {"CacheMode", "MatchMode", "DisplayMode"}
self.assertTrue(required.issubset(self.allowed_types))
if __name__ == '__main__':
print("=" * 60)
print("Crawl4AI Security Fixes - Unit Tests")
print("=" * 60)
print()
unittest.main(verbosity=2)
deploy/docker/tests/test_security_headers_xss.py
+124
View File
@@ -0,0 +1,124 @@
"""
R6 headers / CSP / CORS behavioral tests.
Security headers are emitted unconditionally (independent of security.enabled).
The strict CSP applies to the API / error surface; the still-inline dashboard /
playground keep the baseline headers but not the strict CSP until they are
externalized (CSP-compat refactor - tracked separately). CORS is deny-by-default.
"""
import pytest
pytestmark = pytest.mark.posture
class TestBaselineHeaders:
def test_present_on_api_response(self, stock_client):
r = stock_client.get("/health")
h = {k.lower(): v for k, v in r.headers.items()}
assert h.get("x-content-type-options") == "nosniff"
assert h.get("x-frame-options", "").upper() == "DENY"
assert h.get("referrer-policy") == "no-referrer"
assert h.get("cross-origin-opener-policy") == "same-origin"
def test_present_even_with_security_disabled(self, stock_client, server_module, monkeypatch):
# Headers must be emitted independent of the security.enabled flag (the
# old middleware only added them when enabled). Force it off and verify.
monkeypatch.setitem(server_module.config["security"], "enabled", False)
r = stock_client.get("/health")
h = {k.lower() for k in r.headers}
assert "content-security-policy" in h
assert "x-content-type-options" in h
def test_baseline_on_dashboard_mount(self, stock_client, server_module):
from auth import create_access_token
h = {"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'}, scope='admin')}"}
r = stock_client.get("/dashboard/", headers=h)
hh = {k.lower(): v for k, v in r.headers.items()}
assert hh.get("x-content-type-options") == "nosniff"
assert hh.get("x-frame-options", "").upper() == "DENY"
class TestStrictCsp:
def test_api_csp_is_strict(self, stock_client):
csp = stock_client.get("/health").headers.get("content-security-policy", "")
assert "script-src 'self'" in csp
assert "frame-ancestors 'none'" in csp
assert "default-src 'none'" in csp
assert "unsafe-inline" not in csp
def test_ui_mount_not_given_strict_csp(self, stock_client, server_module):
# The inline-script UI must not get the strict CSP yet (it would break).
from auth import create_access_token
h = {"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'}, scope='admin')}"}
csp = stock_client.get("/playground/", headers=h).headers.get("content-security-policy")
assert csp is None
class TestCorsDenyByDefault:
def test_no_cors_allow_origin_by_default(self, stock_client):
r = stock_client.get("/health", headers={"Origin": "https://evil.example"})
assert "access-control-allow-origin" not in {k.lower() for k in r.headers}
class TestErrorSanitization:
def test_5xx_is_generic_with_correlation_id(self, stock_client, server_module):
from auth import create_access_token
h = {"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'}, scope='admin')}"}
# /monitor/stats/reset has no monitor singleton without a lifespan -> the
# handler raises -> our central handler returns a sanitized 500.
r = stock_client.post("/monitor/stats/reset", headers=h)
assert r.status_code >= 500
body = r.json()
assert body.get("error") == "Internal server error"
assert "correlation_id" in body
# No internal detail (exception text / paths) leaked.
assert "Traceback" not in r.text and "/home/" not in r.text
def test_4xx_detail_preserved(self, stock_client):
# /token is public; with no api_token configured it raises a 4xx, which
# the central handler passes through with its developer-facing detail.
r = stock_client.post("/token", json={"email": "x@y.com"})
assert 400 <= r.status_code < 500
assert "detail" in r.json() and r.json()["detail"]
class TestWebhookHeaderSanitization:
def test_crlf_in_value_rejected(self):
from webhook import sanitize_webhook_headers
with pytest.raises(ValueError):
sanitize_webhook_headers({"X-Foo": "bar\r\nInjected: 1"})
def test_hop_by_hop_denied(self):
from webhook import sanitize_webhook_headers
for bad in ("Host", "Content-Length", "Transfer-Encoding", "Authorization"):
with pytest.raises(ValueError):
sanitize_webhook_headers({bad: "x"})
def test_bad_name_rejected(self):
from webhook import sanitize_webhook_headers
with pytest.raises(ValueError):
sanitize_webhook_headers({"X Foo": "bar"})
def test_good_headers_pass(self):
from webhook import sanitize_webhook_headers
assert sanitize_webhook_headers({"X-Trace-Id": "abc123"}) == {"X-Trace-Id": "abc123"}
def test_schema_validator_rejects_early(self):
from schemas import WebhookConfig
import pydantic
with pytest.raises(pydantic.ValidationError):
WebhookConfig(webhook_url="https://example.com/cb",
webhook_headers={"Host": "evil"})
class TestCRLFSafeLogging:
def test_crlf_stripped_from_log_message(self):
import logging
from utils import CRLFSafeFilter
rec = logging.LogRecord("t", logging.INFO, __file__, 1,
"url=http://x/\r\nINJECTED admin login", None, None)
CRLFSafeFilter().filter(rec)
msg = rec.getMessage()
assert "\r" not in msg and "\n" not in msg
assert "INJECTED" in msg # content kept, just de-fanged
deploy/docker/tests/test_security_llm_broker.py
+78
View File
@@ -0,0 +1,78 @@
"""
R3 LLM-broker tests: a request selects a provider BY NAME only; base_url and
api_token are always server-derived, so the configured provider key can never
be redirected to an attacker endpoint (the reported credential-exfil gadget).
"""
import pytest
from llm_broker import resolve_llm, allowed_provider_families, LLMProviderNotAllowed
pytestmark = pytest.mark.posture
CONF = {"llm": {"provider": "openai/gpt-4o-mini"}}
class TestProviderAllowlist:
def test_default_provider_allowed(self):
out = resolve_llm(CONF, None)
assert out["provider"] == "openai/gpt-4o-mini"
def test_same_family_allowed(self):
out = resolve_llm(CONF, "openai/gpt-4o")
assert out["provider"] == "openai/gpt-4o"
def test_other_family_rejected(self):
with pytest.raises(LLMProviderNotAllowed):
resolve_llm(CONF, "anthropic/claude-3-opus")
def test_explicit_allowlist_widens(self):
conf = {"llm": {"provider": "openai/gpt-4o-mini",
"allowed_providers": ["anthropic/claude-3-opus"]}}
assert resolve_llm(conf, "anthropic/claude-3-opus")["provider"].startswith("anthropic/")
class TestBaseUrlIsServerOnly:
def test_resolve_has_no_request_base_url_param(self):
import inspect
params = list(inspect.signature(resolve_llm).parameters)
# only (config, requested_provider) - no way to inject base_url/api_token
assert params == ["config", "requested_provider"]
def test_base_url_is_canonical_from_env(self, monkeypatch):
monkeypatch.setenv("OPENAI_BASE_URL", "https://api.openai.example")
out = resolve_llm(CONF, "openai/gpt-4o-mini")
assert out["base_url"] == "https://api.openai.example"
def test_no_base_url_when_unset(self, monkeypatch):
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
monkeypatch.delenv("LLM_BASE_URL", raising=False)
out = resolve_llm(CONF, "openai/gpt-4o-mini")
assert out["base_url"] is None
class TestRequestSurfaceRemoved:
def test_markdown_request_has_no_base_url(self):
from schemas import MarkdownRequest
assert "base_url" not in MarkdownRequest.model_fields
def test_llm_job_payload_has_no_base_url(self):
from job import LlmJobPayload
assert "base_url" not in LlmJobPayload.model_fields
def test_llm_endpoint_has_no_base_url_query(self, server_module):
import inspect
assert "base_url" not in inspect.signature(server_module.llm_endpoint).parameters
def test_md_endpoint_rejects_disallowed_provider(self, stock_client, server_module):
from auth import create_access_token
h = {"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'})}"}
# LLM filter + a provider outside the allowed family -> 400 (not a 500,
# and no LLM call to an attacker endpoint).
r = stock_client.post(
"/md",
json={"url": "https://example.com", "f": "llm", "provider": "attacker/model",
"base_url": "http://169.254.169.254"},
headers=h,
)
assert r.status_code == 400, r.status_code
deploy/docker/tests/test_security_resource_caps.py
+175
View File
@@ -0,0 +1,175 @@
"""
R5 resource-governance tests: request body-size limit (413) and the
defense-in-depth deep-crawl clamp. (The client-driven unbounded-crawl critical
is closed upstream by R2 forbidding deep_crawl_strategy on untrusted bodies.)
"""
import pytest
pytestmark = pytest.mark.posture
class TestBodySizeLimit:
def test_oversize_content_length_413(self, stock_client, server_module):
from auth import create_access_token
h = {
"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'})}",
"Content-Length": str(50 * 1024 * 1024), # 50 MiB, over the 10 MiB cap
"Content-Type": "application/json",
}
# TestClient won't actually send 50MiB; the declared Content-Length is
# what the middleware checks.
r = stock_client.post("/crawl", data=b"{}", headers=h)
assert r.status_code == 413, r.status_code
def test_normal_body_not_blocked_by_size(self, stock_client, server_module):
from auth import create_access_token
h = {"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'})}"}
# Small body: must pass the size gate (then rejected for missing urls, etc).
r = stock_client.post("/crawl", json={}, headers=h)
assert r.status_code != 413
class TestDeepCrawlClamp:
def test_infinite_max_pages_clamped(self):
from governor import clamp_deep_crawl, DEFAULT_MAX_PAGES
from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
from crawl4ai import CrawlerRunConfig
strat = BFSDeepCrawlStrategy(max_depth=99) # max_pages defaults to infinity
cfg = CrawlerRunConfig(deep_crawl_strategy=strat)
clamp_deep_crawl(cfg)
assert cfg.deep_crawl_strategy.max_pages <= DEFAULT_MAX_PAGES
assert cfg.deep_crawl_strategy.max_depth <= 5
def test_no_deep_crawl_is_noop(self):
from governor import clamp_deep_crawl
from crawl4ai import CrawlerRunConfig
cfg = CrawlerRunConfig()
clamp_deep_crawl(cfg) # must not raise
assert cfg.deep_crawl_strategy is None
class TestUntrustedDeepCrawlStillForbidden:
def test_request_cannot_set_deep_crawl(self):
# The clamp is defense in depth; the primary control is R2 rejecting it.
from crawl4ai.async_configs import CrawlerRunConfig, Provenance, UntrustedConfigError
with pytest.raises(UntrustedConfigError):
CrawlerRunConfig.load(
{"deep_crawl_strategy": {"type": "BFSDeepCrawlStrategy", "params": {"max_depth": 9}}},
provenance=Provenance.UNTRUSTED,
)
class TestConfigCaps:
def test_job_queue_caps_defaults(self):
from governor import job_queue_caps
caps = job_queue_caps({})
assert caps == {"maxsize": 1000, "workers": 4, "per_principal": 0}
def test_zero_means_unbounded(self):
from governor import job_queue_caps, wall_clock_seconds
caps = job_queue_caps({"limits": {"queue": {"maxsize": 0, "per_principal": 0}}})
assert caps["maxsize"] == 0 and caps["per_principal"] == 0
assert wall_clock_seconds({}) == 0 # no deadline by default
def test_wall_clock_configurable(self):
from governor import wall_clock_seconds
assert wall_clock_seconds({"limits": {"wall_clock_s": 30}}) == 30
@pytest.mark.asyncio
class TestWorkQueue:
async def test_queue_full_without_drain(self):
import asyncio
from work_queue import WorkQueue, QueueFull
q = WorkQueue(maxsize=1, workers=1)
q._q = asyncio.Queue(maxsize=1) # bypass workers so nothing drains
async def noop():
pass
q.submit(noop)
with pytest.raises(QueueFull):
q.submit(noop)
async def test_unbounded_never_full(self):
import asyncio
from work_queue import WorkQueue
q = WorkQueue(maxsize=0, workers=1)
q._q = asyncio.Queue(maxsize=0)
async def noop():
pass
for _ in range(200):
q.submit(noop) # maxsize 0 => unbounded, never raises
async def test_per_principal_quota(self):
import asyncio
from work_queue import WorkQueue, QuotaExceeded
q = WorkQueue(maxsize=0, workers=1, per_principal=1)
q._q = asyncio.Queue(maxsize=0)
async def noop():
pass
q.submit(noop, principal="alice")
with pytest.raises(QuotaExceeded):
q.submit(noop, principal="alice")
q.submit(noop, principal="bob") # different caller is fine
async def test_runs_job_and_releases_counter(self):
import asyncio
from work_queue import WorkQueue
q = WorkQueue(maxsize=10, workers=1, per_principal=5)
await q.start()
try:
done = asyncio.Event()
async def job():
done.set()
q.submit(job, principal="p")
await asyncio.wait_for(done.wait(), timeout=2)
await asyncio.sleep(0.05) # allow the finally/release to run
assert q._counts.get("p", 0) == 0
finally:
await q.stop()
class TestEnqueueMapping:
def test_enqueue_maps_queue_full_to_503(self):
import asyncio
import api
import work_queue
from fastapi import HTTPException
q = work_queue.WorkQueue(maxsize=1, workers=1)
q._q = asyncio.Queue(maxsize=1)
work_queue.set_job_queue(q)
async def noop():
pass
try:
api._enqueue_job(None, noop) # fills the queue
with pytest.raises(HTTPException) as exc:
api._enqueue_job(None, noop)
assert exc.value.status_code == 503
assert exc.value.headers.get("Retry-After")
finally:
work_queue.set_job_queue(None)
def test_enqueue_maps_quota_to_429(self):
import asyncio
import api
import work_queue
from fastapi import HTTPException
q = work_queue.WorkQueue(maxsize=0, workers=1, per_principal=1)
q._q = asyncio.Queue(maxsize=0)
work_queue.set_job_queue(q)
async def noop():
pass
try:
api._enqueue_job(None, noop, principal="a")
with pytest.raises(HTTPException) as exc:
api._enqueue_job(None, noop, principal="a")
assert exc.value.status_code == 429
finally:
work_queue.set_job_queue(None)
deploy/docker/tests/test_security_ssrf_crawl.py
+314
View File
@@ -0,0 +1,314 @@
#!/usr/bin/env python3
"""
Adversarial tests for SSRF protection on crawl/md/llm URL entry points.
Reported by secsys_codex (2026-04-18).
Tests that validate_url_destination() blocks internal IPs on all crawl paths,
and that CRAWL4AI_ALLOW_INTERNAL_URLS=true bypasses the check.
"""
import os
import sys
import unittest
import ipaddress
import socket
from urllib.parse import urlparse
DEPLOY_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
# Local copy of validation logic for self-contained testing
_BLOCKED_NETWORKS = [
ipaddress.ip_network("0.0.0.0/8"),
ipaddress.ip_network("10.0.0.0/8"),
ipaddress.ip_network("100.64.0.0/10"),
ipaddress.ip_network("127.0.0.0/8"),
ipaddress.ip_network("169.254.0.0/16"),
ipaddress.ip_network("172.16.0.0/12"),
ipaddress.ip_network("192.0.0.0/24"),
ipaddress.ip_network("192.168.0.0/16"),
ipaddress.ip_network("198.18.0.0/15"),
ipaddress.ip_network("::1/128"),
ipaddress.ip_network("fc00::/7"),
ipaddress.ip_network("fe80::/10"),
]
_BLOCKED_HOSTNAMES = {
"localhost", "metadata.google.internal", "metadata",
"kubernetes.default", "kubernetes.default.svc",
}
def _expand_ip_candidates(ip):
"""Mirror of utils.py: unwrap IPv4 from IPv4-mapped/compat IPv6."""
candidates = [ip]
if isinstance(ip, ipaddress.IPv6Address):
if ip.ipv4_mapped is not None:
candidates.append(ip.ipv4_mapped)
else:
as_int = int(ip)
if 0 < as_int < 2**32:
candidates.append(ipaddress.IPv4Address(as_int))
return candidates
def _validate_webhook_url(url):
parsed = urlparse(str(url))
hostname = parsed.hostname
if not hostname:
raise ValueError("URL must have a valid hostname")
if hostname.lower() in _BLOCKED_HOSTNAMES:
raise ValueError(f"Hostname '{hostname}' is blocked")
if hostname.lower().startswith("host.docker.internal"):
raise ValueError(f"Hostname '{hostname}' is blocked")
try:
resolved = socket.getaddrinfo(hostname, None)
except socket.gaierror:
raise ValueError(f"Cannot resolve hostname '{hostname}'")
for _, _, _, _, sockaddr in resolved:
ip = ipaddress.ip_address(sockaddr[0])
for candidate in _expand_ip_candidates(ip):
for network in _BLOCKED_NETWORKS:
if candidate in network:
raise ValueError(f"URL resolves to blocked address: {ip}")
def validate_url_destination(url, allow_internal=False):
"""Simulates the actual validate_url_destination from utils.py."""
if allow_internal:
return
if str(url).startswith(("raw:", "raw://")):
return
_validate_webhook_url(url)
# ============================================================================
# SSRF attacks on crawl URLs
# ============================================================================
class TestCrawlURLSSRF(unittest.TestCase):
"""Test SSRF protection on URLs that go to crawler.arun()."""
def test_localhost_blocked(self):
with self.assertRaises(ValueError):
validate_url_destination("http://127.0.0.1:8080/admin")
def test_localhost_name_blocked(self):
with self.assertRaises(ValueError):
validate_url_destination("http://localhost:8080/secret")
def test_10_network_blocked(self):
with self.assertRaises(ValueError):
validate_url_destination("http://10.0.0.1/internal")
def test_172_16_blocked(self):
with self.assertRaises(ValueError):
validate_url_destination("http://172.16.0.1/dashboard")
def test_192_168_blocked(self):
with self.assertRaises(ValueError):
validate_url_destination("http://192.168.1.1/router")
def test_aws_metadata(self):
with self.assertRaises(ValueError):
validate_url_destination("http://169.254.169.254/latest/meta-data/")
def test_gcp_metadata(self):
with self.assertRaises(ValueError):
validate_url_destination("http://metadata.google.internal/computeMetadata/v1/")
def test_docker_internal(self):
with self.assertRaises(ValueError):
validate_url_destination("http://host.docker.internal:3000/api")
def test_kubernetes(self):
with self.assertRaises(ValueError):
validate_url_destination("http://kubernetes.default/api/v1/secrets")
# -- Must allow: external URLs --
def test_external_url_allowed(self):
validate_url_destination("https://example.com")
validate_url_destination("https://www.google.com")
# -- raw: URLs bypass (no network fetch) --
def test_raw_url_bypasses(self):
validate_url_destination("raw:<html><body>hello</body></html>")
validate_url_destination("raw://<html>test</html>")
# -- ALLOW_INTERNAL_URLS opt-out --
def test_allow_internal_bypasses(self):
"""When opted in, internal URLs should pass."""
validate_url_destination("http://127.0.0.1:8080", allow_internal=True)
validate_url_destination("http://10.0.0.1/internal", allow_internal=True)
validate_url_destination("http://169.254.169.254/meta", allow_internal=True)
# ============================================================================
# IPv6-mapped IPv4 bypass (caught in internal security review)
# Prior code only checked blocked networks directly; ::ffff:127.0.0.1
# parses as ::ffff:7f00:1 which is not in 127.0.0.0/8, bypassing the guard.
# ============================================================================
class TestIPv6MappedBypass(unittest.TestCase):
"""Test the IPv4-mapped IPv6 SSRF bypass class."""
def test_mapped_loopback_blocked(self):
"""http://[::ffff:127.0.0.1]/ must be blocked (maps to 127.0.0.1)."""
with self.assertRaises(ValueError):
validate_url_destination("http://[::ffff:127.0.0.1]/admin")
def test_mapped_private_10_blocked(self):
"""::ffff:10.0.0.1 must be blocked (maps to RFC 1918)."""
with self.assertRaises(ValueError):
validate_url_destination("http://[::ffff:10.0.0.1]/")
def test_mapped_aws_metadata_blocked(self):
"""::ffff:169.254.169.254 must be blocked (maps to cloud metadata)."""
with self.assertRaises(ValueError):
validate_url_destination("http://[::ffff:169.254.169.254]/latest/meta-data/")
def test_mapped_192_168_blocked(self):
"""::ffff:192.168.1.1 must be blocked."""
with self.assertRaises(ValueError):
validate_url_destination("http://[::ffff:192.168.1.1]/")
def test_ipv4_compatible_loopback_blocked(self):
"""IPv4-compatible IPv6 (deprecated but still exists): ::127.0.0.1."""
with self.assertRaises(ValueError):
validate_url_destination("http://[::127.0.0.1]/")
def test_regular_ipv6_loopback_still_blocked(self):
"""::1 must remain blocked (IPv6 loopback)."""
with self.assertRaises(ValueError):
validate_url_destination("http://[::1]:8080/")
# ============================================================================
# Source-level verification
# ============================================================================
class TestSSRFSourceCoverage(unittest.TestCase):
"""Verify all URL entry points have SSRF validation."""
def test_server_validate_url_scheme_calls_destination(self):
"""validate_url_scheme must also call validate_url_destination."""
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
source = f.read()
# Find validate_url_scheme function body
idx = source.index("def validate_url_scheme")
func_end = source.index("\ndef ", idx + 1) if "\ndef " in source[idx+1:] else idx + 500
func_body = source[idx:func_end]
self.assertIn("validate_url_destination", func_body,
"validate_url_scheme must call validate_url_destination")
def test_api_py_has_destination_validation(self):
"""api.py must call validate_url_destination for all URL entry points."""
with open(os.path.join(DEPLOY_DIR, "api.py")) as f:
source = f.read()
self.assertIn("validate_url_destination", source,
"api.py must import and use validate_url_destination")
# Count occurrences -- should have at least 4 (one per entry point)
count = source.count("validate_url_destination")
self.assertGreaterEqual(count, 5, # 1 import + 4 call sites
f"api.py should call validate_url_destination at all URL entry points (found {count})")
def test_every_crawl_handler_validates_destination(self):
"""Per-handler check: each crawl handler body must validate seed URL
destinations (via the shared _normalize_and_validate_seeds helper). A
bare occurrence count missed the streaming handler (it had zero calls
while the count was still satisfied)."""
with open(os.path.join(DEPLOY_DIR, "api.py")) as f:
source = f.read()
handlers = [
"handle_crawl_request",
"handle_stream_crawl_request",
]
for handler in handlers:
idx = source.index(f"async def {handler}")
nxt = source.find("\nasync def ", idx + 1)
body = source[idx: nxt if nxt != -1 else len(source)]
self.assertIn("_normalize_and_validate_seeds(", body,
f"{handler} must validate its seed URLs via _normalize_and_validate_seeds")
def test_default_browser_config_pins_egress(self):
"""get_default_browser_config must apply enforce_egress so /html,
/screenshot, /pdf, /execute_js fetch through the pinning proxy
(DNS-rebinding / redirect-to-internal protection), not just validate
the seed and then let the browser re-resolve."""
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
source = f.read()
idx = source.index("def get_default_browser_config")
nxt = source.find("\ndef ", idx + 1)
body = source[idx: nxt if nxt != -1 else idx + 800]
self.assertIn("enforce_egress", body,
"get_default_browser_config must call enforce_egress on the returned config")
def test_llm_and_markdown_handlers_pin_egress(self):
"""The /md and /llm handlers and the LLM background worker must pin
egress on their fetch config (same seed-only SSRF root cause)."""
with open(os.path.join(DEPLOY_DIR, "api.py")) as f:
source = f.read()
for handler in ("handle_llm_qa", "handle_markdown_request", "process_llm_extraction"):
idx = source.index(f"def {handler}")
nxt = source.find("\nasync def ", idx + 1)
nxt2 = source.find("\ndef ", idx + 1)
ends = [e for e in (nxt, nxt2) if e != -1]
body = source[idx: min(ends) if ends else len(source)]
self.assertIn("enforce_egress", body,
f"{handler} must call enforce_egress on its fetch config")
def test_utils_has_allow_internal_flag(self):
"""utils.py must have CRAWL4AI_ALLOW_INTERNAL_URLS env var."""
with open(os.path.join(DEPLOY_DIR, "utils.py")) as f:
source = f.read()
self.assertIn("CRAWL4AI_ALLOW_INTERNAL_URLS", source)
self.assertIn("ALLOW_INTERNAL_URLS", source)
def test_validate_url_destination_skips_raw(self):
"""validate_url_destination must skip raw: URLs."""
with open(os.path.join(DEPLOY_DIR, "utils.py")) as f:
source = f.read()
idx = source.index("def validate_url_destination")
func_body = source[idx:idx+500]
self.assertIn("raw:", func_body,
"validate_url_destination must skip raw: URLs")
import pytest
class TestStreamCrawlSSRFBehavioral:
"""End-to-end: an internal seed URL must be 400-blocked on the streaming
path, not just asserted to exist in source. This is the behavioral guard
for the KOH stream-SSRF fix (and proves the deliberate 400 is not masked
to 500 by the handler's generic except)."""
def _auth(self):
from auth import create_access_token
return {"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'})}"}
INTERNAL = "http://169.254.169.254/latest/meta-data/"
def test_crawl_stream_blocks_internal(self, stock_client, server_module):
r = stock_client.post("/crawl/stream", json={"urls": [self.INTERNAL]}, headers=self._auth())
assert r.status_code == 400, f"expected 400 SSRF block, got {r.status_code}: {r.text[:200]}"
assert "SSRF" in r.text or "blocked" in r.text.lower()
def test_crawl_with_stream_true_blocks_internal(self, stock_client, server_module):
body = {"urls": [self.INTERNAL],
"crawler_config": {"type": "CrawlerRunConfig", "params": {"stream": True}}}
r = stock_client.post("/crawl", json=body, headers=self._auth())
assert r.status_code == 400, f"expected 400 SSRF block, got {r.status_code}: {r.text[:200]}"
def test_non_stream_crawl_still_blocks_internal(self, stock_client, server_module):
r = stock_client.post("/crawl", json={"urls": [self.INTERNAL]}, headers=self._auth())
assert r.status_code == 400, f"expected 400 SSRF block, got {r.status_code}: {r.text[:200]}"
if __name__ == "__main__":
print("=" * 70)
print("Crawl4AI SSRF Tests - Crawl/MD/LLM Endpoints (secsys_codex)")
print("=" * 70)
print()
unittest.main(verbosity=2)
deploy/docker/tests/test_security_ssrf_egress.py
+142
View File
@@ -0,0 +1,142 @@
"""
R3 egress-broker behavioral tests (fully offline via the conftest DNS fixtures).
One rule: reject any resolved IP where not ip.is_global, on every transition
form (v4-mapped / NAT64 / 6to4 / v4-compat). Resolve once and pin, so DNS
rebinding never reaches the second (internal) answer. Errors are opaque.
"""
import socket
import pytest
from egress_broker import (
EgressBlocked,
check_redirect,
is_forbidden_ip,
resolve_and_pin,
)
pytestmark = pytest.mark.posture
class TestForbiddenIp:
@pytest.mark.parametrize("ip", [
"127.0.0.1", "169.254.169.254", "10.0.0.5", "192.168.1.1",
"172.16.0.1", "100.64.0.1", "0.0.0.0",
"::1", "::", "fc00::1", "fe80::1",
"::ffff:127.0.0.1", # v4-mapped loopback
"::ffff:169.254.169.254", # v4-mapped metadata
"64:ff9b::a9fe:a9fe", # NAT64 -> 169.254.169.254
"2002:a9fe:a9fe::1", # 6to4 embedding 169.254.169.254
])
def test_internal_forms_forbidden(self, ip):
assert is_forbidden_ip(ip) is True
@pytest.mark.parametrize("ip", ["8.8.8.8", "1.1.1.1", "2606:4700:4700::1111"])
def test_global_allowed(self, ip):
assert is_forbidden_ip(ip) is False
class TestResolveAndPin:
def test_public_host_pins_ip(self, offline_dns):
offline_dns.set("good.example", "93.184.216.34")
t = resolve_and_pin("https://good.example/path")
assert t.ip == "93.184.216.34" and t.host == "good.example" and t.port == 443
def test_metadata_blocked(self, offline_dns):
offline_dns.set("meta.example", "169.254.169.254")
with pytest.raises(EgressBlocked):
resolve_and_pin("http://meta.example/latest/meta-data/")
def test_nat64_metadata_blocked(self, offline_dns):
offline_dns.set("nat64.example", "64:ff9b::a9fe:a9fe")
with pytest.raises(EgressBlocked):
resolve_and_pin("http://nat64.example/")
def test_localhost_name_blocked(self, offline_dns):
with pytest.raises(EgressBlocked):
resolve_and_pin("http://localhost/")
def test_non_http_scheme_blocked(self):
with pytest.raises(EgressBlocked):
resolve_and_pin("file:///etc/passwd")
with pytest.raises(EgressBlocked):
resolve_and_pin("gopher://x/")
def test_error_is_opaque(self, offline_dns):
offline_dns.set("meta.example", "169.254.169.254")
try:
resolve_and_pin("http://meta.example/")
assert False
except EgressBlocked as e:
# must not leak the resolved IP or hostname
assert "169.254" not in e.reason
assert "meta.example" not in e.reason
assert e.reason == "URL blocked"
def test_multi_record_one_internal_rejects_host(self, monkeypatch):
def fake(host, port, *a, **k):
return [
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", port or 0)),
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", port or 0)),
]
monkeypatch.setattr(socket, "getaddrinfo", fake)
with pytest.raises(EgressBlocked):
resolve_and_pin("http://mixed.example/")
class TestDnsRebinding:
def test_pins_first_answer_never_reresolves_internal(self, rebinding_dns):
r = rebinding_dns("rebind.example", "93.184.216.34", "169.254.169.254")
t = resolve_and_pin("http://rebind.example/")
# Pinned to the public answer; the caller dials t.ip, not the host, so
# the internal second answer is never used.
assert t.ip == "93.184.216.34"
assert r.calls == 1 # resolved exactly once
class TestEnforceEgress:
def test_tls_verification_forced_on(self, monkeypatch):
import egress_broker
monkeypatch.setattr(egress_broker, "ALLOW_INSECURE_TLS", False)
from crawl4ai import BrowserConfig
b = BrowserConfig(ignore_https_errors=True)
egress_broker.enforce_egress(b)
assert b.ignore_https_errors is False
def test_proxy_nulled(self):
import egress_broker
from crawl4ai import BrowserConfig
b = BrowserConfig()
b.proxy_config = object()
egress_broker.enforce_egress(b)
assert b.proxy_config is None
def test_dangerous_args_stripped(self):
import egress_broker
from crawl4ai import BrowserConfig
b = BrowserConfig(extra_args=["--proxy-server=http://evil", "--ignore-certificate-errors", "--headless"])
egress_broker.enforce_egress(b)
assert b.extra_args == ["--headless"]
class TestRedirectRevalidation:
def test_redirect_to_internal_blocked(self, offline_dns):
offline_dns.set("evil-redirect.example", "169.254.169.254")
with pytest.raises(EgressBlocked):
check_redirect("http://evil-redirect.example/")
class TestWebhookValidatorUsesBroker:
def test_webhook_internal_blocked_opaque(self, offline_dns):
from utils import validate_webhook_url
offline_dns.set("hook.example", "10.0.0.9")
with pytest.raises(ValueError) as exc:
validate_webhook_url("http://hook.example/cb")
assert "10.0.0" not in str(exc.value) # no IP leak
def test_webhook_public_ok(self, offline_dns):
from utils import validate_webhook_url
offline_dns.set("hook.example", "93.184.216.34")
validate_webhook_url("http://hook.example/cb") # no raise
deploy/docker/tests/test_security_trust_boundary.py
+236
View File
@@ -0,0 +1,236 @@
"""
R2 trust-boundary behavioral tests (library-level provenance machinery).
Untrusted request bodies may only construct a strict subset of types and may
only set scalar, non-power fields, which are then clamped. Trusted (SDK /
in-process) construction is unchanged.
Docker-layer wiring tests (handlers passing UNTRUSTED, /config/dump
validate-only, declarative hooks) live alongside as the docker pieces land.
"""
import pytest
from crawl4ai.async_configs import (
BrowserConfig,
CrawlerRunConfig,
Provenance,
UntrustedConfigError,
)
T = Provenance.TRUSTED
U = Provenance.UNTRUSTED
pytestmark = pytest.mark.posture
class TestTrustedUnchanged:
"""SDK/in-process callers (default TRUSTED) keep full power."""
def test_trusted_keeps_power_fields(self):
c = CrawlerRunConfig.load(
{"js_code": "x=1", "page_timeout": 999999, "word_count_threshold": 5},
provenance=T,
)
assert c.js_code == "x=1"
assert c.page_timeout == 999999
def test_trusted_is_the_default(self):
# No provenance argument => behaves exactly as before.
c = CrawlerRunConfig.load({"js_code": "y=2"})
assert c.js_code == "y=2"
def test_trusted_roundtrip(self):
c = CrawlerRunConfig(js_code="z=3", screenshot=True, page_timeout=120000)
c2 = CrawlerRunConfig.load(c.dump())
assert c2.js_code == "z=3" and c2.page_timeout == 120000
class TestUntrustedForbiddenFields:
@pytest.mark.parametrize(
"field,value",
[
("js_code", "alert(1)"),
("js_code_before_wait", "x"),
("deep_crawl_strategy", {"type": "BFSDeepCrawlStrategy", "params": {}}),
("proxy_config", {"server": "http://evil"}),
("base_url", "http://evil"),
("simulate_user", True),
("magic", True),
("process_in_browser", True),
("session_id", "shared"),
],
)
def test_crawler_power_field_rejected(self, field, value):
with pytest.raises(UntrustedConfigError):
CrawlerRunConfig.load({field: value}, provenance=U)
@pytest.mark.parametrize(
"field,value",
[
("extra_args", ["--proxy-server=http://evil"]),
("proxy", "http://evil"),
("user_data_dir", "/etc"),
("cdp_url", "ws://evil"),
("cookies", [{"name": "s", "value": "x"}]),
("headers", {"X": "y"}),
("init_scripts", ["evil()"]),
],
)
def test_browser_power_field_rejected(self, field, value):
with pytest.raises(UntrustedConfigError):
BrowserConfig.load({field: value}, provenance=U)
@pytest.mark.parametrize(
"payload",
[
["--no-zygote", "--utility-cmd-prefix=sh -c id>/tmp/pwn"],
["--renderer-cmd-prefix=/bin/sh"],
["--gpu-launcher=touch /tmp/x"],
["--browser-subprocess-path=/bin/sh"],
],
)
def test_extra_args_chromium_cmd_injection_rejected(self, payload):
"""Y4tacker RCE: Chromium launch-arg command injection via extra_args.
extra_args is a forbidden field for untrusted bodies, so ANY value
(including the --*-cmd-prefix / --no-zygote exec primitives) is rejected
outright rather than scrubbed by an always-incomplete denylist."""
with pytest.raises(UntrustedConfigError):
BrowserConfig.load({"extra_args": payload}, provenance=U)
class TestUntrustedTypeGate:
def test_llm_strategy_rejected(self):
body = {
"type": "CrawlerRunConfig",
"params": {
"extraction_strategy": {
"type": "LLMExtractionStrategy",
"params": {},
}
},
}
with pytest.raises(UntrustedConfigError):
CrawlerRunConfig.load(body, provenance=U)
def test_env_exfil_poc_rejected(self):
"""The verified os.getenv-via-'env:' credential-exfil PoC must 400."""
poc = {
"type": "CrawlerRunConfig",
"params": {
"extraction_strategy": {
"type": "LLMExtractionStrategy",
"params": {
"llm_config": {
"type": "LLMConfig",
"params": {"api_token": "env:SECRET_KEY"},
}
},
}
},
}
with pytest.raises(UntrustedConfigError):
CrawlerRunConfig.load(poc, provenance=U)
def test_allowed_non_llm_strategy_accepted(self):
body = {
"type": "CrawlerRunConfig",
"params": {
"extraction_strategy": {
"type": "JsonCssExtractionStrategy",
"params": {"schema": {"name": "x", "baseSelector": "div", "fields": []}},
}
},
}
c = CrawlerRunConfig.load(body, provenance=U)
assert c.extraction_strategy is not None
class TestUntrustedClampAndDrop:
def test_timeout_zero_becomes_cap(self):
c = CrawlerRunConfig.load({"page_timeout": 0}, provenance=U)
assert c.page_timeout == 60_000 # 0 ("no timeout") clamped, never unbounded
def test_timeout_clamped(self):
c = CrawlerRunConfig.load({"wait_for_timeout": 500000}, provenance=U)
assert c.wait_for_timeout == 60_000
def test_unknown_field_dropped_not_raised(self):
c = CrawlerRunConfig.load(
{"css_selector": ".x", "totally_unknown_field": 1}, provenance=U
)
assert c.css_selector == ".x"
def test_viewport_clamped(self):
b = BrowserConfig.load({"viewport_width": 99999, "headless": True}, provenance=U)
assert b.viewport_width == 4000
assert b.headless is True
def test_safe_scalar_kept(self):
c = CrawlerRunConfig.load(
{"word_count_threshold": 7, "screenshot": True, "wait_until": "load"},
provenance=U,
)
assert c.word_count_threshold == 7 and c.screenshot is True
class TestLlmConfigGuard:
def test_untrusted_env_token_rejected(self):
from crawl4ai.async_configs import LLMConfig
with pytest.raises(UntrustedConfigError):
LLMConfig(api_token="env:SECRET_KEY", provenance=U)
def test_untrusted_never_reads_env(self, monkeypatch):
from crawl4ai.async_configs import LLMConfig
monkeypatch.setenv("OPENAI_API_KEY", "leak-me")
cfg = LLMConfig(provider="openai/gpt-4o-mini", provenance=U)
assert cfg.api_token != "leak-me"
class TestDockerHandlersEnforceUntrusted:
"""The Docker endpoints map UntrustedConfigError -> HTTP 400 (not 500)."""
def _auth(self, server_module):
from auth import create_access_token
return {"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'})}"}
def test_crawl_rejects_js_code(self, stock_client, server_module):
r = stock_client.post(
"/crawl",
json={"urls": ["https://example.com"], "crawler_config": {"js_code": "fetch('http://169.254.169.254')"}},
headers=self._auth(server_module),
)
assert r.status_code == 400, r.status_code
def test_crawl_rejects_proxy_config(self, stock_client, server_module):
r = stock_client.post(
"/crawl",
json={"urls": ["https://example.com"],
"browser_config": {"proxy_config": {"server": "http://evil"}}},
headers=self._auth(server_module),
)
assert r.status_code == 400, r.status_code
def test_config_dump_rejects_power_field(self, stock_client, server_module):
r = stock_client.post(
"/config/dump",
json={"type": "BrowserConfig", "params": {"extra_args": ["--proxy-server=x"]}},
headers=self._auth(server_module),
)
assert r.status_code == 400, r.status_code
def test_config_dump_validates_safe_config(self, stock_client, server_module):
r = stock_client.post(
"/config/dump",
json={"type": "CrawlerRunConfig", "params": {"word_count_threshold": 5, "screenshot": True}},
headers=self._auth(server_module),
)
assert r.status_code == 200, r.status_code
def test_config_dump_drops_unknown_keeps_safe(self, stock_client, server_module):
r = stock_client.post(
"/config/dump",
json={"type": "CrawlerRunConfig", "params": {"css_selector": ".x", "bogus_field": 1}},
headers=self._auth(server_module),
)
assert r.status_code == 200, r.status_code
deploy/docker/tests/test_security_webhook_pinning.py
+82
View File
@@ -0,0 +1,82 @@
"""
R3 webhook connection-pinning tests (real loopback server, offline).
The webhook sender pins the connection to the validated IP (aiohttp resolver),
follows redirects manually, and re-validates every hop. We stub the egress
broker so a "good" host pins to the loopback test server and "internal" hosts /
redirects are refused.
"""
import asyncio
import pytest
import egress_broker
from egress_broker import EgressBlocked, PinnedTarget
from webhook import WebhookDeliveryService
pytestmark = pytest.mark.posture
async def _raw_server(response: bytes):
async def handle(reader, writer):
await reader.read(65536)
writer.write(response)
await writer.drain()
writer.close()
server = await asyncio.start_server(handle, "127.0.0.1", 0)
return server, server.sockets[0].getsockname()[1]
def _svc():
return WebhookDeliveryService({"webhooks": {"retry": {"max_attempts": 2,
"initial_delay_ms": 1, "timeout_ms": 5000}}})
@pytest.mark.asyncio
class TestWebhookPinning:
async def test_delivered_to_pinned_host(self, monkeypatch):
server, port = await _raw_server(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
monkeypatch.setattr(
egress_broker, "resolve_and_pin",
lambda url: PinnedTarget("http", "good.example", port, "127.0.0.1"),
)
try:
ok = await _svc().send_webhook(f"http://good.example:{port}/cb", {"x": 1})
assert ok is True
finally:
server.close()
async def test_internal_target_blocked_no_retry(self, monkeypatch):
def boom(url):
raise EgressBlocked()
monkeypatch.setattr(egress_broker, "resolve_and_pin", boom)
ok = await _svc().send_webhook("http://169.254.169.254/cb", {"x": 1})
assert ok is False
async def test_redirect_to_internal_blocked(self, monkeypatch):
server, port = await _raw_server(
b"HTTP/1.1 302 Found\r\nLocation: http://evil.internal/\r\nContent-Length: 0\r\n\r\n"
)
monkeypatch.setattr(
egress_broker, "resolve_and_pin",
lambda url: PinnedTarget("http", "good.example", port, "127.0.0.1"),
)
def check(loc):
raise EgressBlocked() # the redirect target is internal
monkeypatch.setattr(egress_broker, "check_redirect", check)
try:
ok = await _svc().send_webhook(f"http://good.example:{port}/cb", {"x": 1})
assert ok is False # redirect re-validation refused it
finally:
server.close()
async def test_uses_pinned_resolver(self):
# The resolver returns the pinned IP for any host (TLS still verifies the
# hostname); confirm the wiring.
from webhook import _PinnedResolver
r = _PinnedResolver("good.example", "203.0.113.7")
out = await r.resolve("good.example", 443)
assert out[0]["host"] == "203.0.113.7"
assert out[0]["hostname"] == "good.example"