chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,31 @@
|
||||
# .dockerignore
|
||||
*
|
||||
|
||||
# Allow specific files and directories when using local installation
|
||||
!crawl4ai/
|
||||
!docs/
|
||||
!deploy/docker/
|
||||
!setup.py
|
||||
!pyproject.toml
|
||||
!README.md
|
||||
!LICENSE
|
||||
!MANIFEST.in
|
||||
!setup.cfg
|
||||
!mkdocs.yml
|
||||
|
||||
.git/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.pyd
|
||||
.DS_Store
|
||||
.env
|
||||
.venv
|
||||
venv/
|
||||
tests/
|
||||
coverage.xml
|
||||
*.log
|
||||
*.swp
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
@@ -0,0 +1,32 @@
|
||||
# LLM Provider Keys
|
||||
OPENAI_API_KEY=your_openai_key_here
|
||||
DEEPSEEK_API_KEY=your_deepseek_key_here
|
||||
ANTHROPIC_API_KEY=your_anthropic_key_here
|
||||
GROQ_API_KEY=your_groq_key_here
|
||||
TOGETHER_API_KEY=your_together_key_here
|
||||
MISTRAL_API_KEY=your_mistral_key_here
|
||||
GEMINI_API_TOKEN=your_gemini_key_here
|
||||
|
||||
# Optional: Override the default LLM provider
|
||||
# Examples: "openai/gpt-4", "anthropic/claude-3-opus", "deepseek/chat", etc.
|
||||
# If not set, uses the provider specified in config.yml (default: openai/gpt-4o-mini)
|
||||
# LLM_PROVIDER=anthropic/claude-3-opus
|
||||
|
||||
# Optional: Global LLM temperature setting (0.0-2.0)
|
||||
# Controls randomness in responses. Lower = more focused, Higher = more creative
|
||||
# LLM_TEMPERATURE=0.7
|
||||
|
||||
# Optional: Global custom API base URL
|
||||
# Use this to point to custom endpoints or proxy servers
|
||||
# LLM_BASE_URL=https://api.custom.com/v1
|
||||
|
||||
# Optional: Provider-specific temperature overrides
|
||||
# These take precedence over the global LLM_TEMPERATURE
|
||||
# OPENAI_TEMPERATURE=0.5
|
||||
# ANTHROPIC_TEMPERATURE=0.3
|
||||
# GROQ_TEMPERATURE=0.8
|
||||
|
||||
# Optional: Provider-specific base URL overrides
|
||||
# Use for provider-specific proxy endpoints
|
||||
# OPENAI_BASE_URL=https://custom-openai.company.com/v1
|
||||
# GROQ_BASE_URL=https://custom-groq.company.com/v1
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,179 @@
|
||||
# Migration Guide — Docker Server Hardening Release
|
||||
|
||||
This is a major, **secure-by-default** release of the Crawl4AI **Docker API
|
||||
server** (`deploy/docker/`). Several defaults changed in breaking ways so the
|
||||
out-of-the-box deployment is safe. The core pip library (SDK / in-process use)
|
||||
is **unchanged** — these notes apply only to the self-hosted HTTP server.
|
||||
|
||||
How much you have to do scales with how much you drove through the API. A plain
|
||||
"crawl these URLs with a normal config" user only does the two steps in
|
||||
**Everyone**. The rest applies only if you used that specific feature.
|
||||
|
||||
> Upgrading from a self-hosted server? Read this first, then roll out behind a
|
||||
> staging environment. See `SECURITY-VERIFY.md` for the deployment checklist.
|
||||
|
||||
---
|
||||
|
||||
## Everyone (2 steps)
|
||||
|
||||
### 1. Set an API token
|
||||
|
||||
The server no longer serves an unauthenticated API on `0.0.0.0`. It binds
|
||||
loopback by default and will not expose itself without a credential.
|
||||
|
||||
```bash
|
||||
export CRAWL4AI_API_TOKEN="$(openssl rand -hex 32)"
|
||||
```
|
||||
|
||||
- With a token set, you may expose the server (put a TLS-terminating reverse
|
||||
proxy in front) and must send `Authorization: Bearer <token>` on every
|
||||
request except `GET /health`.
|
||||
- With **no** token set, the server binds `127.0.0.1` only and prints a one-off
|
||||
token at startup for local use.
|
||||
|
||||
WebSocket clients (MCP, monitor) that can't set headers may pass `?token=...`.
|
||||
|
||||
### 2. Re-issue any tokens
|
||||
|
||||
The JWT implementation changed; tokens issued by older versions are no longer
|
||||
valid. Re-mint via `POST /token` (which now requires the server to have an
|
||||
`api_token` configured).
|
||||
|
||||
---
|
||||
|
||||
## Only if you used that feature
|
||||
|
||||
### Request bodies accept declarative options only
|
||||
|
||||
A crawl request body now carries scalar, declarative options only. The
|
||||
following are **rejected with HTTP 400** when sent over the network; configure
|
||||
them server-side or run a self-hosted in-process build (the SDK keeps full
|
||||
control):
|
||||
|
||||
`js_code`, `js_code_before_wait`, `c4a_script`, `proxy` / `proxy_config`,
|
||||
`extra_args`, `user_data_dir`, `cdp_url`, `cookies`, `headers`, `init_scripts`,
|
||||
`base_url`, `deep_crawl_strategy`, `simulate_user`, `magic`,
|
||||
`process_in_browser`, and nested LLM config objects.
|
||||
|
||||
Unknown fields are dropped; timeouts, viewport and scroll counts are clamped to
|
||||
safe maximums.
|
||||
|
||||
### Hooks: declarative actions instead of code
|
||||
|
||||
`hooks.code` (Python strings) is replaced by a fixed set of declarative actions:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"hooks": {
|
||||
"hooks": [
|
||||
{"action": "block_resources", "params": {"resource_types": ["image", "font"]}},
|
||||
{"action": "scroll_to_bottom", "params": {"max_steps": 10, "delay_ms": 500}}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Available actions: `block_resources`, `add_cookies`, `set_headers`,
|
||||
`scroll_to_bottom`, `wait_for_timeout`. Call `GET /hooks/info` for the parameter
|
||||
schemas. Arbitrary hook code is available in a self-hosted in-process build.
|
||||
|
||||
### Screenshot / PDF: artifact id instead of `output_path`
|
||||
|
||||
`output_path` is removed. The server stores the result and returns an id + URL:
|
||||
|
||||
```jsonc
|
||||
{"success": true, "screenshot": "<base64>",
|
||||
"artifact_id": "….", "url": "/artifacts/….", "mime": "image/png", "size": 12345}
|
||||
```
|
||||
|
||||
Fetch the file with `GET /artifacts/{artifact_id}` (authenticated). Artifacts
|
||||
have a TTL and a storage quota.
|
||||
|
||||
### LLM endpoints: provider by name
|
||||
|
||||
`base_url` is removed from `/md`, `/llm`, and `/llm/job`. Select a provider by
|
||||
**name** only; the endpoint and key are configured server-side via env
|
||||
(`OPENAI_BASE_URL` / `LLM_BASE_URL`) and `config.llm.allowed_providers`. A
|
||||
provider outside the allowed family returns 400.
|
||||
|
||||
### Monitor actions need an admin token
|
||||
|
||||
`POST /monitor/actions/cleanup|kill_browser|restart_browser` and
|
||||
`/monitor/stats/reset` require an **admin-scope** principal (the static
|
||||
`CRAWL4AI_API_TOKEN` is admin; `/token`-issued JWTs are `data` scope).
|
||||
|
||||
### Browser / JS clients: allowlist your origin (CORS)
|
||||
|
||||
Cross-origin browser requests are denied unless allowlisted:
|
||||
|
||||
```yaml
|
||||
security:
|
||||
cors_allow_origins: ["https://your-frontend.example"]
|
||||
```
|
||||
|
||||
### TLS verification is on
|
||||
|
||||
Self-signed / internal TLS crawl targets now fail by default. For trusted
|
||||
internal testing only: `CRAWL4AI_ALLOW_INSECURE_TLS=true`. Internal-network
|
||||
crawling escape hatch: `CRAWL4AI_ALLOW_INTERNAL_URLS=true`.
|
||||
|
||||
### Webhook headers are validated
|
||||
|
||||
Custom webhook headers must be well-formed names with no control characters and
|
||||
may not set hop-by-hop / sensitive headers (`Host`, `Content-Length`,
|
||||
`Transfer-Encoding`, `Authorization`, `Cookie`, …). Invalid headers → 422.
|
||||
|
||||
### Redis requires a password
|
||||
|
||||
Redis runs in-container, loopback-only, password-protected, and its port is no
|
||||
longer published. For an **external** redis, set `REDIS_PASSWORD`.
|
||||
|
||||
### Resource limits (all configurable; `0` = unbounded)
|
||||
|
||||
```yaml
|
||||
limits:
|
||||
max_body_bytes: 10485760 # request body cap (413); 0 = unbounded
|
||||
wall_clock_s: 0 # per-crawl deadline (504); 0 = none
|
||||
queue:
|
||||
maxsize: 1000 # background job queue (503 when full); 0 = unbounded
|
||||
workers: 4
|
||||
per_principal: 0 # max concurrent jobs per caller (429); 0 = unlimited
|
||||
```
|
||||
|
||||
To keep the previous behavior exactly, set the caps you don't want to `0`.
|
||||
|
||||
### Error responses are generic
|
||||
|
||||
5xx responses return `{"error": "Internal server error", "correlation_id": "…"}`.
|
||||
Match the correlation id in the server logs for detail. Developer-facing 4xx
|
||||
messages are unchanged.
|
||||
|
||||
---
|
||||
|
||||
## Defaults summary
|
||||
|
||||
| Setting | Old | New |
|
||||
| --- | --- | --- |
|
||||
| Bind | `0.0.0.0`, open | `127.0.0.1`; exposing requires a token |
|
||||
| Auth | off by default | on by default |
|
||||
| Security headers / CSP | off | on (strict on the API surface) |
|
||||
| CORS | none | deny-by-default |
|
||||
| TLS verification | disabled | enabled |
|
||||
| Redis | no password, port published | password, loopback, not published |
|
||||
| `output_path` | accepted | removed (artifact store) |
|
||||
| LLM `base_url` in request | honored | removed |
|
||||
| Hooks | Python code | declarative actions |
|
||||
| Background jobs | unbounded | bounded queue (configurable, 0 = unbounded) |
|
||||
|
||||
## Operational notes
|
||||
|
||||
- **`--no-sandbox`** is still set by default (the container runs as non-root
|
||||
without a usable sandbox). To drop it, run the container with an unprivileged
|
||||
user namespace (`unprivileged_userns_clone=1`) or a seccomp profile, then set
|
||||
`CRAWL4AI_CHROMIUM_SANDBOX=true`. See `SECURITY-VERIFY.md`.
|
||||
- The hardened `docker-compose.yml` uses `read_only: true` + tmpfs, `cap_drop:
|
||||
[ALL]`, `no-new-privileges`, and `shm_size` instead of a host `/dev/shm` bind.
|
||||
Mirror these in a custom compose file.
|
||||
- The `/dashboard` and `/playground` UIs get baseline headers
|
||||
(`nosniff`, `X-Frame-Options: DENY`) and are auth-gated; a stricter CSP for
|
||||
the UIs is planned in a follow-up.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,153 @@
|
||||
# Security Hardening — Build/Runtime Verification Runbook
|
||||
|
||||
The offline test suite (`pytest deploy/docker/tests/test_security_*.py`) covers
|
||||
the logic of the hardening. A handful of items can only be confirmed with a real
|
||||
`docker build` + boot or a browser. This runbook lists those, with exact
|
||||
commands and expected results. Run it before relying on the hardened image.
|
||||
|
||||
Prereqs: Docker + docker compose, a checkout of this branch.
|
||||
|
||||
---
|
||||
|
||||
## 0. Offline suite (no Docker) — should already pass
|
||||
|
||||
```bash
|
||||
python -m pip install -e . && pip install -r deploy/docker/requirements.txt pytest pytest-asyncio
|
||||
pytest deploy/docker/tests/test_security_*.py -q
|
||||
```
|
||||
Expected: all pass, `1 xfailed` (the `--no-sandbox` posture test, see §3).
|
||||
|
||||
---
|
||||
|
||||
## 1. Build the hardened image
|
||||
|
||||
```bash
|
||||
IMAGE=local-sec docker compose build
|
||||
# or: docker build -t unclecode/crawl4ai:local-sec .
|
||||
```
|
||||
Expected: build succeeds. Note the `/app` dir is now root-owned + read-only and
|
||||
the artifact dir `/var/lib/crawl4ai/outputs` is created 0700.
|
||||
|
||||
---
|
||||
|
||||
## 2. Boot + bind posture (entrypoint)
|
||||
|
||||
### 2a. No credential -> loopback only (refuses to expose)
|
||||
|
||||
```bash
|
||||
docker run --rm -p 11235:11235 unclecode/crawl4ai:local-sec &
|
||||
sleep 8
|
||||
docker logs <id> 2>&1 | grep -i "binding loopback only" # expect this line
|
||||
curl -fsS http://localhost:11235/health # expect: NOT reachable
|
||||
```
|
||||
Expected: entrypoint logs "no CRAWL4AI_API_TOKEN set; binding loopback only";
|
||||
the mapped port is **not** reachable from the host (gunicorn bound 127.0.0.1
|
||||
inside the container). This is the fail-closed default.
|
||||
|
||||
### 2b. With a credential -> may expose 0.0.0.0
|
||||
|
||||
```bash
|
||||
TOKEN=$(openssl rand -hex 32)
|
||||
docker run --rm -e CRAWL4AI_API_TOKEN=$TOKEN -p 11235:11235 unclecode/crawl4ai:local-sec &
|
||||
sleep 8
|
||||
curl -fsS http://localhost:11235/health # expect 200
|
||||
curl -fsS http://localhost:11235/schema # expect 401
|
||||
curl -fsS -H "Authorization: Bearer $TOKEN" http://localhost:11235/schema # expect 200
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Chromium sandbox (`--no-sandbox`)
|
||||
|
||||
Default keeps `--no-sandbox` (works as today). To verify the hardened path:
|
||||
|
||||
### Option A — unprivileged user namespace (preferred)
|
||||
Host: `sysctl -w kernel.unprivileged_userns_clone=1` (Debian/Ubuntu).
|
||||
```bash
|
||||
docker run --rm -e CRAWL4AI_API_TOKEN=$TOKEN -e CRAWL4AI_CHROMIUM_SANDBOX=true \
|
||||
-p 11235:11235 unclecode/crawl4ai:local-sec &
|
||||
sleep 8
|
||||
curl -fsS -X POST http://localhost:11235/crawl \
|
||||
-H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \
|
||||
-d '{"urls":["https://example.com"]}'
|
||||
```
|
||||
Expected: a successful crawl (Chromium started sandboxed). If it fails to launch,
|
||||
the host lacks userns — keep the default or use Option B.
|
||||
|
||||
### Option B — seccomp profile
|
||||
Provide a Chrome seccomp profile and wire it in compose:
|
||||
```yaml
|
||||
security_opt:
|
||||
- seccomp=./seccomp-chrome.json
|
||||
```
|
||||
then set `CRAWL4AI_CHROMIUM_SANDBOX=true` and re-run the crawl above.
|
||||
|
||||
If verified, flip the default: remove `--no-sandbox` from `config.yml` and the
|
||||
`test_no_no_sandbox_flag` xfail in `test_security_default_posture.py` becomes a
|
||||
normal pass.
|
||||
|
||||
---
|
||||
|
||||
## 4. Read-only rootfs + tmpfs (compose)
|
||||
|
||||
```bash
|
||||
docker compose up -d # uses read_only: true + tmpfs
|
||||
sleep 8
|
||||
# Writes outside tmpfs must fail:
|
||||
docker compose exec crawl4ai sh -c 'echo x > /app/should_fail' ; echo "exit=$?" # expect non-zero
|
||||
# Artifact write (tmpfs) must work via the API:
|
||||
curl -fsS -X POST http://localhost:11235/screenshot \
|
||||
-H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \
|
||||
-d '{"url":"https://example.com"}' | grep artifact_id
|
||||
```
|
||||
Expected: `/app` write fails (read-only), screenshot returns an `artifact_id`,
|
||||
and `GET /artifacts/{id}` (with the token) returns the PNG.
|
||||
|
||||
---
|
||||
|
||||
## 5. Redis is loopback + password-protected, not exposed
|
||||
|
||||
```bash
|
||||
docker compose exec crawl4ai sh -c 'redis-cli -p 6379 ping' # expect: NOAUTH / error
|
||||
docker compose exec crawl4ai sh -c 'redis-cli -a "$REDIS_PASSWORD" ping' # expect: PONG
|
||||
# From the host, the redis port must NOT be reachable (no EXPOSE / publish):
|
||||
nc -z localhost 6379 ; echo "exit=$?" # expect non-zero
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Browser egress proxy (DNS-rebinding control)
|
||||
|
||||
The browser is routed through the localhost pinning proxy. To confirm end to end:
|
||||
```bash
|
||||
# A normal public crawl works:
|
||||
curl -fsS -X POST http://localhost:11235/crawl -H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" -d '{"urls":["https://example.com"]}' | grep '"success": true'
|
||||
# An internal target is refused up front (and the proxy would 403 a rebind):
|
||||
curl -s -X POST http://localhost:11235/crawl -H "Authorization: Bearer $TOKEN" \
|
||||
-H "Content-Type: application/json" -d '{"urls":["http://169.254.169.254/"]}' # expect 400 blocked
|
||||
```
|
||||
(For a true rebinding test, point a host at a TTL-0 record that flips public->
|
||||
internal and confirm the crawl never reaches the internal IP.)
|
||||
|
||||
---
|
||||
|
||||
## 7. Dashboard / playground (browser, CSP)
|
||||
|
||||
Open `http://localhost:11235/dashboard` and `/playground` in a browser (send the
|
||||
token). Expected today: they load and work; response headers include
|
||||
`X-Content-Type-Options: nosniff` and `X-Frame-Options: DENY`, but **no** strict
|
||||
CSP (they still use inline scripts + CDN assets). Extending the strict CSP to
|
||||
these mounts requires the externalization work noted in MIGRATION.md.
|
||||
|
||||
---
|
||||
|
||||
## Sign-off checklist
|
||||
|
||||
- [ ] §0 offline suite green (1 xfail)
|
||||
- [ ] §2a loopback-only without a token; §2b exposed + gated with a token
|
||||
- [ ] §3 Chromium starts sandboxed under userns/seccomp (if pursuing no-sandbox removal)
|
||||
- [ ] §4 `/app` read-only; artifacts work on tmpfs
|
||||
- [ ] §5 redis requires auth + not host-reachable
|
||||
- [ ] §6 public crawl works; internal target blocked
|
||||
- [ ] §7 dashboard/playground load with baseline headers
|
||||
@@ -0,0 +1,241 @@
|
||||
# Crawl4AI Docker Memory & Pool Optimization - Implementation Log
|
||||
|
||||
## Critical Issues Identified
|
||||
|
||||
### Memory Management
|
||||
- **Host vs Container**: `psutil.virtual_memory()` reported host memory, not container limits
|
||||
- **Browser Pooling**: No pool reuse - every endpoint created new browsers
|
||||
- **Warmup Waste**: Permanent browser sat idle with mismatched config signature
|
||||
- **Idle Cleanup**: 30min TTL too long, janitor ran every 60s
|
||||
- **Endpoint Inconsistency**: 75% of endpoints bypassed pool (`/md`, `/html`, `/screenshot`, `/pdf`, `/execute_js`, `/llm`)
|
||||
|
||||
### Pool Design Flaws
|
||||
- **Config Mismatch**: Permanent browser used `config.yml` args, endpoints used empty `BrowserConfig()`
|
||||
- **Logging Level**: Pool hit markers at DEBUG, invisible with INFO logging
|
||||
|
||||
## Implementation Changes
|
||||
|
||||
### 1. Container-Aware Memory Detection (`utils.py`)
|
||||
```python
|
||||
def get_container_memory_percent() -> float:
|
||||
# Try cgroup v2 → v1 → fallback to psutil
|
||||
# Reads /sys/fs/cgroup/memory.{current,max} OR memory/memory.{usage,limit}_in_bytes
|
||||
```
|
||||
|
||||
### 2. Smart Browser Pool (`crawler_pool.py`)
|
||||
**3-Tier System:**
|
||||
- **PERMANENT**: Always-ready default browser (never cleaned)
|
||||
- **HOT_POOL**: Configs used 3+ times (longer TTL)
|
||||
- **COLD_POOL**: New/rare configs (short TTL)
|
||||
|
||||
**Key Functions:**
|
||||
- `get_crawler(cfg)`: Check permanent → hot → cold → create new
|
||||
- `init_permanent(cfg)`: Initialize permanent at startup
|
||||
- `janitor()`: Adaptive cleanup (10s/30s/60s intervals based on memory)
|
||||
- `_sig(cfg)`: SHA1 hash of config dict for pool keys
|
||||
|
||||
**Logging Fix**: Changed `logger.debug()` → `logger.info()` for pool hits
|
||||
|
||||
### 3. Endpoint Unification
|
||||
**Helper Function** (`server.py`):
|
||||
```python
|
||||
def get_default_browser_config() -> BrowserConfig:
|
||||
return BrowserConfig(
|
||||
extra_args=config["crawler"]["browser"].get("extra_args", []),
|
||||
**config["crawler"]["browser"].get("kwargs", {}),
|
||||
)
|
||||
```
|
||||
|
||||
**Migrated Endpoints:**
|
||||
- `/html`, `/screenshot`, `/pdf`, `/execute_js` → use `get_default_browser_config()`
|
||||
- `handle_llm_qa()`, `handle_markdown_request()` → same
|
||||
|
||||
**Result**: All endpoints now hit permanent browser pool
|
||||
|
||||
### 4. Config Updates (`config.yml`)
|
||||
- `idle_ttl_sec: 1800` → `300` (30min → 5min base TTL)
|
||||
- `port: 11234` → `11235` (fixed mismatch with Gunicorn)
|
||||
|
||||
### 5. Lifespan Fix (`server.py`)
|
||||
```python
|
||||
await init_permanent(BrowserConfig(
|
||||
extra_args=config["crawler"]["browser"].get("extra_args", []),
|
||||
**config["crawler"]["browser"].get("kwargs", {}),
|
||||
))
|
||||
```
|
||||
Permanent browser now matches endpoint config signatures
|
||||
|
||||
## Test Results
|
||||
|
||||
### Test 1: Basic Health
|
||||
- 10 requests to `/health`
|
||||
- **Result**: 100% success, avg 3ms latency
|
||||
- **Baseline**: Container starts in ~5s, 270 MB idle
|
||||
|
||||
### Test 2: Memory Monitoring
|
||||
- 20 requests with Docker stats tracking
|
||||
- **Result**: 100% success, no memory leak (-0.2 MB delta)
|
||||
- **Baseline**: 269.7 MB container overhead
|
||||
|
||||
### Test 3: Pool Validation
|
||||
- 30 requests to `/html` endpoint
|
||||
- **Result**: **100% permanent browser hits**, 0 new browsers created
|
||||
- **Memory**: 287 MB baseline → 396 MB active (+109 MB)
|
||||
- **Latency**: Avg 4s (includes network to httpbin.org)
|
||||
|
||||
### Test 4: Concurrent Load
|
||||
- Light (10) → Medium (50) → Heavy (100) concurrent
|
||||
- **Total**: 320 requests
|
||||
- **Result**: 100% success, **320/320 permanent hits**, 0 new browsers
|
||||
- **Memory**: 269 MB → peak 1533 MB → final 993 MB
|
||||
- **Latency**: P99 at 100 concurrent = 34s (expected with single browser)
|
||||
|
||||
### Test 5: Pool Stress (Mixed Configs)
|
||||
- 20 requests with 4 different viewport configs
|
||||
- **Result**: 4 new browsers, 4 cold hits, **4 promotions to hot**, 8 hot hits
|
||||
- **Reuse Rate**: 60% (12 pool hits / 20 requests)
|
||||
- **Memory**: 270 MB → 928 MB peak (+658 MB = ~165 MB per browser)
|
||||
- **Proves**: Cold → hot promotion at 3 uses working perfectly
|
||||
|
||||
### Test 6: Multi-Endpoint
|
||||
- 10 requests each: `/html`, `/screenshot`, `/pdf`, `/crawl`
|
||||
- **Result**: 100% success across all 4 endpoints
|
||||
- **Latency**: 5-8s avg (PDF slowest at 7.2s)
|
||||
|
||||
### Test 7: Cleanup Verification
|
||||
- 20 requests (load spike) → 90s idle
|
||||
- **Memory**: 269 MB → peak 1107 MB → final 780 MB
|
||||
- **Recovery**: 327 MB (39%) - partial cleanup
|
||||
- **Note**: Hot pool browsers persist (by design), janitor working correctly
|
||||
|
||||
## Performance Metrics
|
||||
|
||||
| Metric | Before | After | Improvement |
|
||||
|--------|--------|-------|-------------|
|
||||
| Pool Reuse | 0% | 100% (default config) | ∞ |
|
||||
| Memory Leak | Unknown | 0 MB/cycle | Stable |
|
||||
| Browser Reuse | No | Yes | ~3-5s saved per request |
|
||||
| Idle Memory | 500-700 MB × N | 270-400 MB | 10x reduction |
|
||||
| Concurrent Capacity | ~20 | 100+ | 5x |
|
||||
|
||||
## Key Learnings
|
||||
|
||||
1. **Config Signature Matching**: Permanent browser MUST match endpoint default config exactly (SHA1 hash)
|
||||
2. **Logging Levels**: Pool diagnostics need INFO level, not DEBUG
|
||||
3. **Memory in Docker**: Must read cgroup files, not host metrics
|
||||
4. **Janitor Timing**: 60s interval adequate, but TTLs should be short (5min) for cold pool
|
||||
5. **Hot Promotion**: 3-use threshold works well for production patterns
|
||||
6. **Memory Per Browser**: ~150-200 MB per Chromium instance with headless + text_mode
|
||||
|
||||
## Test Infrastructure
|
||||
|
||||
**Location**: `deploy/docker/tests/`
|
||||
**Dependencies**: `httpx`, `docker` (Python SDK)
|
||||
**Pattern**: Sequential build - each test adds one capability
|
||||
|
||||
**Files**:
|
||||
- `test_1_basic.py`: Health check + container lifecycle
|
||||
- `test_2_memory.py`: + Docker stats monitoring
|
||||
- `test_3_pool.py`: + Log analysis for pool markers
|
||||
- `test_4_concurrent.py`: + asyncio.Semaphore for concurrency control
|
||||
- `test_5_pool_stress.py`: + Config variants (viewports)
|
||||
- `test_6_multi_endpoint.py`: + Multiple endpoint testing
|
||||
- `test_7_cleanup.py`: + Time-series memory tracking for janitor
|
||||
|
||||
**Run Pattern**:
|
||||
```bash
|
||||
cd deploy/docker/tests
|
||||
pip install -r requirements.txt
|
||||
# Rebuild after code changes:
|
||||
cd /path/to/repo && docker buildx build -t crawl4ai-local:latest --load .
|
||||
# Run test:
|
||||
python test_N_name.py
|
||||
```
|
||||
|
||||
## Architecture Decisions
|
||||
|
||||
**Why Permanent Browser?**
|
||||
- 90% of requests use default config → single browser serves most traffic
|
||||
- Eliminates 3-5s startup overhead per request
|
||||
|
||||
**Why 3-Tier Pool?**
|
||||
- Permanent: Zero cost for common case
|
||||
- Hot: Amortized cost for frequent variants
|
||||
- Cold: Lazy allocation for rare configs
|
||||
|
||||
**Why Adaptive Janitor?**
|
||||
- Memory pressure triggers aggressive cleanup
|
||||
- Low memory allows longer TTLs for better reuse
|
||||
|
||||
**Why Not Close After Each Request?**
|
||||
- Browser startup: 3-5s overhead
|
||||
- Pool reuse: <100ms overhead
|
||||
- Net: 30-50x faster
|
||||
|
||||
## Future Optimizations
|
||||
|
||||
1. **Request Queuing**: When at capacity, queue instead of reject
|
||||
2. **Pre-warming**: Predict common configs, pre-create browsers
|
||||
3. **Metrics Export**: Prometheus metrics for pool efficiency
|
||||
4. **Config Normalization**: Group similar viewports (e.g., 1920±50 → 1920)
|
||||
|
||||
## Critical Code Paths
|
||||
|
||||
**Browser Acquisition** (`crawler_pool.py:34-78`):
|
||||
```
|
||||
get_crawler(cfg) →
|
||||
_sig(cfg) →
|
||||
if sig == DEFAULT_CONFIG_SIG → PERMANENT
|
||||
elif sig in HOT_POOL → HOT_POOL[sig]
|
||||
elif sig in COLD_POOL → promote if count >= 3
|
||||
else → create new in COLD_POOL
|
||||
```
|
||||
|
||||
**Janitor Loop** (`crawler_pool.py:107-146`):
|
||||
```
|
||||
while True:
|
||||
mem% = get_container_memory_percent()
|
||||
if mem% > 80: interval=10s, cold_ttl=30s
|
||||
elif mem% > 60: interval=30s, cold_ttl=60s
|
||||
else: interval=60s, cold_ttl=300s
|
||||
sleep(interval)
|
||||
close idle browsers (COLD then HOT)
|
||||
```
|
||||
|
||||
**Endpoint Pattern** (`server.py` example):
|
||||
```python
|
||||
@app.post("/html")
|
||||
async def generate_html(...):
|
||||
from crawler_pool import get_crawler
|
||||
crawler = await get_crawler(get_default_browser_config())
|
||||
results = await crawler.arun(url=body.url, config=cfg)
|
||||
# No crawler.close() - returned to pool
|
||||
```
|
||||
|
||||
## Debugging Tips
|
||||
|
||||
**Check Pool Activity**:
|
||||
```bash
|
||||
docker logs crawl4ai-test | grep -E "(🔥|♨️|❄️|🆕|⬆️)"
|
||||
```
|
||||
|
||||
**Verify Config Signature**:
|
||||
```python
|
||||
from crawl4ai import BrowserConfig
|
||||
import json, hashlib
|
||||
cfg = BrowserConfig(...)
|
||||
sig = hashlib.sha1(json.dumps(cfg.to_dict(), sort_keys=True).encode()).hexdigest()
|
||||
print(sig[:8]) # Compare with logs
|
||||
```
|
||||
|
||||
**Monitor Memory**:
|
||||
```bash
|
||||
docker stats crawl4ai-test
|
||||
```
|
||||
|
||||
## Known Limitations
|
||||
|
||||
- **Mac Docker Stats**: CPU metrics unreliable, memory works
|
||||
- **PDF Generation**: Slowest endpoint (~7s), no optimization yet
|
||||
- **Hot Pool Persistence**: May hold memory longer than needed (trade-off for performance)
|
||||
- **Janitor Lag**: Up to 60s before cleanup triggers in low-memory scenarios
|
||||
@@ -0,0 +1,378 @@
|
||||
# Webhook Feature Examples
|
||||
|
||||
This document provides examples of how to use the webhook feature for crawl jobs in Crawl4AI.
|
||||
|
||||
## Overview
|
||||
|
||||
The webhook feature allows you to receive notifications when crawl jobs complete, eliminating the need for polling. Webhooks are sent with exponential backoff retry logic to ensure reliable delivery.
|
||||
|
||||
## Configuration
|
||||
|
||||
### Global Configuration (config.yml)
|
||||
|
||||
You can configure default webhook settings in `config.yml`:
|
||||
|
||||
```yaml
|
||||
webhooks:
|
||||
enabled: true
|
||||
default_url: null # Optional: default webhook URL for all jobs
|
||||
data_in_payload: false # Optional: default behavior for including data
|
||||
retry:
|
||||
max_attempts: 5
|
||||
initial_delay_ms: 1000 # 1s, 2s, 4s, 8s, 16s exponential backoff
|
||||
max_delay_ms: 32000
|
||||
timeout_ms: 30000 # 30s timeout per webhook call
|
||||
headers: # Optional: default headers to include
|
||||
User-Agent: "Crawl4AI-Webhook/1.0"
|
||||
```
|
||||
|
||||
## API Usage Examples
|
||||
|
||||
### Example 1: Basic Webhook (Notification Only)
|
||||
|
||||
Send a webhook notification without including the crawl data in the payload.
|
||||
|
||||
**Request:**
|
||||
```bash
|
||||
curl -X POST http://localhost:11235/crawl/job \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"urls": ["https://example.com"],
|
||||
"webhook_config": {
|
||||
"webhook_url": "https://myapp.com/webhooks/crawl-complete",
|
||||
"webhook_data_in_payload": false
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"task_id": "crawl_a1b2c3d4"
|
||||
}
|
||||
```
|
||||
|
||||
**Webhook Payload Received:**
|
||||
```json
|
||||
{
|
||||
"task_id": "crawl_a1b2c3d4",
|
||||
"task_type": "crawl",
|
||||
"status": "completed",
|
||||
"timestamp": "2025-10-21T10:30:00.000000+00:00",
|
||||
"urls": ["https://example.com"]
|
||||
}
|
||||
```
|
||||
|
||||
Your webhook handler should then fetch the results:
|
||||
```bash
|
||||
curl http://localhost:11235/crawl/job/crawl_a1b2c3d4
|
||||
```
|
||||
|
||||
### Example 2: Webhook with Data Included
|
||||
|
||||
Include the full crawl results in the webhook payload.
|
||||
|
||||
**Request:**
|
||||
```bash
|
||||
curl -X POST http://localhost:11235/crawl/job \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"urls": ["https://example.com"],
|
||||
"webhook_config": {
|
||||
"webhook_url": "https://myapp.com/webhooks/crawl-complete",
|
||||
"webhook_data_in_payload": true
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
**Webhook Payload Received:**
|
||||
```json
|
||||
{
|
||||
"task_id": "crawl_a1b2c3d4",
|
||||
"task_type": "crawl",
|
||||
"status": "completed",
|
||||
"timestamp": "2025-10-21T10:30:00.000000+00:00",
|
||||
"urls": ["https://example.com"],
|
||||
"data": {
|
||||
"markdown": "...",
|
||||
"html": "...",
|
||||
"links": {...},
|
||||
"metadata": {...}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Example 3: Webhook with Custom Headers
|
||||
|
||||
Include custom headers for authentication or identification.
|
||||
|
||||
**Request:**
|
||||
```bash
|
||||
curl -X POST http://localhost:11235/crawl/job \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"urls": ["https://example.com"],
|
||||
"webhook_config": {
|
||||
"webhook_url": "https://myapp.com/webhooks/crawl-complete",
|
||||
"webhook_data_in_payload": false,
|
||||
"webhook_headers": {
|
||||
"X-Webhook-Secret": "my-secret-token",
|
||||
"X-Service-ID": "crawl4ai-production"
|
||||
}
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
The webhook will be sent with these additional headers plus the default headers from config.
|
||||
|
||||
### Example 4: Failure Notification
|
||||
|
||||
When a crawl job fails, a webhook is sent with error details.
|
||||
|
||||
**Webhook Payload on Failure:**
|
||||
```json
|
||||
{
|
||||
"task_id": "crawl_a1b2c3d4",
|
||||
"task_type": "crawl",
|
||||
"status": "failed",
|
||||
"timestamp": "2025-10-21T10:30:00.000000+00:00",
|
||||
"urls": ["https://example.com"],
|
||||
"error": "Connection timeout after 30s"
|
||||
}
|
||||
```
|
||||
|
||||
### Example 5: Using Global Default Webhook
|
||||
|
||||
If you set a `default_url` in config.yml, jobs without webhook_config will use it:
|
||||
|
||||
**config.yml:**
|
||||
```yaml
|
||||
webhooks:
|
||||
enabled: true
|
||||
default_url: "https://myapp.com/webhooks/default"
|
||||
data_in_payload: false
|
||||
```
|
||||
|
||||
**Request (no webhook_config needed):**
|
||||
```bash
|
||||
curl -X POST http://localhost:11235/crawl/job \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"urls": ["https://example.com"]
|
||||
}'
|
||||
```
|
||||
|
||||
The webhook will be sent to the default URL configured in config.yml.
|
||||
|
||||
### Example 6: LLM Extraction Job with Webhook
|
||||
|
||||
Use webhooks with the LLM extraction endpoint for asynchronous processing.
|
||||
|
||||
**Request:**
|
||||
```bash
|
||||
curl -X POST http://localhost:11235/llm/job \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"url": "https://example.com/article",
|
||||
"q": "Extract the article title, author, and publication date",
|
||||
"schema": "{\"type\": \"object\", \"properties\": {\"title\": {\"type\": \"string\"}, \"author\": {\"type\": \"string\"}, \"date\": {\"type\": \"string\"}}}",
|
||||
"cache": false,
|
||||
"provider": "openai/gpt-4o-mini",
|
||||
"webhook_config": {
|
||||
"webhook_url": "https://myapp.com/webhooks/llm-complete",
|
||||
"webhook_data_in_payload": true
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
**Response:**
|
||||
```json
|
||||
{
|
||||
"task_id": "llm_1698765432_12345"
|
||||
}
|
||||
```
|
||||
|
||||
**Webhook Payload Received:**
|
||||
```json
|
||||
{
|
||||
"task_id": "llm_1698765432_12345",
|
||||
"task_type": "llm_extraction",
|
||||
"status": "completed",
|
||||
"timestamp": "2025-10-21T10:30:00.000000+00:00",
|
||||
"urls": ["https://example.com/article"],
|
||||
"data": {
|
||||
"extracted_content": {
|
||||
"title": "Understanding Web Scraping",
|
||||
"author": "John Doe",
|
||||
"date": "2025-10-21"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Webhook Handler Example
|
||||
|
||||
Here's a simple Python Flask webhook handler that supports both crawl and LLM extraction jobs:
|
||||
|
||||
```python
|
||||
from flask import Flask, request, jsonify
|
||||
import requests
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
@app.route('/webhooks/crawl-complete', methods=['POST'])
|
||||
def handle_crawl_webhook():
|
||||
payload = request.json
|
||||
|
||||
task_id = payload['task_id']
|
||||
task_type = payload['task_type']
|
||||
status = payload['status']
|
||||
|
||||
if status == 'completed':
|
||||
# If data not in payload, fetch it
|
||||
if 'data' not in payload:
|
||||
# Determine endpoint based on task type
|
||||
endpoint = 'crawl' if task_type == 'crawl' else 'llm'
|
||||
response = requests.get(f'http://localhost:11235/{endpoint}/job/{task_id}')
|
||||
data = response.json()
|
||||
else:
|
||||
data = payload['data']
|
||||
|
||||
# Process based on task type
|
||||
if task_type == 'crawl':
|
||||
print(f"Processing crawl results for {task_id}")
|
||||
# Handle crawl results
|
||||
results = data.get('results', [])
|
||||
for result in results:
|
||||
print(f" - {result.get('url')}: {len(result.get('markdown', ''))} chars")
|
||||
|
||||
elif task_type == 'llm_extraction':
|
||||
print(f"Processing LLM extraction for {task_id}")
|
||||
# Handle LLM extraction
|
||||
# Note: Webhook sends 'extracted_content', API returns 'result'
|
||||
extracted = data.get('extracted_content', data.get('result', {}))
|
||||
print(f" - Extracted: {extracted}")
|
||||
|
||||
# Your business logic here...
|
||||
|
||||
elif status == 'failed':
|
||||
error = payload.get('error', 'Unknown error')
|
||||
print(f"{task_type} job {task_id} failed: {error}")
|
||||
# Handle failure...
|
||||
|
||||
return jsonify({"status": "received"}), 200
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(port=8080)
|
||||
```
|
||||
|
||||
## Retry Logic
|
||||
|
||||
The webhook delivery service uses exponential backoff retry logic:
|
||||
|
||||
- **Attempts:** Up to 5 attempts by default
|
||||
- **Delays:** 1s → 2s → 4s → 8s → 16s
|
||||
- **Timeout:** 30 seconds per attempt
|
||||
- **Retry Conditions:**
|
||||
- Server errors (5xx status codes)
|
||||
- Network errors
|
||||
- Timeouts
|
||||
- **No Retry:**
|
||||
- Client errors (4xx status codes)
|
||||
- Successful delivery (2xx status codes)
|
||||
|
||||
## Benefits
|
||||
|
||||
1. **No Polling Required** - Eliminates constant API calls to check job status
|
||||
2. **Real-time Notifications** - Immediate notification when jobs complete
|
||||
3. **Reliable Delivery** - Exponential backoff ensures webhooks are delivered
|
||||
4. **Flexible** - Choose between notification-only or full data delivery
|
||||
5. **Secure** - Support for custom headers for authentication
|
||||
6. **Configurable** - Global defaults or per-job configuration
|
||||
7. **Universal Support** - Works with both `/crawl/job` and `/llm/job` endpoints
|
||||
|
||||
## TypeScript Client Example
|
||||
|
||||
```typescript
|
||||
interface WebhookConfig {
|
||||
webhook_url: string;
|
||||
webhook_data_in_payload?: boolean;
|
||||
webhook_headers?: Record<string, string>;
|
||||
}
|
||||
|
||||
interface CrawlJobRequest {
|
||||
urls: string[];
|
||||
browser_config?: Record<string, any>;
|
||||
crawler_config?: Record<string, any>;
|
||||
webhook_config?: WebhookConfig;
|
||||
}
|
||||
|
||||
interface LLMJobRequest {
|
||||
url: string;
|
||||
q: string;
|
||||
schema?: string;
|
||||
cache?: boolean;
|
||||
provider?: string;
|
||||
webhook_config?: WebhookConfig;
|
||||
}
|
||||
|
||||
async function createCrawlJob(request: CrawlJobRequest) {
|
||||
const response = await fetch('http://localhost:11235/crawl/job', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(request)
|
||||
});
|
||||
|
||||
const { task_id } = await response.json();
|
||||
return task_id;
|
||||
}
|
||||
|
||||
async function createLLMJob(request: LLMJobRequest) {
|
||||
const response = await fetch('http://localhost:11235/llm/job', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(request)
|
||||
});
|
||||
|
||||
const { task_id } = await response.json();
|
||||
return task_id;
|
||||
}
|
||||
|
||||
// Usage - Crawl Job
|
||||
const crawlTaskId = await createCrawlJob({
|
||||
urls: ['https://example.com'],
|
||||
webhook_config: {
|
||||
webhook_url: 'https://myapp.com/webhooks/crawl-complete',
|
||||
webhook_data_in_payload: false,
|
||||
webhook_headers: {
|
||||
'X-Webhook-Secret': 'my-secret'
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Usage - LLM Extraction Job
|
||||
const llmTaskId = await createLLMJob({
|
||||
url: 'https://example.com/article',
|
||||
q: 'Extract the main points from this article',
|
||||
provider: 'openai/gpt-4o-mini',
|
||||
webhook_config: {
|
||||
webhook_url: 'https://myapp.com/webhooks/llm-complete',
|
||||
webhook_data_in_payload: true,
|
||||
webhook_headers: {
|
||||
'X-Webhook-Secret': 'my-secret'
|
||||
}
|
||||
}
|
||||
});
|
||||
```
|
||||
|
||||
## Monitoring and Debugging
|
||||
|
||||
Webhook delivery attempts are logged at INFO level:
|
||||
- Successful deliveries
|
||||
- Retry attempts with delays
|
||||
- Final failures after max attempts
|
||||
|
||||
Check the application logs for webhook delivery status:
|
||||
```bash
|
||||
docker logs crawl4ai-container | grep -i webhook
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,166 @@
|
||||
"""
|
||||
artifacts.py - sandboxed, opaque-ID artifact store.
|
||||
|
||||
The screenshot/PDF endpoints used to accept a caller-supplied `output_path` and
|
||||
write to it. `validate_output_path` was string-only (no realpath / O_NOFOLLOW),
|
||||
so a symlink or a sibling-prefix name (".../outputs-evil") bypassed it and gave
|
||||
an arbitrary write -> root RCE (e.g. /etc/cron.d) since /app was writable.
|
||||
|
||||
The fix: callers never name a path. The server owns the directory, the names,
|
||||
and the bytes. Each artifact gets a server-generated opaque id; writes are
|
||||
O_EXCL|O_NOFOLLOW 0600 into a 0700 dir off /tmp, size-capped and dir-quota'd;
|
||||
retrieval validates a 32-hex id, lstat()s a non-symlink regular file, and
|
||||
enforces a TTL. A janitor reaps expired/over-quota files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from typing import Dict, Tuple
|
||||
|
||||
ARTIFACT_DIR = os.environ.get("CRAWL4AI_ARTIFACT_DIR", "/var/lib/crawl4ai/outputs")
|
||||
MAX_ARTIFACT_BYTES = int(os.environ.get("CRAWL4AI_MAX_ARTIFACT_BYTES", str(50 * 1024 * 1024)))
|
||||
ARTIFACT_QUOTA_BYTES = int(os.environ.get("CRAWL4AI_ARTIFACT_QUOTA_BYTES", str(2 * 1024 * 1024 * 1024)))
|
||||
ARTIFACT_TTL_SECONDS = int(os.environ.get("CRAWL4AI_ARTIFACT_TTL_SECONDS", "3600"))
|
||||
|
||||
_HEX32 = re.compile(r"^[0-9a-f]{32}$")
|
||||
_KIND = {
|
||||
"png": (".png", "image/png"),
|
||||
"pdf": (".pdf", "application/pdf"),
|
||||
}
|
||||
|
||||
|
||||
class ArtifactError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class ArtifactTooLarge(ArtifactError):
|
||||
pass
|
||||
|
||||
|
||||
class QuotaExceeded(ArtifactError):
|
||||
pass
|
||||
|
||||
|
||||
class ArtifactNotFound(ArtifactError):
|
||||
pass
|
||||
|
||||
|
||||
def init_store() -> None:
|
||||
"""Create the artifact dir 0700 (idempotent)."""
|
||||
os.makedirs(ARTIFACT_DIR, mode=0o700, exist_ok=True)
|
||||
try:
|
||||
os.chmod(ARTIFACT_DIR, 0o700)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _dir_size() -> int:
|
||||
total = 0
|
||||
try:
|
||||
with os.scandir(ARTIFACT_DIR) as it:
|
||||
for entry in it:
|
||||
try:
|
||||
st = entry.stat(follow_symlinks=False)
|
||||
if os.path.stat.S_ISREG(st.st_mode):
|
||||
total += st.st_size
|
||||
except OSError:
|
||||
continue
|
||||
except FileNotFoundError:
|
||||
return 0
|
||||
return total
|
||||
|
||||
|
||||
def write_artifact(kind: str, data: bytes) -> Dict:
|
||||
"""Write bytes under a server-generated opaque id. Returns metadata."""
|
||||
ext, mime = _KIND.get(kind, (".bin", "application/octet-stream"))
|
||||
if len(data) > MAX_ARTIFACT_BYTES:
|
||||
raise ArtifactTooLarge(f"artifact exceeds {MAX_ARTIFACT_BYTES} bytes")
|
||||
init_store()
|
||||
if _dir_size() + len(data) > ARTIFACT_QUOTA_BYTES:
|
||||
raise QuotaExceeded("artifact storage quota exceeded")
|
||||
|
||||
artifact_id = uuid.uuid4().hex # 32 hex chars, unguessable
|
||||
path = os.path.join(ARTIFACT_DIR, artifact_id + ext)
|
||||
# O_EXCL: never follow/overwrite an existing entry; O_NOFOLLOW: refuse a
|
||||
# symlink at the final component. Server-chosen name => no traversal.
|
||||
flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0)
|
||||
fd = os.open(path, flags, 0o600)
|
||||
with os.fdopen(fd, "wb") as f:
|
||||
f.write(data)
|
||||
return {"artifact_id": artifact_id, "mime": mime, "size": len(data)}
|
||||
|
||||
|
||||
def resolve_artifact(artifact_id: str) -> Tuple[str, str]:
|
||||
"""Map an opaque id to (path, mime), enforcing hex-id, no-symlink, TTL.
|
||||
|
||||
Raises ArtifactNotFound for an invalid id, a non-regular/symlink file, a
|
||||
missing file, or an expired artifact (so existence is never revealed).
|
||||
"""
|
||||
if not isinstance(artifact_id, str) or not _HEX32.match(artifact_id):
|
||||
raise ArtifactNotFound()
|
||||
for ext, mime in _KIND.values():
|
||||
path = os.path.join(ARTIFACT_DIR, artifact_id + ext)
|
||||
try:
|
||||
st = os.lstat(path) # do NOT follow symlinks
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
if not os.path.stat.S_ISREG(st.st_mode):
|
||||
raise ArtifactNotFound() # symlink / special file
|
||||
if time.time() - st.st_mtime > ARTIFACT_TTL_SECONDS:
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
raise ArtifactNotFound()
|
||||
return path, mime
|
||||
raise ArtifactNotFound()
|
||||
|
||||
|
||||
def janitor() -> int:
|
||||
"""Reap expired artifacts; if still over quota, reap oldest first."""
|
||||
init_store()
|
||||
now = time.time()
|
||||
reaped = 0
|
||||
entries = []
|
||||
try:
|
||||
with os.scandir(ARTIFACT_DIR) as it:
|
||||
for entry in it:
|
||||
try:
|
||||
st = entry.stat(follow_symlinks=False)
|
||||
except OSError:
|
||||
continue
|
||||
if not os.path.stat.S_ISREG(st.st_mode):
|
||||
# remove anything that is not a regular file (e.g. a symlink)
|
||||
try:
|
||||
os.unlink(entry.path)
|
||||
reaped += 1
|
||||
except OSError:
|
||||
pass
|
||||
continue
|
||||
if now - st.st_mtime > ARTIFACT_TTL_SECONDS:
|
||||
try:
|
||||
os.unlink(entry.path)
|
||||
reaped += 1
|
||||
except OSError:
|
||||
pass
|
||||
continue
|
||||
entries.append((st.st_mtime, st.st_size, entry.path))
|
||||
except FileNotFoundError:
|
||||
return reaped
|
||||
|
||||
total = sum(sz for _, sz, _ in entries)
|
||||
if total > ARTIFACT_QUOTA_BYTES:
|
||||
for _mtime, sz, path in sorted(entries): # oldest first
|
||||
if total <= ARTIFACT_QUOTA_BYTES:
|
||||
break
|
||||
try:
|
||||
os.unlink(path)
|
||||
total -= sz
|
||||
reaped += 1
|
||||
except OSError:
|
||||
pass
|
||||
return reaped
|
||||
@@ -0,0 +1,146 @@
|
||||
"""
|
||||
Authentication primitives for the Crawl4AI Docker server.
|
||||
|
||||
This module is PyJWT-only. The previous dual dependency on the GehirnInc `jwt`
|
||||
package *and* `PyJWT` (both install a top-level `jwt` module) meant the meaning
|
||||
of `import jwt` depended on install order, and the security tests could exercise
|
||||
a different code path than production. We now depend on PyJWT exclusively.
|
||||
|
||||
Auth is decided in one place: the AuthGateMiddleware (auth_gate.py), which runs
|
||||
as the outermost ASGI layer and fails closed. The helpers here are what that
|
||||
gate (and the /token endpoint) call:
|
||||
|
||||
* create_access_token - mint an HS256 JWT carrying a scope claim
|
||||
* decode_token - verify an HS256 JWT (algorithms passed as a LIST,
|
||||
which kills the substring-match bug and rejects
|
||||
alg:none / other algorithms)
|
||||
* constant_time_eq - timing-safe comparison for the static API token
|
||||
* resolve_secret_key - fail fast when a real secret is required but missing
|
||||
"""
|
||||
|
||||
import hmac
|
||||
import logging
|
||||
import os
|
||||
import secrets as _secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Dict, Optional
|
||||
|
||||
import jwt
|
||||
from fastapi import HTTPException, Request
|
||||
from pydantic import BaseModel, EmailStr
|
||||
|
||||
ACCESS_TOKEN_EXPIRE_MINUTES = 60
|
||||
ALGORITHM = "HS256"
|
||||
_ALGORITHMS = [ALGORITHM] # a LIST on purpose: no substring matching, no alg:none
|
||||
|
||||
_WEAK_SECRETS = {"mysecret", "secret", "password", "changeme", "test", "12345678"}
|
||||
_MIN_SECRET_LEN = 32
|
||||
|
||||
_log = logging.getLogger("crawl4ai.security")
|
||||
|
||||
|
||||
def resolve_secret_key(*, required: bool) -> str:
|
||||
"""Resolve and validate SECRET_KEY.
|
||||
|
||||
required=True -> fail fast (RuntimeError) if unset/weak/short. Used when a
|
||||
real auth deployment is in effect; an ephemeral key would
|
||||
silently invalidate every issued token on restart.
|
||||
required=False -> auto-generate an ephemeral key (and warn) when unset, so
|
||||
loopback/dev still works. A set-but-weak key still fails.
|
||||
"""
|
||||
key = os.environ.get("SECRET_KEY", "")
|
||||
if key:
|
||||
if key.lower() in _WEAK_SECRETS:
|
||||
raise RuntimeError(
|
||||
"FATAL: SECRET_KEY is a known weak value. Generate a strong one: "
|
||||
'python3 -c "import secrets; print(secrets.token_hex(32))"'
|
||||
)
|
||||
if len(key) < _MIN_SECRET_LEN:
|
||||
raise RuntimeError(
|
||||
f"FATAL: SECRET_KEY must be at least {_MIN_SECRET_LEN} characters. "
|
||||
'Generate one: python3 -c "import secrets; print(secrets.token_hex(32))"'
|
||||
)
|
||||
return key
|
||||
|
||||
if required:
|
||||
raise RuntimeError(
|
||||
"FATAL: authentication is enabled but SECRET_KEY is not set. "
|
||||
'Set it: SECRET_KEY=$(python3 -c "import secrets; print(secrets.token_hex(32))")'
|
||||
)
|
||||
|
||||
generated = _secrets.token_hex(32)
|
||||
_log.warning(
|
||||
"No SECRET_KEY set. Auto-generated an ephemeral key (changes on restart, "
|
||||
"invalidating issued tokens). Set SECRET_KEY for any real deployment."
|
||||
)
|
||||
return generated
|
||||
|
||||
|
||||
# Module-level key, resolved leniently at import. The server's startup
|
||||
# _resolve_auth() performs the fail-fast check when a real deployment is
|
||||
# detected (credential set and/or non-loopback bind).
|
||||
SECRET_KEY = resolve_secret_key(required=False)
|
||||
|
||||
|
||||
def create_access_token(
|
||||
data: dict,
|
||||
*,
|
||||
scope: str = "data",
|
||||
expires_delta: Optional[timedelta] = None,
|
||||
) -> str:
|
||||
"""Mint an HS256 JWT. `scope` is "data" (normal) or "admin"."""
|
||||
to_encode = dict(data)
|
||||
to_encode["scope"] = scope
|
||||
expire = datetime.now(timezone.utc) + (
|
||||
expires_delta or timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)
|
||||
)
|
||||
to_encode["exp"] = expire
|
||||
return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
|
||||
|
||||
|
||||
def decode_token(token: str) -> Dict:
|
||||
"""Verify an HS256 JWT and return its claims.
|
||||
|
||||
Raises jwt.InvalidTokenError (incl. ExpiredSignatureError) on any failure.
|
||||
`algorithms` is a list, so alg:none and every non-HS256 algorithm are
|
||||
rejected outright.
|
||||
"""
|
||||
return jwt.decode(token, SECRET_KEY, algorithms=_ALGORITHMS)
|
||||
|
||||
|
||||
def constant_time_eq(a: str, b: str) -> bool:
|
||||
"""Timing-safe string comparison for the static API token."""
|
||||
return hmac.compare_digest(a.encode("utf-8"), b.encode("utf-8"))
|
||||
|
||||
|
||||
def get_principal(request: Request) -> Optional[Dict]:
|
||||
"""The principal the AuthGateMiddleware already validated (or None)."""
|
||||
return getattr(request.state, "principal", None)
|
||||
|
||||
|
||||
def get_token_dependency(config: Dict):
|
||||
"""Backward-compatible dependency factory.
|
||||
|
||||
Auth enforcement now lives in the AuthGateMiddleware (the outermost ASGI
|
||||
layer); by the time any route dependency runs, the request was already
|
||||
authenticated by the gate or rejected with 401. This dependency simply
|
||||
surfaces the validated principal to handlers that declared `_td`.
|
||||
"""
|
||||
|
||||
def _principal(request: Request) -> Optional[Dict]:
|
||||
return get_principal(request)
|
||||
|
||||
return _principal
|
||||
|
||||
|
||||
def require_admin(request: Request) -> Dict:
|
||||
"""Dependency: require an admin-scope principal (destructive actions)."""
|
||||
principal = get_principal(request)
|
||||
if not principal or principal.get("scope") != "admin":
|
||||
raise HTTPException(status_code=403, detail="Admin scope required")
|
||||
return principal
|
||||
|
||||
|
||||
class TokenRequest(BaseModel):
|
||||
email: EmailStr
|
||||
api_token: Optional[str] = None
|
||||
@@ -0,0 +1,126 @@
|
||||
"""
|
||||
AuthGateMiddleware - the single, fail-closed authentication boundary.
|
||||
|
||||
The previous design decided auth in a per-route FastAPI dependency that, when
|
||||
`jwt_enabled` was false (the default), returned `lambda: None` - so every
|
||||
`Depends(token_dep)` was decorative and the whole API was open. Static mounts,
|
||||
the MCP transports and the Prometheus endpoint were never covered at all.
|
||||
|
||||
This middleware moves auth to the outermost ASGI layer so it covers EVERY
|
||||
route, mount and sub-app (HTTP + WebSocket) uniformly, and it fails closed: a
|
||||
request without a valid credential is rejected before it reaches any handler.
|
||||
|
||||
Accepted credentials:
|
||||
* the static operator API token (constant-time compared) -> admin scope, or
|
||||
* a valid HS256 JWT minted by this server -> the token's own scope claim.
|
||||
|
||||
Public paths (the health check and the token-issuing endpoint) pass through.
|
||||
Public prefixes (the UI static shells) also pass through - they serve no data.
|
||||
On failure: HTTP 401 JSON, or WebSocket close 4401.
|
||||
On success: the validated principal is attached at scope["state"]["principal"]
|
||||
(readable downstream as request.state.principal) for scope/ownership checks.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Callable, Dict, Iterable, Optional
|
||||
from urllib.parse import parse_qs
|
||||
|
||||
import jwt
|
||||
|
||||
from auth import constant_time_eq, decode_token
|
||||
|
||||
|
||||
class AuthGateMiddleware:
|
||||
def __init__(
|
||||
self,
|
||||
app,
|
||||
*,
|
||||
token_provider: Callable[[], str],
|
||||
public_paths: Iterable[str] = (),
|
||||
public_prefixes: Iterable[str] = (),
|
||||
):
|
||||
self.app = app
|
||||
self._token_provider = token_provider
|
||||
self.public_paths = set(public_paths)
|
||||
self.public_prefixes = tuple(public_prefixes)
|
||||
|
||||
# ─────────────────────────── ASGI entry ───────────────────────────
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] not in ("http", "websocket"):
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
|
||||
path = scope.get("path", "")
|
||||
if path in self.public_paths:
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
if self.public_prefixes and path.startswith(self.public_prefixes):
|
||||
await self.app(scope, receive, send)
|
||||
return
|
||||
|
||||
principal = self._authenticate(scope)
|
||||
if principal is None:
|
||||
await self._reject(scope, receive, send)
|
||||
return
|
||||
|
||||
# Expose the principal to downstream handlers/dependencies.
|
||||
state = scope.setdefault("state", {})
|
||||
state["principal"] = principal
|
||||
await self.app(scope, receive, send)
|
||||
|
||||
# ──────────────────────────── helpers ─────────────────────────────
|
||||
def _authenticate(self, scope) -> Optional[Dict]:
|
||||
token = self._extract_token(scope)
|
||||
if not token:
|
||||
return None
|
||||
|
||||
# 1) static operator token -> admin scope
|
||||
static_token = self._token_provider() or ""
|
||||
if static_token and constant_time_eq(token, static_token):
|
||||
return {"sub": "operator", "scope": "admin", "via": "api_token"}
|
||||
|
||||
# 2) HS256 JWT minted by this server
|
||||
try:
|
||||
claims = decode_token(token)
|
||||
except jwt.InvalidTokenError:
|
||||
return None
|
||||
claims.setdefault("scope", "data")
|
||||
return claims
|
||||
|
||||
@staticmethod
|
||||
def _extract_token(scope) -> Optional[str]:
|
||||
# Authorization: Bearer <token>
|
||||
for name, value in scope.get("headers", []):
|
||||
if name == b"authorization":
|
||||
raw = value.decode("latin-1")
|
||||
if raw[:7].lower() == "bearer ":
|
||||
return raw[7:].strip()
|
||||
return None
|
||||
# WebSocket clients that cannot set headers may pass ?token=
|
||||
if scope["type"] == "websocket":
|
||||
qs = parse_qs(scope.get("query_string", b"").decode("latin-1"))
|
||||
vals = qs.get("token")
|
||||
if vals:
|
||||
return vals[0]
|
||||
return None
|
||||
|
||||
async def _reject(self, scope, receive, send):
|
||||
if scope["type"] == "websocket":
|
||||
# Close before accept; 4401 = application "unauthorized".
|
||||
await send({"type": "websocket.close", "code": 4401})
|
||||
return
|
||||
body = json.dumps({"detail": "Authentication required"}).encode()
|
||||
await send(
|
||||
{
|
||||
"type": "http.response.start",
|
||||
"status": 401,
|
||||
"headers": [
|
||||
(b"content-type", b"application/json"),
|
||||
(b"www-authenticate", b"Bearer"),
|
||||
(b"content-length", str(len(body)).encode()),
|
||||
],
|
||||
}
|
||||
)
|
||||
await send({"type": "http.response.body", "body": body})
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,133 @@
|
||||
# Application Configuration
|
||||
app:
|
||||
title: "Crawl4AI API"
|
||||
version: "1.0.0"
|
||||
# Loopback by default. Exposing on 0.0.0.0 requires a credential
|
||||
# (CRAWL4AI_API_TOKEN); the startup auth guard refuses an open non-loopback
|
||||
# bind without one. Put a TLS-terminating reverse proxy in front for exposure.
|
||||
host: "127.0.0.1"
|
||||
port: 11235
|
||||
reload: False
|
||||
workers: 1
|
||||
timeout_keep_alive: 300
|
||||
|
||||
# Default LLM Configuration
|
||||
llm:
|
||||
provider: "openai/gpt-4o-mini"
|
||||
# api_key: sk-... # If you pass the API key directly (not recommended)
|
||||
|
||||
# Redis Configuration
|
||||
# Set task_ttl_seconds to automatically expire task data in Redis.
|
||||
# This prevents unbounded memory growth from accumulated task results.
|
||||
# Default: 3600 (1 hour). Set to 0 to disable TTL (not recommended).
|
||||
# Can be overridden with REDIS_TASK_TTL environment variable.
|
||||
redis:
|
||||
host: "localhost"
|
||||
port: 6379
|
||||
db: 0
|
||||
password: ""
|
||||
task_ttl_seconds: 3600 # TTL for task data (1 hour default)
|
||||
ssl: False
|
||||
ssl_cert_reqs: None
|
||||
ssl_ca_certs: None
|
||||
ssl_certfile: None
|
||||
ssl_keyfile: None
|
||||
|
||||
# Resource governance (DoS protection). Any limit set to 0 means "unbounded"
|
||||
# (the pre-hardening behavior), so you can relax or disable each one.
|
||||
limits:
|
||||
max_body_bytes: 10485760 # 10 MiB request body cap (413 if exceeded); 0 = unbounded
|
||||
max_pages: 100 # deep-crawl page budget clamp (defense in depth)
|
||||
max_depth: 5 # deep-crawl depth clamp
|
||||
wall_clock_s: 0 # per-crawl deadline in seconds (504 on timeout); 0 = no deadline
|
||||
queue: # background job queue for /crawl/job and /llm/job
|
||||
maxsize: 1000 # max queued jobs (503 when full); 0 = unbounded
|
||||
workers: 4 # concurrent background workers
|
||||
per_principal: 0 # max concurrent jobs per caller (429 over cap); 0 = unlimited
|
||||
|
||||
# Rate Limiting Configuration
|
||||
rate_limiting:
|
||||
enabled: True
|
||||
default_limit: "1000/minute"
|
||||
trusted_proxies: []
|
||||
storage_uri: "memory://" # Use "redis://localhost:6379" for production
|
||||
|
||||
# Security Configuration
|
||||
# WARNING: For production deployments, enable security and use proper SECRET_KEY:
|
||||
# - Set jwt_enabled: true for authentication
|
||||
# - Set SECRET_KEY environment variable to a secure random value
|
||||
# - Set CRAWL4AI_HOOKS_ENABLED=true only if you need hooks (RCE risk)
|
||||
security:
|
||||
enabled: true
|
||||
jwt_enabled: false
|
||||
api_token: "" # When set, /token endpoint requires this secret to issue JWTs
|
||||
https_redirect: false
|
||||
trusted_hosts: ["*"]
|
||||
cors_allow_origins: [] # deny-by-default; list explicit origins to allow CORS
|
||||
headers:
|
||||
x_content_type_options: "nosniff"
|
||||
x_frame_options: "DENY"
|
||||
content_security_policy: "default-src 'self'"
|
||||
strict_transport_security: "max-age=63072000; includeSubDomains"
|
||||
|
||||
# Crawler Configuration
|
||||
crawler:
|
||||
base_config:
|
||||
simulate_user: true
|
||||
memory_threshold_percent: 95.0
|
||||
rate_limiter:
|
||||
enabled: true
|
||||
base_delay: [1.0, 2.0]
|
||||
timeouts:
|
||||
stream_init: 30.0 # Timeout for stream initialization
|
||||
batch_process: 300.0 # Timeout for batch processing
|
||||
pool:
|
||||
max_pages: 40 # ← GLOBAL_SEM permits
|
||||
idle_ttl_sec: 300 # ← 30 min janitor cutoff
|
||||
browser:
|
||||
kwargs:
|
||||
headless: true
|
||||
text_mode: true
|
||||
extra_args:
|
||||
# - "--single-process"
|
||||
# SECURITY: --no-sandbox disables the Chromium renderer sandbox (a renderer
|
||||
# exploit can then reach the container). It is required only because the
|
||||
# container runs as non-root without a usable sandbox. To remove it, run
|
||||
# the container with an unprivileged user namespace
|
||||
# (host: unprivileged_userns_clone=1) OR a verified seccomp profile
|
||||
# (compose: security_opt: seccomp=...), then delete this flag and confirm
|
||||
# Chromium still starts. Tracked as a build-gated hardening item.
|
||||
- "--no-sandbox"
|
||||
- "--disable-dev-shm-usage"
|
||||
- "--disable-gpu"
|
||||
- "--disable-software-rasterizer"
|
||||
# --disable-web-security removed for security (enables cross-origin access)
|
||||
# --allow-insecure-localhost / --ignore-certificate-errors removed: they
|
||||
# disabled TLS verification for every crawl (MITM / SSRF-to-internal-TLS).
|
||||
# Set CRAWL4AI_ALLOW_INSECURE_TLS=true only for trusted internal testing.
|
||||
|
||||
# Logging Configuration
|
||||
logging:
|
||||
level: "INFO"
|
||||
format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||||
|
||||
# Observability Configuration
|
||||
observability:
|
||||
prometheus:
|
||||
enabled: True
|
||||
endpoint: "/metrics"
|
||||
health_check:
|
||||
endpoint: "/health"
|
||||
|
||||
# Webhook Configuration
|
||||
webhooks:
|
||||
enabled: true
|
||||
default_url: null # Optional: default webhook URL for all jobs
|
||||
data_in_payload: false # Optional: default behavior for including data
|
||||
retry:
|
||||
max_attempts: 5
|
||||
initial_delay_ms: 1000 # 1s, 2s, 4s, 8s, 16s exponential backoff
|
||||
max_delay_ms: 32000
|
||||
timeout_ms: 30000 # 30s timeout per webhook call
|
||||
headers: # Optional: default headers to include
|
||||
User-Agent: "Crawl4AI-Webhook/1.0"
|
||||
@@ -0,0 +1,220 @@
|
||||
# crawler_pool.py - Smart browser pool with tiered management
|
||||
import asyncio, json, hashlib, time
|
||||
from contextlib import suppress
|
||||
from typing import Dict, Optional
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig
|
||||
from utils import load_config, get_container_memory_percent
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
CONFIG = load_config()
|
||||
|
||||
# Pool tiers
|
||||
PERMANENT: Optional[AsyncWebCrawler] = None # Always-ready default browser
|
||||
HOT_POOL: Dict[str, AsyncWebCrawler] = {} # Frequent configs
|
||||
COLD_POOL: Dict[str, AsyncWebCrawler] = {} # Rare configs
|
||||
LAST_USED: Dict[str, float] = {}
|
||||
USAGE_COUNT: Dict[str, int] = {}
|
||||
LOCK = asyncio.Lock()
|
||||
|
||||
# Config
|
||||
MEM_LIMIT = CONFIG.get("crawler", {}).get("memory_threshold_percent", 95.0)
|
||||
BASE_IDLE_TTL = CONFIG.get("crawler", {}).get("pool", {}).get("idle_ttl_sec", 300)
|
||||
DEFAULT_CONFIG_SIG = None # Cached sig for default config
|
||||
|
||||
|
||||
def get_pool_snapshot() -> dict:
|
||||
"""Return a point-in-time snapshot of pool state for monitoring.
|
||||
|
||||
This is intentionally lock-free. Under CPython's GIL, reading
|
||||
``len(dict)``, ``dict.copy()``, and ``x is not None`` are atomic
|
||||
operations, so the monitor can safely call this without contending
|
||||
on the pool LOCK that is held during slow browser start/close ops.
|
||||
The worst case is a slightly stale count, which is acceptable for
|
||||
dashboard display purposes.
|
||||
"""
|
||||
return {
|
||||
"permanent": PERMANENT,
|
||||
"permanent_sig": DEFAULT_CONFIG_SIG,
|
||||
"hot_pool": HOT_POOL.copy(),
|
||||
"cold_pool": COLD_POOL.copy(),
|
||||
"last_used": LAST_USED.copy(),
|
||||
"usage_count": USAGE_COUNT.copy(),
|
||||
}
|
||||
|
||||
|
||||
def _sig(cfg: BrowserConfig) -> str:
|
||||
"""Generate config signature."""
|
||||
payload = json.dumps(cfg.to_dict(), sort_keys=True, separators=(",",":"))
|
||||
return hashlib.sha1(payload.encode()).hexdigest()
|
||||
|
||||
def _is_default_config(sig: str) -> bool:
|
||||
"""Check if config matches default."""
|
||||
return sig == DEFAULT_CONFIG_SIG
|
||||
|
||||
async def get_crawler(cfg: BrowserConfig) -> AsyncWebCrawler:
|
||||
"""Get crawler from pool with tiered strategy."""
|
||||
sig = _sig(cfg)
|
||||
async with LOCK:
|
||||
# Check permanent browser for default config
|
||||
if PERMANENT and _is_default_config(sig):
|
||||
LAST_USED[sig] = time.time()
|
||||
USAGE_COUNT[sig] = USAGE_COUNT.get(sig, 0) + 1
|
||||
if not hasattr(PERMANENT, 'active_requests'):
|
||||
PERMANENT.active_requests = 0
|
||||
PERMANENT.active_requests += 1
|
||||
logger.info("🔥 Using permanent browser")
|
||||
return PERMANENT
|
||||
|
||||
# Check hot pool
|
||||
if sig in HOT_POOL:
|
||||
LAST_USED[sig] = time.time()
|
||||
USAGE_COUNT[sig] = USAGE_COUNT.get(sig, 0) + 1
|
||||
crawler = HOT_POOL[sig]
|
||||
if not hasattr(crawler, 'active_requests'):
|
||||
crawler.active_requests = 0
|
||||
crawler.active_requests += 1
|
||||
logger.info(f"♨️ Using hot pool browser (sig={sig[:8]}, active={crawler.active_requests})")
|
||||
return crawler
|
||||
|
||||
# Check cold pool (promote to hot if used 3+ times)
|
||||
if sig in COLD_POOL:
|
||||
LAST_USED[sig] = time.time()
|
||||
USAGE_COUNT[sig] = USAGE_COUNT.get(sig, 0) + 1
|
||||
crawler = COLD_POOL[sig]
|
||||
if not hasattr(crawler, 'active_requests'):
|
||||
crawler.active_requests = 0
|
||||
crawler.active_requests += 1
|
||||
|
||||
if USAGE_COUNT[sig] >= 3:
|
||||
logger.info(f"⬆️ Promoting to hot pool (sig={sig[:8]}, count={USAGE_COUNT[sig]})")
|
||||
HOT_POOL[sig] = COLD_POOL.pop(sig)
|
||||
|
||||
# Track promotion in monitor
|
||||
try:
|
||||
from monitor import get_monitor
|
||||
await get_monitor().track_janitor_event("promote", sig, {"count": USAGE_COUNT[sig]})
|
||||
except:
|
||||
pass
|
||||
|
||||
return HOT_POOL[sig]
|
||||
|
||||
logger.info(f"❄️ Using cold pool browser (sig={sig[:8]})")
|
||||
return crawler
|
||||
|
||||
# Memory check before creating new
|
||||
mem_pct = get_container_memory_percent()
|
||||
if mem_pct >= MEM_LIMIT:
|
||||
logger.error(f"💥 Memory pressure: {mem_pct:.1f}% >= {MEM_LIMIT}%")
|
||||
raise MemoryError(f"Memory at {mem_pct:.1f}%, refusing new browser")
|
||||
|
||||
# Create new in cold pool
|
||||
logger.info(f"🆕 Creating new browser in cold pool (sig={sig[:8]}, mem={mem_pct:.1f}%)")
|
||||
crawler = AsyncWebCrawler(config=cfg, thread_safe=False)
|
||||
await crawler.start()
|
||||
crawler.active_requests = 1
|
||||
COLD_POOL[sig] = crawler
|
||||
LAST_USED[sig] = time.time()
|
||||
USAGE_COUNT[sig] = 1
|
||||
return crawler
|
||||
|
||||
async def release_crawler(crawler: AsyncWebCrawler):
|
||||
"""Decrement active request count for a pooled crawler.
|
||||
|
||||
Call this in a finally block after finishing work with a crawler
|
||||
obtained via get_crawler() so the janitor knows when it's safe
|
||||
to close idle browsers.
|
||||
"""
|
||||
async with LOCK:
|
||||
if hasattr(crawler, 'active_requests'):
|
||||
crawler.active_requests = max(0, crawler.active_requests - 1)
|
||||
|
||||
async def init_permanent(cfg: BrowserConfig):
|
||||
"""Initialize permanent default browser."""
|
||||
global PERMANENT, DEFAULT_CONFIG_SIG
|
||||
async with LOCK:
|
||||
if PERMANENT:
|
||||
return
|
||||
DEFAULT_CONFIG_SIG = _sig(cfg)
|
||||
logger.info("🔥 Creating permanent default browser")
|
||||
PERMANENT = AsyncWebCrawler(config=cfg, thread_safe=False)
|
||||
await PERMANENT.start()
|
||||
LAST_USED[DEFAULT_CONFIG_SIG] = time.time()
|
||||
USAGE_COUNT[DEFAULT_CONFIG_SIG] = 0
|
||||
|
||||
async def close_all():
|
||||
"""Close all browsers."""
|
||||
async with LOCK:
|
||||
tasks = []
|
||||
if PERMANENT:
|
||||
tasks.append(PERMANENT.close())
|
||||
tasks.extend([c.close() for c in HOT_POOL.values()])
|
||||
tasks.extend([c.close() for c in COLD_POOL.values()])
|
||||
await asyncio.gather(*tasks, return_exceptions=True)
|
||||
HOT_POOL.clear()
|
||||
COLD_POOL.clear()
|
||||
LAST_USED.clear()
|
||||
USAGE_COUNT.clear()
|
||||
|
||||
async def janitor():
|
||||
"""Adaptive cleanup based on memory pressure."""
|
||||
while True:
|
||||
mem_pct = get_container_memory_percent()
|
||||
|
||||
# Adaptive intervals and TTLs
|
||||
if mem_pct > 80:
|
||||
interval, cold_ttl, hot_ttl = 10, 30, 120
|
||||
elif mem_pct > 60:
|
||||
interval, cold_ttl, hot_ttl = 30, 60, 300
|
||||
else:
|
||||
interval, cold_ttl, hot_ttl = 60, BASE_IDLE_TTL, BASE_IDLE_TTL * 2
|
||||
|
||||
await asyncio.sleep(interval)
|
||||
|
||||
now = time.time()
|
||||
async with LOCK:
|
||||
# Clean cold pool
|
||||
for sig in list(COLD_POOL.keys()):
|
||||
if now - LAST_USED.get(sig, now) > cold_ttl:
|
||||
crawler = COLD_POOL[sig]
|
||||
if getattr(crawler, 'active_requests', 0) > 0:
|
||||
continue # still serving requests, skip
|
||||
idle_time = now - LAST_USED[sig]
|
||||
logger.info(f"🧹 Closing cold browser (sig={sig[:8]}, idle={idle_time:.0f}s)")
|
||||
with suppress(Exception):
|
||||
await crawler.close()
|
||||
COLD_POOL.pop(sig, None)
|
||||
LAST_USED.pop(sig, None)
|
||||
USAGE_COUNT.pop(sig, None)
|
||||
|
||||
# Track in monitor
|
||||
try:
|
||||
from monitor import get_monitor
|
||||
await get_monitor().track_janitor_event("close_cold", sig, {"idle_seconds": int(idle_time), "ttl": cold_ttl})
|
||||
except:
|
||||
pass
|
||||
|
||||
# Clean hot pool (more conservative)
|
||||
for sig in list(HOT_POOL.keys()):
|
||||
if now - LAST_USED.get(sig, now) > hot_ttl:
|
||||
crawler = HOT_POOL[sig]
|
||||
if getattr(crawler, 'active_requests', 0) > 0:
|
||||
continue # still serving requests, skip
|
||||
idle_time = now - LAST_USED[sig]
|
||||
logger.info(f"🧹 Closing hot browser (sig={sig[:8]}, idle={idle_time:.0f}s)")
|
||||
with suppress(Exception):
|
||||
await crawler.close()
|
||||
HOT_POOL.pop(sig, None)
|
||||
LAST_USED.pop(sig, None)
|
||||
USAGE_COUNT.pop(sig, None)
|
||||
|
||||
# Track in monitor
|
||||
try:
|
||||
from monitor import get_monitor
|
||||
await get_monitor().track_janitor_event("close_hot", sig, {"idle_seconds": int(idle_time), "ttl": hot_ttl})
|
||||
except:
|
||||
pass
|
||||
|
||||
# Log pool stats
|
||||
if mem_pct > 60:
|
||||
logger.info(f"📊 Pool: hot={len(HOT_POOL)}, cold={len(COLD_POOL)}, mem={mem_pct:.1f}%")
|
||||
@@ -0,0 +1,209 @@
|
||||
"""
|
||||
egress_broker.py - the single rule and primitive for all outbound traffic.
|
||||
|
||||
Every SSRF defense before this was enumerate-badness (a hand-maintained list of
|
||||
blocked CIDRs) and resolve-then-discard: the validator resolved a hostname,
|
||||
checked the IP against the list, then threw the IP away and let the real
|
||||
connection re-resolve (DNS rebinding / TOCTOU). It also missed whole families:
|
||||
the IPv6 unspecified ::, NAT64 64:ff9b::/96, 6to4 2002::/16, and
|
||||
host.docker.internal as anything but a literal prefix.
|
||||
|
||||
This module replaces the blocklist with one rule:
|
||||
|
||||
reject any resolved IP where `not ip.is_global`
|
||||
|
||||
evaluated on every transition-embedded IPv4 form as well (v4-mapped, NAT64,
|
||||
6to4, v4-compatible), and makes the entity that resolves DNS the same entity
|
||||
that hands back the pinned IP to dial. `resolve_and_pin()` resolves once and
|
||||
returns the exact IP to connect to; callers must dial that IP (Host/SNI
|
||||
preserved) so a second, attacker-controlled resolution is never used.
|
||||
|
||||
Errors are opaque (`EgressBlocked.reason == "URL blocked"`) so the API never
|
||||
leaks a resolved internal IP, hostname, or traceback (the old DNS oracle).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import os
|
||||
import socket
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# Operator escape hatch for trusted internal deployments (off by default).
|
||||
ALLOW_INTERNAL = os.environ.get("CRAWL4AI_ALLOW_INTERNAL_URLS", "false").lower() == "true"
|
||||
|
||||
_NAT64 = ipaddress.ip_network("64:ff9b::/96")
|
||||
_V4COMPAT = ipaddress.ip_network("::/96")
|
||||
_6TO4 = ipaddress.ip_network("2002::/16")
|
||||
|
||||
# Hostnames we refuse regardless of resolution (belt-and-suspenders; they also
|
||||
# resolve to non-global addresses and would be caught anyway).
|
||||
_BLOCKED_HOSTNAMES = {
|
||||
"localhost", "metadata.google.internal", "metadata",
|
||||
"kubernetes.default", "kubernetes.default.svc",
|
||||
}
|
||||
|
||||
|
||||
class EgressBlocked(Exception):
|
||||
"""Outbound target rejected. Carries an OPAQUE reason - never an IP/host."""
|
||||
|
||||
def __init__(self, reason: str = "URL blocked"):
|
||||
self.reason = reason
|
||||
super().__init__(reason)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PinnedTarget:
|
||||
scheme: str
|
||||
host: str # original hostname (for Host header / SNI)
|
||||
port: int
|
||||
ip: str # the exact IP to dial - do NOT re-resolve `host`
|
||||
|
||||
|
||||
def _embedded_v4_forms(ip: ipaddress._BaseAddress) -> List[ipaddress._BaseAddress]:
|
||||
"""The address plus any IPv4 embedded in a transition IPv6 form.
|
||||
|
||||
Only the well-defined transition ranges are unwrapped, so a normal global
|
||||
IPv6 is never mis-derived into a bogus (and possibly non-global) IPv4.
|
||||
"""
|
||||
forms: List[ipaddress._BaseAddress] = [ip]
|
||||
if isinstance(ip, ipaddress.IPv6Address):
|
||||
mapped = ip.ipv4_mapped
|
||||
if mapped is not None:
|
||||
forms.append(mapped)
|
||||
elif ip in _NAT64 or ip in _V4COMPAT:
|
||||
forms.append(ipaddress.IPv4Address(int(ip) & 0xFFFFFFFF))
|
||||
elif ip in _6TO4:
|
||||
forms.append(ipaddress.IPv4Address((int(ip) >> 80) & 0xFFFFFFFF))
|
||||
return forms
|
||||
|
||||
|
||||
def is_forbidden_ip(ip_str: str) -> bool:
|
||||
"""True if the IP (or any embedded transition form) is not globally routable."""
|
||||
try:
|
||||
ip = ipaddress.ip_address(ip_str)
|
||||
except ValueError:
|
||||
return True
|
||||
return any(not form.is_global for form in _embedded_v4_forms(ip))
|
||||
|
||||
|
||||
def _resolve(host: str, port: int):
|
||||
try:
|
||||
return socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)
|
||||
except socket.gaierror:
|
||||
raise EgressBlocked()
|
||||
|
||||
|
||||
def assert_host_allowed(host: str, port: int = 0) -> None:
|
||||
"""Resolve `host` and reject if ANY answer is non-global. Opaque on failure."""
|
||||
if ALLOW_INTERNAL:
|
||||
return
|
||||
if not host:
|
||||
raise EgressBlocked()
|
||||
low = host.lower()
|
||||
if low in _BLOCKED_HOSTNAMES or low.startswith("host.docker.internal"):
|
||||
raise EgressBlocked()
|
||||
for *_, sockaddr in _resolve(host, port):
|
||||
if is_forbidden_ip(sockaddr[0]):
|
||||
raise EgressBlocked()
|
||||
|
||||
|
||||
def resolve_and_pin(url: str) -> PinnedTarget:
|
||||
"""Resolve `url` once, reject if any answer is non-global, and pin one IP.
|
||||
|
||||
The returned PinnedTarget.ip is the address the caller must dial; resolving
|
||||
`host` again at connect time would reopen the rebinding hole.
|
||||
"""
|
||||
parsed = urlparse(str(url))
|
||||
scheme = (parsed.scheme or "").lower()
|
||||
if scheme not in ("http", "https"):
|
||||
raise EgressBlocked()
|
||||
host = parsed.hostname
|
||||
if not host:
|
||||
raise EgressBlocked()
|
||||
port = parsed.port or (443 if scheme == "https" else 80)
|
||||
|
||||
if ALLOW_INTERNAL:
|
||||
# Still resolve so we can pin, but skip the global check.
|
||||
answers = _resolve(host, port)
|
||||
return PinnedTarget(scheme, host, port, answers[0][4][0])
|
||||
|
||||
low = host.lower()
|
||||
if low in _BLOCKED_HOSTNAMES or low.startswith("host.docker.internal"):
|
||||
raise EgressBlocked()
|
||||
|
||||
answers = _resolve(host, port)
|
||||
pinned = None
|
||||
for *_, sockaddr in answers:
|
||||
if is_forbidden_ip(sockaddr[0]):
|
||||
# Reject the host outright if ANY of its records is internal.
|
||||
raise EgressBlocked()
|
||||
if pinned is None:
|
||||
pinned = sockaddr[0]
|
||||
if pinned is None:
|
||||
raise EgressBlocked()
|
||||
return PinnedTarget(scheme, host, port, pinned)
|
||||
|
||||
|
||||
def check_redirect(location: str) -> PinnedTarget:
|
||||
"""Re-validate (and pin) a redirect Location. Same rule as the initial hop."""
|
||||
return resolve_and_pin(location)
|
||||
|
||||
|
||||
ALLOW_INSECURE_TLS = os.environ.get("CRAWL4AI_ALLOW_INSECURE_TLS", "false").lower() == "true"
|
||||
|
||||
# URL of the localhost pinning forward-proxy (egress_proxy.py), set at boot.
|
||||
# When present, enforce_egress routes the browser through it so Chromium never
|
||||
# resolves the target itself (closes DNS rebinding on the browser path).
|
||||
_EGRESS_PROXY_URL: Optional[str] = None
|
||||
|
||||
|
||||
def set_egress_proxy(url: Optional[str]) -> None:
|
||||
global _EGRESS_PROXY_URL
|
||||
_EGRESS_PROXY_URL = url
|
||||
|
||||
|
||||
def get_egress_proxy() -> Optional[str]:
|
||||
return _EGRESS_PROXY_URL
|
||||
|
||||
# Chromium flags that would re-route or weaken egress; scrubbed server-side.
|
||||
_DANGEROUS_BROWSER_ARGS = (
|
||||
"--proxy-server", "--proxy-pac-url", "--proxy-bypass-list",
|
||||
"--host-resolver-rules", "--ignore-certificate-errors",
|
||||
"--allow-insecure-localhost",
|
||||
)
|
||||
|
||||
|
||||
def enforce_egress(browser_config) -> None:
|
||||
"""Server-side egress hardening applied to the effective browser config.
|
||||
|
||||
R2 already forbids untrusted bodies from setting proxy/extra_args, so this
|
||||
is defense in depth that also covers server/SDK-built configs:
|
||||
- TLS verification ON (ignore_https_errors=False) unless the operator
|
||||
opts into CRAWL4AI_ALLOW_INSECURE_TLS;
|
||||
- no caller proxy (the key/SSRF redirect vector);
|
||||
- strip any proxy/TLS-weakening Chromium launch flags.
|
||||
"""
|
||||
if browser_config is None:
|
||||
return
|
||||
if not ALLOW_INSECURE_TLS and hasattr(browser_config, "ignore_https_errors"):
|
||||
browser_config.ignore_https_errors = False
|
||||
# Drop any caller proxy, then route the browser through the pinning proxy so
|
||||
# Chromium never resolves the target host itself (DNS-rebinding control).
|
||||
for attr in ("proxy", "proxy_config"):
|
||||
if getattr(browser_config, attr, None) is not None:
|
||||
setattr(browser_config, attr, None)
|
||||
if _EGRESS_PROXY_URL and hasattr(browser_config, "proxy_config"):
|
||||
try:
|
||||
from crawl4ai import ProxyConfig
|
||||
browser_config.proxy_config = ProxyConfig(server=_EGRESS_PROXY_URL)
|
||||
except Exception:
|
||||
pass
|
||||
args = getattr(browser_config, "extra_args", None)
|
||||
if args:
|
||||
browser_config.extra_args = [
|
||||
a for a in args
|
||||
if not any(str(a).startswith(bad) for bad in _DANGEROUS_BROWSER_ARGS)
|
||||
]
|
||||
@@ -0,0 +1,212 @@
|
||||
"""
|
||||
egress_proxy.py - localhost pinning forward-proxy for the browser.
|
||||
|
||||
context.route() sees URLs, not IPs, so it cannot stop DNS rebinding: Chromium
|
||||
resolves the target host itself at connect time, and an attacker can answer
|
||||
"public" to our up-front validation and "169.254.169.254" to the browser.
|
||||
|
||||
This proxy is the real control. Chromium is pointed at it (proxy_config), so it
|
||||
never resolves the target itself - it asks us to CONNECT host:port. We run the
|
||||
single egress rule (egress_broker.resolve_and_pin: resolve once, reject any
|
||||
non-global IP, pin one IP), dial the PINNED IP ourselves, and splice raw bytes.
|
||||
TLS stays end-to-end (we tunnel ciphertext; Chromium verifies the cert/SNI
|
||||
against the real host - no MITM).
|
||||
|
||||
Bound to 127.0.0.1 on an ephemeral port; started at server boot.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from egress_broker import EgressBlocked, resolve_and_pin
|
||||
|
||||
logger = logging.getLogger("crawl4ai.egress")
|
||||
|
||||
_CONNECT_OK = b"HTTP/1.1 200 Connection established\r\n\r\n"
|
||||
_BLOCKED = b"HTTP/1.1 403 Forbidden\r\nContent-Length: 11\r\n\r\nURL blocked"
|
||||
_BAD = b"HTTP/1.1 400 Bad Request\r\nContent-Length: 11\r\n\r\nBad Request"
|
||||
_MAX_HEADER_BYTES = 64 * 1024
|
||||
|
||||
|
||||
class PinningProxy:
|
||||
"""Async HTTP forward-proxy that connects only to pinned, global IPs."""
|
||||
|
||||
def __init__(self, host: str = "127.0.0.1", port: int = 0):
|
||||
self._host = host
|
||||
self._port = port
|
||||
self._server: asyncio.AbstractServer | None = None
|
||||
self.bound_host: str | None = None
|
||||
self.bound_port: int | None = None
|
||||
|
||||
@property
|
||||
def url(self) -> str | None:
|
||||
if self.bound_port is None:
|
||||
return None
|
||||
return f"http://{self.bound_host}:{self.bound_port}"
|
||||
|
||||
async def start(self) -> str:
|
||||
self._server = await asyncio.start_server(self._handle, self._host, self._port)
|
||||
sock = self._server.sockets[0]
|
||||
self.bound_host, self.bound_port = sock.getsockname()[:2]
|
||||
logger.info("egress pinning proxy listening on %s", self.url)
|
||||
return self.url
|
||||
|
||||
async def stop(self) -> None:
|
||||
if self._server is not None:
|
||||
self._server.close()
|
||||
try:
|
||||
await self._server.wait_closed()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ─────────────────────────── connection handling ───────────────────────────
|
||||
async def _handle(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
|
||||
try:
|
||||
request_line = await asyncio.wait_for(reader.readline(), timeout=30)
|
||||
if not request_line:
|
||||
return
|
||||
parts = request_line.split()
|
||||
if len(parts) < 3:
|
||||
await self._reply(writer, _BAD)
|
||||
return
|
||||
method = parts[0].decode("latin-1", "replace").upper()
|
||||
target = parts[1].decode("latin-1", "replace")
|
||||
|
||||
if method == "CONNECT":
|
||||
await self._handle_connect(target, reader, writer)
|
||||
else:
|
||||
await self._handle_absolute(method, target, request_line, reader, writer)
|
||||
except asyncio.TimeoutError:
|
||||
await self._reply(writer, _BAD)
|
||||
except Exception as e:
|
||||
logger.debug("proxy connection error: %s", e)
|
||||
await self._safe_close(writer)
|
||||
|
||||
async def _handle_connect(self, target, client_reader, client_writer):
|
||||
# target is "host:port"
|
||||
host, _, port_s = target.rpartition(":")
|
||||
if not host or not port_s.isdigit():
|
||||
await self._reply(client_writer, _BAD)
|
||||
return
|
||||
try:
|
||||
pin = resolve_and_pin(f"https://{host}:{port_s}")
|
||||
except EgressBlocked:
|
||||
await self._reply(client_writer, _BLOCKED)
|
||||
return
|
||||
|
||||
# Drain the rest of the client's CONNECT headers.
|
||||
await self._drain_headers(client_reader)
|
||||
|
||||
try:
|
||||
up_reader, up_writer = await asyncio.wait_for(
|
||||
asyncio.open_connection(pin.ip, int(port_s)), timeout=30
|
||||
)
|
||||
except Exception:
|
||||
await self._reply(client_writer, _BLOCKED)
|
||||
return
|
||||
|
||||
client_writer.write(_CONNECT_OK)
|
||||
await client_writer.drain()
|
||||
await self._splice(client_reader, client_writer, up_reader, up_writer)
|
||||
|
||||
async def _handle_absolute(self, method, target, request_line, client_reader, client_writer):
|
||||
# Plain HTTP proxying: target is an absolute URI "http://host/path".
|
||||
sp = urlsplit(target)
|
||||
if sp.scheme != "http" or not sp.hostname:
|
||||
await self._reply(client_writer, _BAD)
|
||||
return
|
||||
port = sp.port or 80
|
||||
try:
|
||||
pin = resolve_and_pin(f"http://{sp.hostname}:{port}")
|
||||
except EgressBlocked:
|
||||
await self._reply(client_writer, _BLOCKED)
|
||||
return
|
||||
|
||||
headers = await self._read_headers(client_reader)
|
||||
path = sp.path or "/"
|
||||
if sp.query:
|
||||
path += "?" + sp.query
|
||||
try:
|
||||
up_reader, up_writer = await asyncio.wait_for(
|
||||
asyncio.open_connection(pin.ip, port), timeout=30
|
||||
)
|
||||
except Exception:
|
||||
await self._reply(client_writer, _BLOCKED)
|
||||
return
|
||||
# Re-issue in origin form with Host preserved.
|
||||
out = f"{method} {path} HTTP/1.1\r\n".encode("latin-1")
|
||||
out += b"Host: " + sp.hostname.encode("latin-1")
|
||||
if sp.port:
|
||||
out += f":{sp.port}".encode("latin-1")
|
||||
out += b"\r\n" + headers + b"\r\n"
|
||||
up_writer.write(out)
|
||||
await up_writer.drain()
|
||||
await self._splice(client_reader, client_writer, up_reader, up_writer)
|
||||
|
||||
# ─────────────────────────── helpers ───────────────────────────
|
||||
async def _drain_headers(self, reader):
|
||||
read = 0
|
||||
while True:
|
||||
line = await asyncio.wait_for(reader.readline(), timeout=30)
|
||||
read += len(line)
|
||||
if line in (b"\r\n", b"\n", b""):
|
||||
return
|
||||
if read > _MAX_HEADER_BYTES:
|
||||
return
|
||||
|
||||
async def _read_headers(self, reader) -> bytes:
|
||||
buf = b""
|
||||
while True:
|
||||
line = await asyncio.wait_for(reader.readline(), timeout=30)
|
||||
if line in (b"\r\n", b"\n", b""):
|
||||
break
|
||||
buf += line
|
||||
if len(buf) > _MAX_HEADER_BYTES:
|
||||
break
|
||||
# strip any proxy-only / connection headers
|
||||
kept = []
|
||||
for ln in buf.split(b"\r\n"):
|
||||
name = ln.split(b":", 1)[0].strip().lower()
|
||||
if name in (b"proxy-connection", b"proxy-authorization", b"host"):
|
||||
continue
|
||||
if ln:
|
||||
kept.append(ln)
|
||||
return (b"\r\n".join(kept) + b"\r\n") if kept else b""
|
||||
|
||||
async def _splice(self, c_reader, c_writer, u_reader, u_writer):
|
||||
async def pipe(src, dst):
|
||||
try:
|
||||
while True:
|
||||
data = await src.read(65536)
|
||||
if not data:
|
||||
break
|
||||
dst.write(data)
|
||||
await dst.drain()
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
await self._safe_close(dst)
|
||||
|
||||
await asyncio.gather(
|
||||
pipe(c_reader, u_writer),
|
||||
pipe(u_reader, c_writer),
|
||||
)
|
||||
|
||||
async def _reply(self, writer, payload: bytes):
|
||||
try:
|
||||
writer.write(payload)
|
||||
await writer.drain()
|
||||
except Exception:
|
||||
pass
|
||||
await self._safe_close(writer)
|
||||
|
||||
@staticmethod
|
||||
async def _safe_close(writer):
|
||||
try:
|
||||
if not writer.is_closing():
|
||||
writer.close()
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,37 @@
|
||||
#!/usr/bin/env bash
|
||||
# entrypoint.sh - resolve the socket-level egress/auth posture before supervisord.
|
||||
#
|
||||
# This is the authoritative socket-level guard (gunicorn binds the socket, not
|
||||
# Python). It must agree with the in-process _resolve_auth() check in server.py.
|
||||
set -euo pipefail
|
||||
|
||||
# --- Redis password: prefer a mounted secret, else an existing env var. ------
|
||||
if [[ -z "${REDIS_PASSWORD:-}" && -f /run/secrets/redis_password ]]; then
|
||||
REDIS_PASSWORD="$(cat /run/secrets/redis_password)"
|
||||
fi
|
||||
if [[ -z "${REDIS_PASSWORD:-}" ]]; then
|
||||
# Generate an ephemeral in-container password so redis is never open even
|
||||
# if the operator forgot to mount one. (Loopback + requirepass.)
|
||||
REDIS_PASSWORD="$(python3 -c 'import secrets; print(secrets.token_hex(32))')"
|
||||
echo "entrypoint: no REDIS_PASSWORD provided; generated an ephemeral one." >&2
|
||||
fi
|
||||
export REDIS_PASSWORD
|
||||
|
||||
# --- API token: prefer a mounted secret, else an existing env var. -----------
|
||||
if [[ -z "${CRAWL4AI_API_TOKEN:-}" && -f /run/secrets/api_token ]]; then
|
||||
export CRAWL4AI_API_TOKEN="$(cat /run/secrets/api_token)"
|
||||
fi
|
||||
|
||||
# --- Bind resolution: loopback unless a credential is present. ---------------
|
||||
PORT="${CRAWL4AI_PORT:-11235}"
|
||||
if [[ -n "${CRAWL4AI_API_TOKEN:-}" || "${CRAWL4AI_JWT_ENABLED:-false}" == "true" ]]; then
|
||||
# A credential is configured -> the operator may expose all interfaces.
|
||||
GUNICORN_BIND="${GUNICORN_BIND:-[::]:${PORT}}"
|
||||
else
|
||||
# No credential -> refuse to expose; serve loopback only.
|
||||
GUNICORN_BIND="127.0.0.1:${PORT}"
|
||||
echo "entrypoint: no CRAWL4AI_API_TOKEN set; binding loopback only (${GUNICORN_BIND})." >&2
|
||||
fi
|
||||
export GUNICORN_BIND
|
||||
|
||||
exec supervisord -c supervisord.conf --pidfile /tmp/supervisord.pid
|
||||
@@ -0,0 +1,111 @@
|
||||
"""
|
||||
governor.py - server-enforced resource governance (R5).
|
||||
|
||||
The deep-crawl DoS via a client-supplied unbounded max_pages is already closed
|
||||
upstream: R2 forbids `deep_crawl_strategy` on an untrusted request body, and the
|
||||
`urls` list is capped at 100 by the request schema. This module adds the two
|
||||
remaining verifiable chokepoints:
|
||||
|
||||
* a request body-size limit (ASGI middleware) so a giant body / inline `raw:`
|
||||
HTML cannot be buffered and processed in-process -> 413;
|
||||
* clamp_deep_crawl(): defense in depth for any *trusted* / server-built config
|
||||
that still carries a deep_crawl strategy with an unbounded page/depth count.
|
||||
|
||||
Heavier governance (bounded work queue replacing BackgroundTasks, per-principal
|
||||
Redis quotas, wall-clock deadlines, stream decoupling) is left for the
|
||||
integration-tested pass; gunicorn --limit-request-* covers the transport layer.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
|
||||
DEFAULT_MAX_BODY_BYTES = 10 * 1024 * 1024 # 10 MiB
|
||||
DEFAULT_MAX_PAGES = 100
|
||||
DEFAULT_MAX_DEPTH = 5
|
||||
|
||||
|
||||
class BodySizeLimitMiddleware:
|
||||
"""Reject HTTP requests whose declared Content-Length exceeds the limit.
|
||||
|
||||
(Chunked/unknown-length bodies are additionally bounded at the transport by
|
||||
gunicorn --limit-request-* in the hardened deployment.)
|
||||
"""
|
||||
|
||||
def __init__(self, app, max_bytes: int = DEFAULT_MAX_BODY_BYTES):
|
||||
self.app = app
|
||||
self.max_bytes = max_bytes
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] == "http":
|
||||
for name, value in scope.get("headers", []):
|
||||
if name == b"content-length":
|
||||
try:
|
||||
if int(value) > self.max_bytes:
|
||||
await self._reject(send)
|
||||
return
|
||||
except ValueError:
|
||||
pass
|
||||
break
|
||||
await self.app(scope, receive, send)
|
||||
|
||||
async def _reject(self, send):
|
||||
body = json.dumps({"detail": "Request body too large"}).encode()
|
||||
await send({
|
||||
"type": "http.response.start",
|
||||
"status": 413,
|
||||
"headers": [
|
||||
(b"content-type", b"application/json"),
|
||||
(b"content-length", str(len(body)).encode()),
|
||||
],
|
||||
})
|
||||
await send({"type": "http.response.body", "body": body})
|
||||
|
||||
|
||||
def clamp_deep_crawl(crawler_config, *, max_pages: int = DEFAULT_MAX_PAGES,
|
||||
max_depth: int = DEFAULT_MAX_DEPTH) -> None:
|
||||
"""Clamp an attached deep-crawl strategy's page/depth budget in place.
|
||||
|
||||
Defense in depth: untrusted bodies cannot set deep_crawl_strategy at all
|
||||
(R2), but a server/base config might, and the library default for max_pages
|
||||
is infinity.
|
||||
"""
|
||||
dc = getattr(crawler_config, "deep_crawl_strategy", None)
|
||||
if dc is None:
|
||||
return
|
||||
mp = getattr(dc, "max_pages", None)
|
||||
if mp is None or (isinstance(mp, float) and math.isinf(mp)) or mp > max_pages:
|
||||
try:
|
||||
dc.max_pages = max_pages
|
||||
except Exception:
|
||||
pass
|
||||
md = getattr(dc, "max_depth", None)
|
||||
if md is None or md > max_depth:
|
||||
try:
|
||||
dc.max_depth = max_depth
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def max_body_bytes_from_config(config: dict) -> int:
|
||||
return int((config.get("limits", {}) or {}).get("max_body_bytes", DEFAULT_MAX_BODY_BYTES))
|
||||
|
||||
|
||||
def _limits(config: dict) -> dict:
|
||||
return config.get("limits", {}) or {}
|
||||
|
||||
|
||||
def wall_clock_seconds(config: dict) -> float:
|
||||
"""Per-crawl wall-clock deadline in seconds; 0 (default) => no deadline."""
|
||||
return float(_limits(config).get("wall_clock_s", 0) or 0)
|
||||
|
||||
|
||||
def job_queue_caps(config: dict) -> dict:
|
||||
"""Bounded-job-queue settings; 0 => unbounded/unlimited (current behavior)."""
|
||||
q = _limits(config).get("queue", {}) or {}
|
||||
return {
|
||||
"maxsize": int(q.get("maxsize", 1000) or 0),
|
||||
"workers": int(q.get("workers", 4) or 1),
|
||||
"per_principal": int(q.get("per_principal", 0) or 0),
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
"""
|
||||
Declarative hook registry - the safe replacement for the exec-based hook system.
|
||||
|
||||
The old hook_manager.py compiled and exec()'d user-supplied Python at crawl hook
|
||||
points. Its sandbox was unsound (escapable via __subclasses__ MRO walks, injected
|
||||
module __globals__, frame inspection), giving unauthenticated RCE when hooks were
|
||||
enabled. There is no safe way to run attacker Python in-process.
|
||||
|
||||
Instead, a request may only choose from a fixed set of declarative ACTIONS whose
|
||||
parameters are schema-validated scalars. Each action maps to a server-authored
|
||||
async function that calls exactly one specific Playwright API - no user string
|
||||
ever reaches an interpreter. This covers the documented hook use cases (block
|
||||
assets, inject auth cookies/headers, scroll for lazy content, wait).
|
||||
|
||||
Power users who genuinely need arbitrary hook code use a self-hosted in-process
|
||||
build where crawler_strategy.set_hook(...) remains available and trusted.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any, Callable, Dict, List
|
||||
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
|
||||
class HookValidationError(ValueError):
|
||||
"""A declarative hook spec was invalid. The Docker layer maps this to 400."""
|
||||
|
||||
|
||||
_HEADER_NAME_RE = re.compile(r"^[A-Za-z0-9-]{1,64}$")
|
||||
_ALLOWED_RESOURCE_TYPES = {"image", "stylesheet", "font", "media"}
|
||||
_MAX_SCROLL_STEPS = 50
|
||||
_MAX_SCROLL_DELAY_MS = 5000
|
||||
_MAX_WAIT_MS = 60_000
|
||||
_MAX_COOKIES = 20
|
||||
_MAX_HEADERS = 20
|
||||
|
||||
|
||||
# ───────────────────────── per-action parameter schemas ─────────────────────────
|
||||
class BlockResourcesParams(BaseModel):
|
||||
resource_types: List[str] = Field(..., min_length=1)
|
||||
|
||||
@field_validator("resource_types")
|
||||
@classmethod
|
||||
def _check(cls, v):
|
||||
bad = sorted(set(v) - _ALLOWED_RESOURCE_TYPES)
|
||||
if bad:
|
||||
raise ValueError(f"unsupported resource_types {bad}; allowed: {sorted(_ALLOWED_RESOURCE_TYPES)}")
|
||||
return v
|
||||
|
||||
|
||||
class _Cookie(BaseModel):
|
||||
name: str = Field(..., min_length=1, max_length=256)
|
||||
value: str = Field(..., max_length=4096)
|
||||
domain: str = Field(..., min_length=1, max_length=253)
|
||||
path: str = "/"
|
||||
secure: bool = True
|
||||
httpOnly: bool = False
|
||||
|
||||
|
||||
class AddCookiesParams(BaseModel):
|
||||
cookies: List[_Cookie] = Field(..., min_length=1, max_length=_MAX_COOKIES)
|
||||
|
||||
|
||||
class SetHeadersParams(BaseModel):
|
||||
headers: Dict[str, str]
|
||||
|
||||
@field_validator("headers")
|
||||
@classmethod
|
||||
def _check(cls, v):
|
||||
if len(v) > _MAX_HEADERS:
|
||||
raise ValueError(f"too many headers (max {_MAX_HEADERS})")
|
||||
for name, value in v.items():
|
||||
if not _HEADER_NAME_RE.match(name):
|
||||
raise ValueError(f"invalid header name {name!r}")
|
||||
if any(c in value for c in "\r\n\x00"):
|
||||
raise ValueError(f"control characters in value for header {name!r}")
|
||||
return v
|
||||
|
||||
|
||||
class ScrollToBottomParams(BaseModel):
|
||||
max_steps: int = Field(10, ge=1, le=_MAX_SCROLL_STEPS)
|
||||
delay_ms: int = Field(500, ge=0, le=_MAX_SCROLL_DELAY_MS)
|
||||
|
||||
|
||||
class WaitForTimeoutParams(BaseModel):
|
||||
timeout_ms: int = Field(..., ge=0, le=_MAX_WAIT_MS)
|
||||
|
||||
|
||||
# ───────────────────────── server-authored hook factories ─────────────────────────
|
||||
def _factory_block_resources(p: BlockResourcesParams):
|
||||
types = set(p.resource_types)
|
||||
|
||||
async def hook(page, **kwargs):
|
||||
context = kwargs.get("context")
|
||||
|
||||
async def _route(route):
|
||||
try:
|
||||
if route.request.resource_type in types:
|
||||
await route.abort()
|
||||
else:
|
||||
await route.continue_()
|
||||
except Exception:
|
||||
# never let a routing decision crash the crawl
|
||||
await route.continue_()
|
||||
|
||||
if context is not None:
|
||||
await context.route("**/*", _route)
|
||||
return page
|
||||
|
||||
return hook
|
||||
|
||||
|
||||
def _factory_add_cookies(p: AddCookiesParams):
|
||||
cookies = [c.model_dump() for c in p.cookies]
|
||||
|
||||
async def hook(page, **kwargs):
|
||||
context = kwargs.get("context")
|
||||
if context is not None:
|
||||
await context.add_cookies(cookies)
|
||||
return page
|
||||
|
||||
return hook
|
||||
|
||||
|
||||
def _factory_set_headers(p: SetHeadersParams):
|
||||
headers = dict(p.headers)
|
||||
|
||||
async def hook(page, **kwargs):
|
||||
await page.set_extra_http_headers(headers)
|
||||
return page
|
||||
|
||||
return hook
|
||||
|
||||
|
||||
def _factory_scroll_to_bottom(p: ScrollToBottomParams):
|
||||
max_steps, delay_ms = p.max_steps, p.delay_ms
|
||||
|
||||
async def hook(page, **kwargs):
|
||||
for _ in range(max_steps):
|
||||
await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
||||
if delay_ms:
|
||||
await page.wait_for_timeout(delay_ms)
|
||||
return page
|
||||
|
||||
return hook
|
||||
|
||||
|
||||
def _factory_wait_for_timeout(p: WaitForTimeoutParams):
|
||||
timeout_ms = p.timeout_ms
|
||||
|
||||
async def hook(page, **kwargs):
|
||||
await page.wait_for_timeout(timeout_ms)
|
||||
return page
|
||||
|
||||
return hook
|
||||
|
||||
|
||||
# action name -> (hook_point, params model, factory, human description)
|
||||
HOOK_REGISTRY: Dict[str, dict] = {
|
||||
"block_resources": {
|
||||
"hook_point": "on_page_context_created",
|
||||
"params_model": BlockResourcesParams,
|
||||
"factory": _factory_block_resources,
|
||||
"description": "Abort matching resource types (image/stylesheet/font/media).",
|
||||
},
|
||||
"add_cookies": {
|
||||
"hook_point": "on_page_context_created",
|
||||
"params_model": AddCookiesParams,
|
||||
"factory": _factory_add_cookies,
|
||||
"description": "Add cookies to the browser context before navigation (auth).",
|
||||
},
|
||||
"set_headers": {
|
||||
"hook_point": "before_goto",
|
||||
"params_model": SetHeadersParams,
|
||||
"factory": _factory_set_headers,
|
||||
"description": "Set extra HTTP request headers before navigating.",
|
||||
},
|
||||
"scroll_to_bottom": {
|
||||
"hook_point": "before_retrieve_html",
|
||||
"params_model": ScrollToBottomParams,
|
||||
"factory": _factory_scroll_to_bottom,
|
||||
"description": "Scroll to the page bottom in bounded steps (lazy-load).",
|
||||
},
|
||||
"wait_for_timeout": {
|
||||
"hook_point": "before_retrieve_html",
|
||||
"params_model": WaitForTimeoutParams,
|
||||
"factory": _factory_wait_for_timeout,
|
||||
"description": "Wait a bounded number of milliseconds before retrieving HTML.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_declarative_hooks(specs: List[Any]) -> Dict[str, Callable]:
|
||||
"""Validate declarative hook specs and return {hook_point: composed async hook}.
|
||||
|
||||
Each spec is an object/dict with `action` and `params`. Multiple specs that
|
||||
target the same hook point are composed and run in order. Raises
|
||||
HookValidationError on an unknown action or invalid params.
|
||||
"""
|
||||
if not specs:
|
||||
return {}
|
||||
if len(specs) > 10:
|
||||
raise HookValidationError("too many hooks (max 10)")
|
||||
|
||||
grouped: Dict[str, List[Callable]] = {}
|
||||
for spec in specs:
|
||||
action = spec.get("action") if isinstance(spec, dict) else getattr(spec, "action", None)
|
||||
raw_params = (spec.get("params", {}) if isinstance(spec, dict) else getattr(spec, "params", {})) or {}
|
||||
entry = HOOK_REGISTRY.get(action)
|
||||
if entry is None:
|
||||
raise HookValidationError(
|
||||
f"unknown hook action {action!r}; allowed: {sorted(HOOK_REGISTRY)}"
|
||||
)
|
||||
try:
|
||||
params = entry["params_model"](**raw_params)
|
||||
except Exception as e:
|
||||
raise HookValidationError(f"invalid params for hook '{action}': {e}")
|
||||
sub_hook = entry["factory"](params)
|
||||
grouped.setdefault(entry["hook_point"], []).append(sub_hook)
|
||||
|
||||
hooks: Dict[str, Callable] = {}
|
||||
for hook_point, sub_hooks in grouped.items():
|
||||
def _compose(sub_hooks):
|
||||
async def composed(page, **kwargs):
|
||||
for fn in sub_hooks:
|
||||
await fn(page, **kwargs)
|
||||
return page
|
||||
return composed
|
||||
hooks[hook_point] = _compose(sub_hooks)
|
||||
return hooks
|
||||
|
||||
|
||||
def describe_registry() -> dict:
|
||||
"""Enumerate the available declarative actions for /hooks/info."""
|
||||
return {
|
||||
action: {
|
||||
"hook_point": entry["hook_point"],
|
||||
"description": entry["description"],
|
||||
"params_schema": entry["params_model"].model_json_schema(),
|
||||
}
|
||||
for action, entry in HOOK_REGISTRY.items()
|
||||
}
|
||||
@@ -0,0 +1,150 @@
|
||||
"""
|
||||
Job endpoints (enqueue + poll) for long-running LLM extraction and raw crawl.
|
||||
Relies on the existing Redis task helpers in api.py
|
||||
"""
|
||||
|
||||
from typing import Dict, Optional, Callable
|
||||
from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request
|
||||
from pydantic import BaseModel, HttpUrl
|
||||
|
||||
from api import (
|
||||
handle_llm_request,
|
||||
handle_crawl_job,
|
||||
handle_task_status,
|
||||
)
|
||||
from auth import get_principal
|
||||
from schemas import WebhookConfig
|
||||
|
||||
# ------------- dependency placeholders -------------
|
||||
_redis = None # will be injected from server.py
|
||||
_config = None
|
||||
_token_dep: Callable = lambda: None # dummy until injected
|
||||
|
||||
# public router
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def _principal_dep(request: Request) -> Optional[Dict]:
|
||||
"""The principal the AuthGateMiddleware already validated for this request."""
|
||||
return get_principal(request)
|
||||
|
||||
|
||||
def _owner_of(principal: Optional[Dict]) -> Optional[str]:
|
||||
return principal.get("sub") if principal else None
|
||||
|
||||
|
||||
def _is_admin(principal: Optional[Dict]) -> bool:
|
||||
return bool(principal) and principal.get("scope") == "admin"
|
||||
|
||||
|
||||
# === init hook called by server.py =========================================
|
||||
def init_job_router(redis, config, token_dep) -> APIRouter:
|
||||
"""Inject shared singletons and return the router for mounting."""
|
||||
global _redis, _config, _token_dep
|
||||
_redis, _config, _token_dep = redis, config, token_dep
|
||||
return router
|
||||
|
||||
|
||||
# ---------- payload models --------------------------------------------------
|
||||
class LlmJobPayload(BaseModel):
|
||||
url: HttpUrl
|
||||
q: str
|
||||
schema: Optional[str] = None
|
||||
cache: bool = False
|
||||
provider: Optional[str] = None
|
||||
webhook_config: Optional[WebhookConfig] = None
|
||||
temperature: Optional[float] = None
|
||||
# base_url removed: server-derived LLM endpoint only (key-exfil vector).
|
||||
|
||||
|
||||
class CrawlJobPayload(BaseModel):
|
||||
urls: list[HttpUrl]
|
||||
browser_config: Dict = {}
|
||||
crawler_config: Dict = {}
|
||||
webhook_config: Optional[WebhookConfig] = None
|
||||
|
||||
|
||||
# ---------- LLM job ---------------------------------------------------------
|
||||
@router.post("/llm/job", status_code=202)
|
||||
async def llm_job_enqueue(
|
||||
payload: LlmJobPayload,
|
||||
background_tasks: BackgroundTasks,
|
||||
request: Request,
|
||||
_td: Optional[Dict] = Depends(_principal_dep),
|
||||
):
|
||||
webhook_config = None
|
||||
if payload.webhook_config:
|
||||
from utils import validate_webhook_url
|
||||
try:
|
||||
validate_webhook_url(str(payload.webhook_config.webhook_url))
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
webhook_config = payload.webhook_config.model_dump(mode='json')
|
||||
|
||||
return await handle_llm_request(
|
||||
_redis,
|
||||
background_tasks,
|
||||
request,
|
||||
str(payload.url),
|
||||
query=payload.q,
|
||||
schema=payload.schema,
|
||||
cache=payload.cache,
|
||||
config=_config,
|
||||
provider=payload.provider,
|
||||
webhook_config=webhook_config,
|
||||
temperature=payload.temperature,
|
||||
requester=_owner_of(_td),
|
||||
is_admin=_is_admin(_td),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/llm/job/{task_id}")
|
||||
async def llm_job_status(
|
||||
request: Request,
|
||||
task_id: str,
|
||||
_td: Optional[Dict] = Depends(_principal_dep),
|
||||
):
|
||||
return await handle_task_status(
|
||||
_redis, task_id, base_url=str(request.base_url),
|
||||
requester=_owner_of(_td), is_admin=_is_admin(_td),
|
||||
)
|
||||
|
||||
|
||||
# ---------- CRAWL job -------------------------------------------------------
|
||||
@router.post("/crawl/job", status_code=202)
|
||||
async def crawl_job_enqueue(
|
||||
payload: CrawlJobPayload,
|
||||
background_tasks: BackgroundTasks,
|
||||
_td: Optional[Dict] = Depends(_principal_dep),
|
||||
):
|
||||
webhook_config = None
|
||||
if payload.webhook_config:
|
||||
from utils import validate_webhook_url
|
||||
try:
|
||||
validate_webhook_url(str(payload.webhook_config.webhook_url))
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
webhook_config = payload.webhook_config.model_dump(mode='json')
|
||||
|
||||
return await handle_crawl_job(
|
||||
_redis,
|
||||
background_tasks,
|
||||
[str(u) for u in payload.urls],
|
||||
payload.browser_config,
|
||||
payload.crawler_config,
|
||||
config=_config,
|
||||
webhook_config=webhook_config,
|
||||
owner=_owner_of(_td),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/crawl/job/{task_id}")
|
||||
async def crawl_job_status(
|
||||
request: Request,
|
||||
task_id: str,
|
||||
_td: Optional[Dict] = Depends(_principal_dep),
|
||||
):
|
||||
return await handle_task_status(
|
||||
_redis, task_id, base_url=str(request.base_url),
|
||||
requester=_owner_of(_td), is_admin=_is_admin(_td),
|
||||
)
|
||||
@@ -0,0 +1,64 @@
|
||||
"""
|
||||
llm_broker.py - server-side LLM provider resolution.
|
||||
|
||||
The credential-exfil gadget (reported by Geo): a request could pass its own
|
||||
`base_url` (and/or `provider`), and the server would happily send the
|
||||
configured provider API key to that attacker-controlled endpoint, leaking
|
||||
*every* provider key the server holds.
|
||||
|
||||
The fix: a request may select a provider BY NAME only (and only from an
|
||||
allowlisted family). The `base_url` and `api_token` are ALWAYS derived
|
||||
server-side from config/environment and are never taken from the request, so a
|
||||
key can never be redirected to an attacker host.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, Optional
|
||||
|
||||
|
||||
class LLMProviderNotAllowed(ValueError):
|
||||
"""The requested LLM provider is not in the server's allowlist (-> HTTP 400)."""
|
||||
|
||||
|
||||
def _family(provider: Optional[str]) -> str:
|
||||
return (provider or "").split("/")[0].lower()
|
||||
|
||||
|
||||
def allowed_provider_families(config: Dict) -> set:
|
||||
"""Provider families a request may select.
|
||||
|
||||
Sourced from config.llm.allowed_providers plus the default provider's
|
||||
family. Empty set => unrestricted family selection (base_url/key are still
|
||||
server-only, so the exfil path stays closed); set a non-empty
|
||||
allowed_providers to lock it down further.
|
||||
"""
|
||||
cfg = config.get("llm", {}) or {}
|
||||
fams = {_family(p) for p in (cfg.get("allowed_providers") or [])}
|
||||
if cfg.get("provider"):
|
||||
fams.add(_family(cfg["provider"]))
|
||||
return {f for f in fams if f}
|
||||
|
||||
|
||||
def resolve_llm(config: Dict, requested_provider: Optional[str] = None) -> Dict:
|
||||
"""Resolve the LLM call parameters fully server-side.
|
||||
|
||||
A request-supplied base_url/api_token is intentionally NOT a parameter here:
|
||||
callers pass only the provider *name*. Returns {provider, base_url,
|
||||
api_token, temperature}, all server-derived.
|
||||
"""
|
||||
from utils import get_llm_api_key, get_llm_base_url, get_llm_temperature
|
||||
|
||||
default = config["llm"]["provider"]
|
||||
provider = requested_provider or default
|
||||
|
||||
fams = allowed_provider_families(config)
|
||||
if fams and _family(provider) not in fams:
|
||||
raise LLMProviderNotAllowed("LLM provider not allowed")
|
||||
|
||||
return {
|
||||
"provider": provider,
|
||||
"base_url": get_llm_base_url(config, provider), # canonical, never from request
|
||||
"api_token": get_llm_api_key(config, provider), # server credential
|
||||
"temperature": get_llm_temperature(config, provider),
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
# deploy/docker/mcp_bridge.py
|
||||
|
||||
from __future__ import annotations
|
||||
import inspect, json, re, anyio
|
||||
from contextlib import suppress
|
||||
from typing import Any, Callable, Dict, List, Tuple
|
||||
import httpx
|
||||
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
from starlette.routing import Route, Mount
|
||||
from mcp.server.sse import SseServerTransport
|
||||
|
||||
import mcp.types as t
|
||||
from mcp.server.lowlevel.server import Server, NotificationOptions
|
||||
from mcp.server.models import InitializationOptions
|
||||
|
||||
# ── opt‑in decorators ───────────────────────────────────────────
|
||||
def mcp_resource(name: str | None = None):
|
||||
def deco(fn):
|
||||
fn.__mcp_kind__, fn.__mcp_name__ = "resource", name
|
||||
return fn
|
||||
return deco
|
||||
|
||||
def mcp_template(name: str | None = None):
|
||||
def deco(fn):
|
||||
fn.__mcp_kind__, fn.__mcp_name__ = "template", name
|
||||
return fn
|
||||
return deco
|
||||
|
||||
def mcp_tool(name: str | None = None):
|
||||
def deco(fn):
|
||||
fn.__mcp_kind__, fn.__mcp_name__ = "tool", name
|
||||
return fn
|
||||
return deco
|
||||
|
||||
# ── HTTP‑proxy helper for FastAPI endpoints ─────────────────────
|
||||
def _service_auth_headers() -> dict:
|
||||
"""Authenticate the internal loopback call to our own gated endpoints.
|
||||
|
||||
The MCP transport is now behind the AuthGateMiddleware, so by the time a
|
||||
tool is invoked the MCP client is already authenticated. The loopback HTTP
|
||||
call this proxy makes must therefore carry a credential too, or the gate
|
||||
would 401 it. We mint a short-lived, data-scope service token: MCP exposes
|
||||
only data-plane tools (no admin/monitor actions), so this grants no
|
||||
privilege escalation. (Requires a shared SECRET_KEY across workers, which
|
||||
the auth startup check already mandates for any real deployment.)
|
||||
"""
|
||||
from auth import create_access_token
|
||||
token = create_access_token({"sub": "mcp-service"}, scope="data")
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
def _make_http_proxy(base_url: str, route, *, timeout: float | None = None):
|
||||
method = list(route.methods - {"HEAD", "OPTIONS"})[0]
|
||||
async def proxy(**kwargs):
|
||||
# replace `/items/{id}` style params first
|
||||
path = route.path
|
||||
for k, v in list(kwargs.items()):
|
||||
placeholder = "{" + k + "}"
|
||||
if placeholder in path:
|
||||
path = path.replace(placeholder, str(v))
|
||||
kwargs.pop(k)
|
||||
url = base_url.rstrip("/") + path
|
||||
|
||||
headers = _service_auth_headers()
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
try:
|
||||
r = (
|
||||
await client.get(url, params=kwargs, headers=headers)
|
||||
if method == "GET"
|
||||
else await client.request(method, url, json=kwargs, headers=headers)
|
||||
)
|
||||
r.raise_for_status()
|
||||
return r.text if method == "GET" else r.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
# surface FastAPI error details instead of plain 500
|
||||
raise HTTPException(e.response.status_code, e.response.text)
|
||||
except httpx.TimeoutException:
|
||||
raise HTTPException(504, "upstream request timed out")
|
||||
return proxy
|
||||
|
||||
# ── main entry point ────────────────────────────────────────────
|
||||
def attach_mcp(
|
||||
app: FastAPI,
|
||||
*, # keyword‑only
|
||||
base: str = "/mcp",
|
||||
name: str | None = None,
|
||||
base_url: str, # eg. "http://127.0.0.1:8020"
|
||||
timeout: float | None = None, # httpx timeout in seconds; None = no limit
|
||||
) -> None:
|
||||
"""Call once after all routes are declared to expose WS+SSE MCP endpoints."""
|
||||
server_name = name or app.title or "FastAPI-MCP"
|
||||
mcp = Server(server_name)
|
||||
|
||||
# tools: Dict[str, Callable] = {}
|
||||
tools: Dict[str, Tuple[Callable, Callable]] = {}
|
||||
resources: Dict[str, Callable] = {}
|
||||
templates: Dict[str, Callable] = {}
|
||||
|
||||
# register decorated FastAPI routes
|
||||
for route in app.routes:
|
||||
fn = getattr(route, "endpoint", None)
|
||||
kind = getattr(fn, "__mcp_kind__", None)
|
||||
if not kind:
|
||||
continue
|
||||
|
||||
key = fn.__mcp_name__ or re.sub(r"[/{}}]", "_", route.path).strip("_")
|
||||
|
||||
# if kind == "tool":
|
||||
# tools[key] = _make_http_proxy(base_url, route)
|
||||
if kind == "tool":
|
||||
proxy = _make_http_proxy(base_url, route, timeout=timeout)
|
||||
tools[key] = (proxy, fn)
|
||||
continue
|
||||
if kind == "resource":
|
||||
resources[key] = fn
|
||||
if kind == "template":
|
||||
templates[key] = fn
|
||||
|
||||
# helpers for JSON‑Schema
|
||||
def _schema(model: type[BaseModel] | None) -> dict:
|
||||
return {"type": "object"} if model is None else model.model_json_schema()
|
||||
|
||||
def _body_model(fn: Callable) -> type[BaseModel] | None:
|
||||
for p in inspect.signature(fn).parameters.values():
|
||||
a = p.annotation
|
||||
if inspect.isclass(a) and issubclass(a, BaseModel):
|
||||
return a
|
||||
return None
|
||||
|
||||
# MCP handlers
|
||||
@mcp.list_tools()
|
||||
async def _list_tools() -> List[t.Tool]:
|
||||
out = []
|
||||
for k, (proxy, orig_fn) in tools.items():
|
||||
desc = getattr(orig_fn, "__mcp_description__", None) or inspect.getdoc(orig_fn) or ""
|
||||
schema = getattr(orig_fn, "__mcp_schema__", None) or _schema(_body_model(orig_fn))
|
||||
out.append(
|
||||
t.Tool(name=k, description=desc, inputSchema=schema)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
@mcp.call_tool()
|
||||
async def _call_tool(name: str, arguments: Dict | None) -> List[t.TextContent]:
|
||||
if name not in tools:
|
||||
raise HTTPException(404, "tool not found")
|
||||
|
||||
proxy, _ = tools[name]
|
||||
try:
|
||||
res = await proxy(**(arguments or {}))
|
||||
except HTTPException as exc:
|
||||
# map server‑side errors into MCP "text/error" payloads
|
||||
err = {"error": exc.status_code, "detail": exc.detail}
|
||||
return [t.TextContent(type = "text", text=json.dumps(err, ensure_ascii=False))]
|
||||
return [t.TextContent(type = "text", text=json.dumps(res, default=str, ensure_ascii=False))]
|
||||
|
||||
@mcp.list_resources()
|
||||
async def _list_resources() -> List[t.Resource]:
|
||||
return [
|
||||
t.Resource(name=k, description=inspect.getdoc(f) or "", mime_type="application/json")
|
||||
for k, f in resources.items()
|
||||
]
|
||||
|
||||
@mcp.read_resource()
|
||||
async def _read_resource(name: str) -> List[t.TextContent]:
|
||||
if name not in resources:
|
||||
raise HTTPException(404, "resource not found")
|
||||
res = resources[name]()
|
||||
return [t.TextContent(type = "text", text=json.dumps(res, default=str, ensure_ascii=False))]
|
||||
|
||||
@mcp.list_resource_templates()
|
||||
async def _list_templates() -> List[t.ResourceTemplate]:
|
||||
return [
|
||||
t.ResourceTemplate(
|
||||
name=k,
|
||||
description=inspect.getdoc(f) or "",
|
||||
parameters={
|
||||
p: {"type": "string"} for p in _path_params(app, f)
|
||||
},
|
||||
)
|
||||
for k, f in templates.items()
|
||||
]
|
||||
|
||||
init_opts = InitializationOptions(
|
||||
server_name=server_name,
|
||||
server_version="0.1.0",
|
||||
capabilities=mcp.get_capabilities(
|
||||
notification_options=NotificationOptions(),
|
||||
experimental_capabilities={},
|
||||
),
|
||||
)
|
||||
|
||||
# ── WebSocket transport ────────────────────────────────────
|
||||
@app.websocket_route(f"{base}/ws")
|
||||
async def _ws(ws: WebSocket):
|
||||
await ws.accept()
|
||||
c2s_send, c2s_recv = anyio.create_memory_object_stream(100)
|
||||
s2c_send, s2c_recv = anyio.create_memory_object_stream(100)
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
from mcp.types import JSONRPCMessage
|
||||
adapter = TypeAdapter(JSONRPCMessage)
|
||||
|
||||
init_done = anyio.Event()
|
||||
|
||||
async def srv_to_ws():
|
||||
first = True
|
||||
try:
|
||||
async for msg in s2c_recv:
|
||||
await ws.send_json(msg.model_dump())
|
||||
if first:
|
||||
init_done.set()
|
||||
first = False
|
||||
finally:
|
||||
# make sure cleanup survives TaskGroup cancellation
|
||||
with anyio.CancelScope(shield=True):
|
||||
with suppress(RuntimeError): # idempotent close
|
||||
await ws.close()
|
||||
|
||||
async def ws_to_srv():
|
||||
try:
|
||||
# 1st frame is always "initialize"
|
||||
first = adapter.validate_python(await ws.receive_json())
|
||||
await c2s_send.send(first)
|
||||
await init_done.wait() # block until server ready
|
||||
while True:
|
||||
data = await ws.receive_json()
|
||||
await c2s_send.send(adapter.validate_python(data))
|
||||
except WebSocketDisconnect:
|
||||
await c2s_send.aclose()
|
||||
|
||||
async with anyio.create_task_group() as tg:
|
||||
tg.start_soon(mcp.run, c2s_recv, s2c_send, init_opts)
|
||||
tg.start_soon(ws_to_srv)
|
||||
tg.start_soon(srv_to_ws)
|
||||
|
||||
# ── SSE transport (raw ASGI — avoids Starlette middleware conflict) ──
|
||||
sse = SseServerTransport(f"{base}/messages/")
|
||||
|
||||
# Starlette's Route wraps plain async functions in request_response(),
|
||||
# which calls handler(request) instead of handler(scope, receive, send).
|
||||
# Using a callable class bypasses this — Route passes classes through
|
||||
# as raw ASGI apps. See #1594, #1850.
|
||||
class _MCPSseApp:
|
||||
async def __call__(self, scope, receive, send):
|
||||
async with sse.connect_sse(scope, receive, send) as (read_stream, write_stream):
|
||||
await mcp.run(read_stream, write_stream, init_opts)
|
||||
|
||||
app.routes.append(Route(f"{base}/sse", endpoint=_MCPSseApp()))
|
||||
app.routes.append(Mount(f"{base}/messages", app=sse.handle_post_message))
|
||||
|
||||
# ── schema endpoint ───────────────────────────────────────
|
||||
@app.get(f"{base}/schema")
|
||||
async def _schema_endpoint():
|
||||
return JSONResponse({
|
||||
"tools": [x.model_dump() for x in await _list_tools()],
|
||||
"resources": [x.model_dump() for x in await _list_resources()],
|
||||
"resource_templates": [x.model_dump() for x in await _list_templates()],
|
||||
})
|
||||
|
||||
|
||||
# ── helpers ────────────────────────────────────────────────────
|
||||
def _route_name(path: str) -> str:
|
||||
return re.sub(r"[/{}}]", "_", path).strip("_")
|
||||
|
||||
def _path_params(app: FastAPI, fn: Callable) -> List[str]:
|
||||
for r in app.routes:
|
||||
if r.endpoint is fn:
|
||||
return list(r.param_convertors.keys())
|
||||
return []
|
||||
@@ -0,0 +1,391 @@
|
||||
# monitor.py - Real-time monitoring stats with Redis persistence
|
||||
import html
|
||||
import time
|
||||
import json
|
||||
import asyncio
|
||||
from typing import Dict, List, Optional
|
||||
from datetime import datetime, timezone
|
||||
from collections import deque
|
||||
from redis import asyncio as aioredis
|
||||
from utils import get_container_memory_percent
|
||||
import psutil
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class MonitorStats:
|
||||
"""Tracks real-time server stats with Redis persistence."""
|
||||
|
||||
def __init__(self, redis: aioredis.Redis):
|
||||
self.redis = redis
|
||||
self.start_time = time.time()
|
||||
|
||||
# In-memory queues (fast reads, Redis backup)
|
||||
self.active_requests: Dict[str, Dict] = {} # id -> request info
|
||||
self.completed_requests: deque = deque(maxlen=100) # Last 100
|
||||
self.janitor_events: deque = deque(maxlen=100)
|
||||
self.errors: deque = deque(maxlen=100)
|
||||
|
||||
# Endpoint stats (persisted in Redis)
|
||||
self.endpoint_stats: Dict[str, Dict] = {} # endpoint -> {count, total_time, errors, ...}
|
||||
|
||||
# Background persistence queue (max 10 pending persist requests)
|
||||
self._persist_queue: asyncio.Queue = asyncio.Queue(maxsize=10)
|
||||
self._persist_worker_task: Optional[asyncio.Task] = None
|
||||
|
||||
# Timeline data (5min window, 5s resolution = 60 points)
|
||||
self.memory_timeline: deque = deque(maxlen=60)
|
||||
self.requests_timeline: deque = deque(maxlen=60)
|
||||
self.browser_timeline: deque = deque(maxlen=60)
|
||||
|
||||
async def track_request_start(self, request_id: str, endpoint: str, url: str, config: Dict = None):
|
||||
"""Track new request start."""
|
||||
req_info = {
|
||||
"id": request_id,
|
||||
"endpoint": endpoint,
|
||||
"url": html.escape(url[:100]), # Truncate + escape for XSS prevention
|
||||
"start_time": time.time(),
|
||||
"config_sig": config.get("sig", "default") if config else "default",
|
||||
"mem_start": psutil.Process().memory_info().rss / (1024 * 1024)
|
||||
}
|
||||
self.active_requests[request_id] = req_info
|
||||
|
||||
# Increment endpoint counter
|
||||
if endpoint not in self.endpoint_stats:
|
||||
self.endpoint_stats[endpoint] = {
|
||||
"count": 0, "total_time": 0, "errors": 0,
|
||||
"pool_hits": 0, "success": 0
|
||||
}
|
||||
self.endpoint_stats[endpoint]["count"] += 1
|
||||
|
||||
# Queue persistence (handled by background worker)
|
||||
try:
|
||||
self._persist_queue.put_nowait(True)
|
||||
except asyncio.QueueFull:
|
||||
logger.warning("Persistence queue full, skipping")
|
||||
|
||||
async def track_request_end(self, request_id: str, success: bool, error: str = None,
|
||||
pool_hit: bool = True, status_code: int = 200):
|
||||
"""Track request completion."""
|
||||
if request_id not in self.active_requests:
|
||||
return
|
||||
|
||||
req_info = self.active_requests.pop(request_id)
|
||||
end_time = time.time()
|
||||
elapsed = end_time - req_info["start_time"]
|
||||
mem_end = psutil.Process().memory_info().rss / (1024 * 1024)
|
||||
mem_delta = mem_end - req_info["mem_start"]
|
||||
|
||||
# Update stats
|
||||
endpoint = req_info["endpoint"]
|
||||
if endpoint in self.endpoint_stats:
|
||||
self.endpoint_stats[endpoint]["total_time"] += elapsed
|
||||
if success:
|
||||
self.endpoint_stats[endpoint]["success"] += 1
|
||||
else:
|
||||
self.endpoint_stats[endpoint]["errors"] += 1
|
||||
if pool_hit:
|
||||
self.endpoint_stats[endpoint]["pool_hits"] += 1
|
||||
|
||||
# Add to completed queue
|
||||
completed = {
|
||||
**req_info,
|
||||
"end_time": end_time,
|
||||
"elapsed": round(elapsed, 2),
|
||||
"mem_delta": round(mem_delta, 1),
|
||||
"success": success,
|
||||
"error": error,
|
||||
"status_code": status_code,
|
||||
"pool_hit": pool_hit
|
||||
}
|
||||
self.completed_requests.append(completed)
|
||||
|
||||
# Track errors
|
||||
if not success and error:
|
||||
self.errors.append({
|
||||
"timestamp": end_time,
|
||||
"endpoint": endpoint,
|
||||
"url": req_info["url"],
|
||||
"error": html.escape(str(error)),
|
||||
"request_id": request_id
|
||||
})
|
||||
|
||||
await self._persist_endpoint_stats()
|
||||
|
||||
async def track_janitor_event(self, event_type: str, sig: str, details: Dict):
|
||||
"""Track janitor cleanup events."""
|
||||
self.janitor_events.append({
|
||||
"timestamp": time.time(),
|
||||
"type": event_type, # "close_cold", "close_hot", "promote"
|
||||
"sig": sig[:8],
|
||||
"details": details
|
||||
})
|
||||
|
||||
def _cleanup_old_entries(self, max_age_seconds: int = 300):
|
||||
"""Remove entries older than max_age_seconds (default 5min)."""
|
||||
now = time.time()
|
||||
cutoff = now - max_age_seconds
|
||||
|
||||
# Clean completed requests
|
||||
while self.completed_requests and self.completed_requests[0].get("end_time", 0) < cutoff:
|
||||
self.completed_requests.popleft()
|
||||
|
||||
# Clean janitor events
|
||||
while self.janitor_events and self.janitor_events[0].get("timestamp", 0) < cutoff:
|
||||
self.janitor_events.popleft()
|
||||
|
||||
# Clean errors
|
||||
while self.errors and self.errors[0].get("timestamp", 0) < cutoff:
|
||||
self.errors.popleft()
|
||||
|
||||
async def update_timeline(self):
|
||||
"""Update timeline data points (called every 5s)."""
|
||||
now = time.time()
|
||||
mem_pct = get_container_memory_percent()
|
||||
|
||||
# Clean old entries (keep last 5 minutes)
|
||||
self._cleanup_old_entries(max_age_seconds=300)
|
||||
|
||||
# Count requests in last 5s
|
||||
recent_reqs = sum(1 for req in self.completed_requests
|
||||
if now - req.get("end_time", 0) < 5)
|
||||
|
||||
# Browser counts — lock-free snapshot to avoid contending on
|
||||
# the pool LOCK held during slow browser start/close (issue #1754)
|
||||
from crawler_pool import get_pool_snapshot
|
||||
snap = get_pool_snapshot()
|
||||
browser_count = {
|
||||
"permanent": 1 if snap["permanent"] else 0,
|
||||
"hot": len(snap["hot_pool"]),
|
||||
"cold": len(snap["cold_pool"]),
|
||||
}
|
||||
|
||||
self.memory_timeline.append({"time": now, "value": mem_pct})
|
||||
self.requests_timeline.append({"time": now, "value": recent_reqs})
|
||||
self.browser_timeline.append({"time": now, "browsers": browser_count})
|
||||
|
||||
async def _persist_endpoint_stats(self):
|
||||
"""Persist endpoint stats to Redis."""
|
||||
try:
|
||||
await self.redis.set(
|
||||
"monitor:endpoint_stats",
|
||||
json.dumps(self.endpoint_stats),
|
||||
ex=86400 # 24h TTL
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to persist endpoint stats: {e}")
|
||||
|
||||
async def _persistence_worker(self):
|
||||
"""Background worker to persist stats to Redis."""
|
||||
while True:
|
||||
try:
|
||||
await self._persist_queue.get()
|
||||
await self._persist_endpoint_stats()
|
||||
self._persist_queue.task_done()
|
||||
except asyncio.CancelledError:
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Persistence worker error: {e}")
|
||||
|
||||
def start_persistence_worker(self):
|
||||
"""Start the background persistence worker."""
|
||||
if not self._persist_worker_task:
|
||||
self._persist_worker_task = asyncio.create_task(self._persistence_worker())
|
||||
logger.info("Started persistence worker")
|
||||
|
||||
async def stop_persistence_worker(self):
|
||||
"""Stop the background persistence worker."""
|
||||
if self._persist_worker_task:
|
||||
self._persist_worker_task.cancel()
|
||||
try:
|
||||
await self._persist_worker_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
self._persist_worker_task = None
|
||||
logger.info("Stopped persistence worker")
|
||||
|
||||
async def cleanup(self):
|
||||
"""Cleanup on shutdown - persist final stats and stop workers."""
|
||||
logger.info("Monitor cleanup starting...")
|
||||
try:
|
||||
# Persist final stats before shutdown
|
||||
await self._persist_endpoint_stats()
|
||||
# Stop background worker
|
||||
await self.stop_persistence_worker()
|
||||
logger.info("Monitor cleanup completed")
|
||||
except Exception as e:
|
||||
logger.error(f"Monitor cleanup error: {e}")
|
||||
|
||||
async def load_from_redis(self):
|
||||
"""Load persisted stats from Redis."""
|
||||
try:
|
||||
data = await self.redis.get("monitor:endpoint_stats")
|
||||
if data:
|
||||
self.endpoint_stats = json.loads(data)
|
||||
logger.info("Loaded endpoint stats from Redis")
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to load from Redis: {e}")
|
||||
|
||||
async def get_health_summary(self) -> Dict:
|
||||
"""Get current system health snapshot."""
|
||||
mem_pct = get_container_memory_percent()
|
||||
cpu_pct = psutil.cpu_percent(interval=0.1)
|
||||
|
||||
# Network I/O (delta since last call)
|
||||
net = psutil.net_io_counters()
|
||||
|
||||
# Pool status — lock-free snapshot (issue #1754)
|
||||
from crawler_pool import get_pool_snapshot
|
||||
snap = get_pool_snapshot()
|
||||
# TODO: Track actual browser process memory instead of estimates
|
||||
# These are conservative estimates based on typical Chromium usage
|
||||
permanent_mem = 270 if snap["permanent"] else 0 # Estimate: ~270MB for permanent browser
|
||||
hot_mem = len(snap["hot_pool"]) * 180 # Estimate: ~180MB per hot pool browser
|
||||
cold_mem = len(snap["cold_pool"]) * 180 # Estimate: ~180MB per cold pool browser
|
||||
permanent_active = snap["permanent"] is not None
|
||||
hot_count = len(snap["hot_pool"])
|
||||
cold_count = len(snap["cold_pool"])
|
||||
|
||||
return {
|
||||
"container": {
|
||||
"memory_percent": round(mem_pct, 1),
|
||||
"cpu_percent": round(cpu_pct, 1),
|
||||
"network_sent_mb": round(net.bytes_sent / (1024**2), 2),
|
||||
"network_recv_mb": round(net.bytes_recv / (1024**2), 2),
|
||||
"uptime_seconds": int(time.time() - self.start_time)
|
||||
},
|
||||
"pool": {
|
||||
"permanent": {"active": permanent_active, "memory_mb": permanent_mem},
|
||||
"hot": {"count": hot_count, "memory_mb": hot_mem},
|
||||
"cold": {"count": cold_count, "memory_mb": cold_mem},
|
||||
"total_memory_mb": permanent_mem + hot_mem + cold_mem
|
||||
},
|
||||
"janitor": {
|
||||
"next_cleanup_estimate": "adaptive", # Would need janitor state
|
||||
"memory_pressure": "LOW" if mem_pct < 60 else "MEDIUM" if mem_pct < 80 else "HIGH"
|
||||
}
|
||||
}
|
||||
|
||||
def get_active_requests(self) -> List[Dict]:
|
||||
"""Get list of currently active requests."""
|
||||
now = time.time()
|
||||
return [
|
||||
{
|
||||
**req,
|
||||
"elapsed": round(now - req["start_time"], 1),
|
||||
"status": "running"
|
||||
}
|
||||
for req in self.active_requests.values()
|
||||
]
|
||||
|
||||
def get_completed_requests(self, limit: int = 50, filter_status: str = "all") -> List[Dict]:
|
||||
"""Get recent completed requests."""
|
||||
requests = list(self.completed_requests)[-limit:]
|
||||
if filter_status == "success":
|
||||
requests = [r for r in requests if r.get("success")]
|
||||
elif filter_status == "error":
|
||||
requests = [r for r in requests if not r.get("success")]
|
||||
return requests
|
||||
|
||||
async def get_browser_list(self) -> List[Dict]:
|
||||
"""Get detailed browser pool information."""
|
||||
from crawler_pool import get_pool_snapshot
|
||||
|
||||
browsers = []
|
||||
now = time.time()
|
||||
|
||||
# Lock-free snapshot — iterates over copies (issue #1754)
|
||||
snap = get_pool_snapshot()
|
||||
permanent = snap["permanent"]
|
||||
permanent_sig = snap["permanent_sig"]
|
||||
hot_pool = snap["hot_pool"]
|
||||
cold_pool = snap["cold_pool"]
|
||||
last_used = snap["last_used"]
|
||||
usage_count = snap["usage_count"]
|
||||
|
||||
if permanent:
|
||||
browsers.append({
|
||||
"type": "permanent",
|
||||
"sig": permanent_sig[:8] if permanent_sig else "unknown",
|
||||
"age_seconds": int(now - self.start_time),
|
||||
"last_used_seconds": int(now - last_used.get(permanent_sig, now)),
|
||||
"memory_mb": 270,
|
||||
"hits": usage_count.get(permanent_sig, 0),
|
||||
"killable": False
|
||||
})
|
||||
|
||||
for sig, crawler in hot_pool.items():
|
||||
browsers.append({
|
||||
"type": "hot",
|
||||
"sig": sig[:8],
|
||||
"age_seconds": int(now - self.start_time), # Approximation
|
||||
"last_used_seconds": int(now - last_used.get(sig, now)),
|
||||
"memory_mb": 180, # Estimate
|
||||
"hits": usage_count.get(sig, 0),
|
||||
"killable": True
|
||||
})
|
||||
|
||||
for sig, crawler in cold_pool.items():
|
||||
browsers.append({
|
||||
"type": "cold",
|
||||
"sig": sig[:8],
|
||||
"age_seconds": int(now - self.start_time),
|
||||
"last_used_seconds": int(now - last_used.get(sig, now)),
|
||||
"memory_mb": 180,
|
||||
"hits": usage_count.get(sig, 0),
|
||||
"killable": True
|
||||
})
|
||||
|
||||
return browsers
|
||||
|
||||
def get_endpoint_stats_summary(self) -> Dict[str, Dict]:
|
||||
"""Get aggregated endpoint statistics."""
|
||||
summary = {}
|
||||
for endpoint, stats in self.endpoint_stats.items():
|
||||
count = stats["count"]
|
||||
avg_time = (stats["total_time"] / count) if count > 0 else 0
|
||||
success_rate = (stats["success"] / count * 100) if count > 0 else 0
|
||||
pool_hit_rate = (stats["pool_hits"] / count * 100) if count > 0 else 0
|
||||
|
||||
summary[endpoint] = {
|
||||
"count": count,
|
||||
"avg_latency_ms": round(avg_time * 1000, 1),
|
||||
"success_rate_percent": round(success_rate, 1),
|
||||
"pool_hit_rate_percent": round(pool_hit_rate, 1),
|
||||
"errors": stats["errors"]
|
||||
}
|
||||
return summary
|
||||
|
||||
def get_timeline_data(self, metric: str, window: str = "5m") -> Dict:
|
||||
"""Get timeline data for charts."""
|
||||
# For now, only 5m window supported
|
||||
if metric == "memory":
|
||||
data = list(self.memory_timeline)
|
||||
elif metric == "requests":
|
||||
data = list(self.requests_timeline)
|
||||
elif metric == "browsers":
|
||||
data = list(self.browser_timeline)
|
||||
else:
|
||||
return {"timestamps": [], "values": []}
|
||||
|
||||
return {
|
||||
"timestamps": [int(d["time"]) for d in data],
|
||||
"values": [d.get("value", d.get("browsers")) for d in data]
|
||||
}
|
||||
|
||||
def get_janitor_log(self, limit: int = 100) -> List[Dict]:
|
||||
"""Get recent janitor events."""
|
||||
return list(self.janitor_events)[-limit:]
|
||||
|
||||
def get_errors_log(self, limit: int = 100) -> List[Dict]:
|
||||
"""Get recent errors."""
|
||||
return list(self.errors)[-limit:]
|
||||
|
||||
# Global instance (initialized in server.py)
|
||||
monitor_stats: Optional[MonitorStats] = None
|
||||
|
||||
def get_monitor() -> MonitorStats:
|
||||
"""Get global monitor instance."""
|
||||
if monitor_stats is None:
|
||||
raise RuntimeError("Monitor not initialized")
|
||||
return monitor_stats
|
||||
@@ -0,0 +1,409 @@
|
||||
# monitor_routes.py - Monitor API endpoints
|
||||
from fastapi import APIRouter, HTTPException, WebSocket, WebSocketDisconnect, Depends
|
||||
from pydantic import BaseModel
|
||||
from typing import Optional
|
||||
from monitor import get_monitor
|
||||
from auth import require_admin
|
||||
import logging
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
router = APIRouter(prefix="/monitor", tags=["monitor"])
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
async def get_health():
|
||||
"""Get current system health snapshot."""
|
||||
try:
|
||||
monitor = get_monitor()
|
||||
return await monitor.get_health_summary()
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting health: {e}")
|
||||
raise HTTPException(500, str(e))
|
||||
|
||||
|
||||
@router.get("/requests")
|
||||
async def get_requests(status: str = "all", limit: int = 50):
|
||||
"""Get active and completed requests.
|
||||
|
||||
Args:
|
||||
status: Filter by 'active', 'completed', 'success', 'error', or 'all'
|
||||
limit: Max number of completed requests to return (default 50)
|
||||
"""
|
||||
# Input validation
|
||||
if status not in ["all", "active", "completed", "success", "error"]:
|
||||
raise HTTPException(400, f"Invalid status: {status}. Must be one of: all, active, completed, success, error")
|
||||
if limit < 1 or limit > 1000:
|
||||
raise HTTPException(400, f"Invalid limit: {limit}. Must be between 1 and 1000")
|
||||
|
||||
try:
|
||||
monitor = get_monitor()
|
||||
|
||||
if status == "active":
|
||||
return {"active": monitor.get_active_requests(), "completed": []}
|
||||
elif status == "completed":
|
||||
return {"active": [], "completed": monitor.get_completed_requests(limit)}
|
||||
elif status in ["success", "error"]:
|
||||
return {"active": [], "completed": monitor.get_completed_requests(limit, status)}
|
||||
else: # "all"
|
||||
return {
|
||||
"active": monitor.get_active_requests(),
|
||||
"completed": monitor.get_completed_requests(limit)
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting requests: {e}")
|
||||
raise HTTPException(500, str(e))
|
||||
|
||||
|
||||
@router.get("/browsers")
|
||||
async def get_browsers():
|
||||
"""Get detailed browser pool information."""
|
||||
try:
|
||||
monitor = get_monitor()
|
||||
browsers = await monitor.get_browser_list()
|
||||
|
||||
# Calculate summary stats
|
||||
total_browsers = len(browsers)
|
||||
total_memory = sum(b["memory_mb"] for b in browsers)
|
||||
|
||||
# Calculate reuse rate from recent requests
|
||||
recent = monitor.get_completed_requests(100)
|
||||
pool_hits = sum(1 for r in recent if r.get("pool_hit", False))
|
||||
reuse_rate = (pool_hits / len(recent) * 100) if recent else 0
|
||||
|
||||
return {
|
||||
"browsers": browsers,
|
||||
"summary": {
|
||||
"total_count": total_browsers,
|
||||
"total_memory_mb": total_memory,
|
||||
"reuse_rate_percent": round(reuse_rate, 1)
|
||||
}
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting browsers: {e}")
|
||||
raise HTTPException(500, str(e))
|
||||
|
||||
|
||||
@router.get("/endpoints/stats")
|
||||
async def get_endpoint_stats():
|
||||
"""Get aggregated endpoint statistics."""
|
||||
try:
|
||||
monitor = get_monitor()
|
||||
return monitor.get_endpoint_stats_summary()
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting endpoint stats: {e}")
|
||||
raise HTTPException(500, str(e))
|
||||
|
||||
|
||||
@router.get("/timeline")
|
||||
async def get_timeline(metric: str = "memory", window: str = "5m"):
|
||||
"""Get timeline data for charts.
|
||||
|
||||
Args:
|
||||
metric: 'memory', 'requests', or 'browsers'
|
||||
window: Time window (only '5m' supported for now)
|
||||
"""
|
||||
# Input validation
|
||||
if metric not in ["memory", "requests", "browsers"]:
|
||||
raise HTTPException(400, f"Invalid metric: {metric}. Must be one of: memory, requests, browsers")
|
||||
if window != "5m":
|
||||
raise HTTPException(400, f"Invalid window: {window}. Only '5m' is currently supported")
|
||||
|
||||
try:
|
||||
monitor = get_monitor()
|
||||
return monitor.get_timeline_data(metric, window)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting timeline: {e}")
|
||||
raise HTTPException(500, str(e))
|
||||
|
||||
|
||||
@router.get("/logs/janitor")
|
||||
async def get_janitor_log(limit: int = 100):
|
||||
"""Get recent janitor cleanup events."""
|
||||
# Input validation
|
||||
if limit < 1 or limit > 1000:
|
||||
raise HTTPException(400, f"Invalid limit: {limit}. Must be between 1 and 1000")
|
||||
|
||||
try:
|
||||
monitor = get_monitor()
|
||||
return {"events": monitor.get_janitor_log(limit)}
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting janitor log: {e}")
|
||||
raise HTTPException(500, str(e))
|
||||
|
||||
|
||||
@router.get("/logs/errors")
|
||||
async def get_errors_log(limit: int = 100):
|
||||
"""Get recent errors."""
|
||||
# Input validation
|
||||
if limit < 1 or limit > 1000:
|
||||
raise HTTPException(400, f"Invalid limit: {limit}. Must be between 1 and 1000")
|
||||
|
||||
try:
|
||||
monitor = get_monitor()
|
||||
return {"errors": monitor.get_errors_log(limit)}
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting errors log: {e}")
|
||||
raise HTTPException(500, str(e))
|
||||
|
||||
|
||||
# ========== Control Actions ==========
|
||||
|
||||
class KillBrowserRequest(BaseModel):
|
||||
sig: str
|
||||
|
||||
|
||||
@router.post("/actions/cleanup", dependencies=[Depends(require_admin)])
|
||||
async def force_cleanup():
|
||||
"""Force immediate janitor cleanup (kills idle cold pool browsers)."""
|
||||
try:
|
||||
from crawler_pool import COLD_POOL, LAST_USED, USAGE_COUNT, LOCK
|
||||
import time
|
||||
from contextlib import suppress
|
||||
|
||||
killed_count = 0
|
||||
now = time.time()
|
||||
|
||||
async with LOCK:
|
||||
for sig in list(COLD_POOL.keys()):
|
||||
# Kill all cold pool browsers immediately
|
||||
logger.info(f"🧹 Force cleanup: closing cold browser (sig={sig[:8]})")
|
||||
with suppress(Exception):
|
||||
await COLD_POOL[sig].close()
|
||||
COLD_POOL.pop(sig, None)
|
||||
LAST_USED.pop(sig, None)
|
||||
USAGE_COUNT.pop(sig, None)
|
||||
killed_count += 1
|
||||
|
||||
monitor = get_monitor()
|
||||
await monitor.track_janitor_event("force_cleanup", "manual", {"killed": killed_count})
|
||||
|
||||
return {"success": True, "killed_browsers": killed_count}
|
||||
except Exception as e:
|
||||
logger.error(f"Error during force cleanup: {e}")
|
||||
raise HTTPException(500, str(e))
|
||||
|
||||
|
||||
@router.post("/actions/kill_browser", dependencies=[Depends(require_admin)])
|
||||
async def kill_browser(req: KillBrowserRequest):
|
||||
"""Kill a specific browser by signature (hot or cold only).
|
||||
|
||||
Args:
|
||||
sig: Browser config signature (first 8 chars)
|
||||
"""
|
||||
try:
|
||||
from crawler_pool import HOT_POOL, COLD_POOL, LAST_USED, USAGE_COUNT, LOCK, DEFAULT_CONFIG_SIG
|
||||
from contextlib import suppress
|
||||
|
||||
# Find full signature matching prefix
|
||||
target_sig = None
|
||||
pool_type = None
|
||||
|
||||
async with LOCK:
|
||||
# Check hot pool
|
||||
for sig in HOT_POOL.keys():
|
||||
if sig.startswith(req.sig):
|
||||
target_sig = sig
|
||||
pool_type = "hot"
|
||||
break
|
||||
|
||||
# Check cold pool
|
||||
if not target_sig:
|
||||
for sig in COLD_POOL.keys():
|
||||
if sig.startswith(req.sig):
|
||||
target_sig = sig
|
||||
pool_type = "cold"
|
||||
break
|
||||
|
||||
# Check if trying to kill permanent
|
||||
if DEFAULT_CONFIG_SIG and DEFAULT_CONFIG_SIG.startswith(req.sig):
|
||||
raise HTTPException(403, "Cannot kill permanent browser. Use restart instead.")
|
||||
|
||||
if not target_sig:
|
||||
raise HTTPException(404, f"Browser with sig={req.sig} not found")
|
||||
|
||||
# Warn if there are active requests (browser might be in use)
|
||||
monitor = get_monitor()
|
||||
active_count = len(monitor.get_active_requests())
|
||||
if active_count > 0:
|
||||
logger.warning(f"Killing browser {target_sig[:8]} while {active_count} requests are active - may cause failures")
|
||||
|
||||
# Kill the browser
|
||||
if pool_type == "hot":
|
||||
browser = HOT_POOL.pop(target_sig)
|
||||
else:
|
||||
browser = COLD_POOL.pop(target_sig)
|
||||
|
||||
with suppress(Exception):
|
||||
await browser.close()
|
||||
|
||||
LAST_USED.pop(target_sig, None)
|
||||
USAGE_COUNT.pop(target_sig, None)
|
||||
|
||||
logger.info(f"🔪 Killed {pool_type} browser (sig={target_sig[:8]})")
|
||||
|
||||
monitor = get_monitor()
|
||||
await monitor.track_janitor_event("kill_browser", target_sig, {"pool": pool_type, "manual": True})
|
||||
|
||||
return {"success": True, "killed_sig": target_sig[:8], "pool_type": pool_type}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error killing browser: {e}")
|
||||
raise HTTPException(500, str(e))
|
||||
|
||||
|
||||
@router.post("/actions/restart_browser", dependencies=[Depends(require_admin)])
|
||||
async def restart_browser(req: KillBrowserRequest):
|
||||
"""Restart a browser (kill + recreate). Works for permanent too.
|
||||
|
||||
Args:
|
||||
sig: Browser config signature (first 8 chars), or "permanent"
|
||||
"""
|
||||
try:
|
||||
from crawler_pool import (PERMANENT, HOT_POOL, COLD_POOL, LAST_USED,
|
||||
USAGE_COUNT, LOCK, DEFAULT_CONFIG_SIG, init_permanent)
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig
|
||||
from contextlib import suppress
|
||||
import time
|
||||
|
||||
# Handle permanent browser restart
|
||||
if req.sig == "permanent" or (DEFAULT_CONFIG_SIG and DEFAULT_CONFIG_SIG.startswith(req.sig)):
|
||||
async with LOCK:
|
||||
if PERMANENT:
|
||||
with suppress(Exception):
|
||||
await PERMANENT.close()
|
||||
|
||||
# Reinitialize permanent
|
||||
from utils import load_config
|
||||
config = load_config()
|
||||
await init_permanent(BrowserConfig(
|
||||
extra_args=config["crawler"]["browser"].get("extra_args", []),
|
||||
**config["crawler"]["browser"].get("kwargs", {}),
|
||||
))
|
||||
|
||||
logger.info("🔄 Restarted permanent browser")
|
||||
return {"success": True, "restarted": "permanent"}
|
||||
|
||||
# Handle hot/cold browser restart
|
||||
target_sig = None
|
||||
pool_type = None
|
||||
browser_config = None
|
||||
|
||||
async with LOCK:
|
||||
# Find browser
|
||||
for sig in HOT_POOL.keys():
|
||||
if sig.startswith(req.sig):
|
||||
target_sig = sig
|
||||
pool_type = "hot"
|
||||
# Would need to reconstruct config (not stored currently)
|
||||
break
|
||||
|
||||
if not target_sig:
|
||||
for sig in COLD_POOL.keys():
|
||||
if sig.startswith(req.sig):
|
||||
target_sig = sig
|
||||
pool_type = "cold"
|
||||
break
|
||||
|
||||
if not target_sig:
|
||||
raise HTTPException(404, f"Browser with sig={req.sig} not found")
|
||||
|
||||
# Kill existing
|
||||
if pool_type == "hot":
|
||||
browser = HOT_POOL.pop(target_sig)
|
||||
else:
|
||||
browser = COLD_POOL.pop(target_sig)
|
||||
|
||||
with suppress(Exception):
|
||||
await browser.close()
|
||||
|
||||
# Note: We can't easily recreate with same config without storing it
|
||||
# For now, just kill and let new requests create fresh ones
|
||||
LAST_USED.pop(target_sig, None)
|
||||
USAGE_COUNT.pop(target_sig, None)
|
||||
|
||||
logger.info(f"🔄 Restarted {pool_type} browser (sig={target_sig[:8]})")
|
||||
|
||||
monitor = get_monitor()
|
||||
await monitor.track_janitor_event("restart_browser", target_sig, {"pool": pool_type})
|
||||
|
||||
return {"success": True, "restarted_sig": target_sig[:8], "note": "Browser will be recreated on next request"}
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error(f"Error restarting browser: {e}")
|
||||
raise HTTPException(500, str(e))
|
||||
|
||||
|
||||
@router.post("/stats/reset", dependencies=[Depends(require_admin)])
|
||||
async def reset_stats():
|
||||
"""Reset today's endpoint counters."""
|
||||
try:
|
||||
monitor = get_monitor()
|
||||
monitor.endpoint_stats.clear()
|
||||
await monitor._persist_endpoint_stats()
|
||||
|
||||
return {"success": True, "message": "Endpoint stats reset"}
|
||||
except Exception as e:
|
||||
logger.error(f"Error resetting stats: {e}")
|
||||
raise HTTPException(500, str(e))
|
||||
|
||||
|
||||
@router.websocket("/ws")
|
||||
async def websocket_endpoint(websocket: WebSocket):
|
||||
"""WebSocket endpoint for real-time monitoring updates.
|
||||
|
||||
Sends updates every 2 seconds with:
|
||||
- Health stats
|
||||
- Active/completed requests
|
||||
- Browser pool status
|
||||
- Timeline data
|
||||
"""
|
||||
# Auth is enforced by the AuthGateMiddleware (outermost ASGI layer), which
|
||||
# validates the ?token= query param / Authorization header for this
|
||||
# WebSocket and closes 4401 before we are reached. No bespoke check here.
|
||||
await websocket.accept()
|
||||
logger.info("WebSocket client connected")
|
||||
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
# Gather all monitoring data
|
||||
monitor = get_monitor()
|
||||
|
||||
data = {
|
||||
"timestamp": asyncio.get_event_loop().time(),
|
||||
"health": await monitor.get_health_summary(),
|
||||
"requests": {
|
||||
"active": monitor.get_active_requests(),
|
||||
"completed": monitor.get_completed_requests(limit=10)
|
||||
},
|
||||
"browsers": await monitor.get_browser_list(),
|
||||
"timeline": {
|
||||
"memory": monitor.get_timeline_data("memory", "5m"),
|
||||
"requests": monitor.get_timeline_data("requests", "5m"),
|
||||
"browsers": monitor.get_timeline_data("browsers", "5m")
|
||||
},
|
||||
"janitor": monitor.get_janitor_log(limit=10),
|
||||
"errors": monitor.get_errors_log(limit=10)
|
||||
}
|
||||
|
||||
# Send update to client
|
||||
await websocket.send_json(data)
|
||||
|
||||
# Wait 2 seconds before next update
|
||||
await asyncio.sleep(2)
|
||||
|
||||
except WebSocketDisconnect:
|
||||
logger.info("WebSocket client disconnected")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"WebSocket error: {e}", exc_info=True)
|
||||
await asyncio.sleep(2) # Continue trying
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"WebSocket connection error: {e}", exc_info=True)
|
||||
finally:
|
||||
logger.info("WebSocket connection closed")
|
||||
@@ -0,0 +1,75 @@
|
||||
"""Redis URL helpers for the Docker API server.
|
||||
|
||||
The Docker entrypoint protects the in-container Redis with a password. These
|
||||
helpers keep every Redis client (the app and optional rate-limit storage) using
|
||||
the same authenticated URL shape.
|
||||
"""
|
||||
|
||||
import os
|
||||
from urllib.parse import quote, urlsplit, urlunsplit
|
||||
|
||||
|
||||
def redis_auth_from_config(config: dict) -> tuple[str, str]:
|
||||
"""Return Redis ACL username/password from config and environment."""
|
||||
rc = config.get("redis", {})
|
||||
username = os.environ.get("REDIS_USERNAME", rc.get("username", "default")) or "default"
|
||||
password = os.environ.get("REDIS_PASSWORD", rc.get("password", "")) or ""
|
||||
return str(username), str(password)
|
||||
|
||||
|
||||
def redis_auth_netloc(username: str, password: str) -> str:
|
||||
"""Build a URL-safe Redis AUTH fragment for redis-py/limits clients.
|
||||
|
||||
Redis 6+ HELLO AUTH requires both username and password when negotiating
|
||||
RESP3. Supplying the explicit default ACL username avoids clients issuing an
|
||||
unauthenticated HELLO against a password-protected Redis instance.
|
||||
"""
|
||||
if not password:
|
||||
return ""
|
||||
return f"{quote(username, safe='')}:{quote(password, safe='')}@"
|
||||
|
||||
|
||||
def build_redis_url(config: dict) -> str:
|
||||
"""Build Redis URL from config fields and environment variables."""
|
||||
rc = config.get("redis", {})
|
||||
host = os.environ.get("REDIS_HOST", rc.get("host", "localhost"))
|
||||
port = os.environ.get("REDIS_PORT", rc.get("port", 6379))
|
||||
username, password = redis_auth_from_config(config)
|
||||
db = rc.get("db", 0)
|
||||
scheme = "rediss" if rc.get("ssl", False) else "redis"
|
||||
auth = redis_auth_netloc(username, password)
|
||||
return f"{scheme}://{auth}{host}:{port}/{db}"
|
||||
|
||||
|
||||
def build_rate_limit_storage_uri(config: dict) -> str:
|
||||
"""Return a rate-limit storage URI that can auth to protected Redis.
|
||||
|
||||
Older/self-managed Docker configs may set rate_limiting.storage_uri to an
|
||||
unauthenticated Redis URL such as redis://localhost:6379. Since the Docker
|
||||
entrypoint now always protects the in-container Redis with REDIS_PASSWORD,
|
||||
SlowAPI/limits would fail before route handlers with Redis HELLO auth errors.
|
||||
If the storage URI is Redis-like and has no credentials, reuse the configured
|
||||
Redis ACL credentials. Explicit credentials and non-Redis backends are left
|
||||
unchanged.
|
||||
"""
|
||||
storage_uri = config.get("rate_limiting", {}).get("storage_uri", "memory://")
|
||||
if not isinstance(storage_uri, str):
|
||||
return "memory://"
|
||||
parsed = urlsplit(storage_uri)
|
||||
if parsed.scheme not in {"redis", "rediss", "redis+sentinel"}:
|
||||
return storage_uri
|
||||
if parsed.username or parsed.password:
|
||||
return storage_uri
|
||||
|
||||
username, password = redis_auth_from_config(config)
|
||||
if not password:
|
||||
return storage_uri
|
||||
|
||||
auth = redis_auth_netloc(username, password)
|
||||
return urlunsplit((
|
||||
parsed.scheme,
|
||||
f"{auth}{parsed.netloc}",
|
||||
parsed.path,
|
||||
parsed.query,
|
||||
parsed.fragment,
|
||||
))
|
||||
@@ -0,0 +1,16 @@
|
||||
fastapi>=0.115.12,<0.137
|
||||
uvicorn>=0.34.2
|
||||
gunicorn>=23.0.0
|
||||
slowapi==0.1.9
|
||||
prometheus-fastapi-instrumentator>=7.1.0
|
||||
redis>=5.2.1
|
||||
dnspython>=2.7.0
|
||||
email-validator==2.2.0
|
||||
sse-starlette==2.2.1
|
||||
pydantic>=2.11
|
||||
rank-bm25==0.2.2
|
||||
anyio==4.9.0
|
||||
PyJWT==2.10.1
|
||||
mcp>=1.18.0
|
||||
websockets>=15.0.1
|
||||
httpx[http2]>=0.27.2
|
||||
@@ -0,0 +1,130 @@
|
||||
from typing import Any, List, Optional, Dict
|
||||
from enum import Enum
|
||||
from pydantic import BaseModel, Field, HttpUrl, field_validator
|
||||
from utils import FilterType
|
||||
|
||||
|
||||
class CrawlRequest(BaseModel):
|
||||
urls: List[str] = Field(min_length=1, max_length=100)
|
||||
browser_config: Optional[Dict] = Field(default_factory=dict)
|
||||
crawler_config: Optional[Dict] = Field(default_factory=dict)
|
||||
crawler_configs: Optional[List[Dict]] = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"List of per-URL CrawlerRunConfig dicts for arun_many(). "
|
||||
"When provided, each config can include a 'url_matcher' pattern "
|
||||
"to match against specific URLs. Takes precedence over crawler_config."
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class HookSpec(BaseModel):
|
||||
"""A single declarative hook: a fixed action plus schema-validated params.
|
||||
|
||||
Arbitrary Python (the old `code` map) is no longer accepted - it was an
|
||||
exec()-based RCE surface. Available actions are enumerated by GET /hooks/info
|
||||
and validated server-side by hook_registry.py.
|
||||
"""
|
||||
action: str = Field(..., description="One of the registered hook actions")
|
||||
params: Dict[str, Any] = Field(default_factory=dict, description="Action parameters")
|
||||
|
||||
|
||||
class HookConfig(BaseModel):
|
||||
"""Configuration for declarative hooks."""
|
||||
hooks: List[HookSpec] = Field(
|
||||
default_factory=list,
|
||||
max_length=10,
|
||||
description="Declarative hook specs (action + params), max 10",
|
||||
)
|
||||
timeout: int = Field(
|
||||
default=30,
|
||||
ge=1,
|
||||
le=120,
|
||||
description="Timeout in seconds for each hook execution",
|
||||
)
|
||||
|
||||
class Config:
|
||||
json_schema_extra = {
|
||||
"example": {
|
||||
"hooks": [
|
||||
{"action": "block_resources", "params": {"resource_types": ["image", "font"]}},
|
||||
{"action": "scroll_to_bottom", "params": {"max_steps": 10, "delay_ms": 500}},
|
||||
],
|
||||
"timeout": 30,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class CrawlRequestWithHooks(CrawlRequest):
|
||||
"""Extended crawl request with hooks support"""
|
||||
hooks: Optional[HookConfig] = Field(
|
||||
default=None,
|
||||
description="Optional user-provided hook functions"
|
||||
)
|
||||
|
||||
class MarkdownRequest(BaseModel):
|
||||
"""Request body for the /md endpoint."""
|
||||
url: str = Field(..., description="Absolute http/https URL to fetch")
|
||||
f: FilterType = Field(FilterType.FIT, description="Content‑filter strategy: fit, raw, bm25, or llm")
|
||||
q: Optional[str] = Field(None, description="Query string used by BM25/LLM filters")
|
||||
c: Optional[str] = Field("0", description="Cache‑bust / revision counter")
|
||||
provider: Optional[str] = Field(None, description="LLM provider override (e.g., 'anthropic/claude-3-opus')")
|
||||
temperature: Optional[float] = Field(None, description="LLM temperature override (0.0-2.0)")
|
||||
# base_url removed: a request-supplied LLM endpoint was a credential-exfil
|
||||
# vector. The endpoint is derived server-side from the provider name.
|
||||
|
||||
|
||||
class RawCode(BaseModel):
|
||||
code: str
|
||||
|
||||
class HTMLRequest(BaseModel):
|
||||
url: str
|
||||
|
||||
class ScreenshotRequest(BaseModel):
|
||||
url: str
|
||||
screenshot_wait_for: Optional[float] = 2
|
||||
wait_for_images: Optional[bool] = False
|
||||
# output_path removed: callers never name a filesystem path (it was an
|
||||
# arbitrary-write -> RCE vector). The server writes to the sandboxed
|
||||
# artifact store and returns an opaque artifact_id.
|
||||
|
||||
|
||||
class PDFRequest(BaseModel):
|
||||
url: str
|
||||
# output_path removed (see ScreenshotRequest).
|
||||
|
||||
|
||||
class JSEndpointRequest(BaseModel):
|
||||
url: str
|
||||
scripts: List[str] = Field(
|
||||
...,
|
||||
description="List of separated JavaScript snippets to execute"
|
||||
)
|
||||
|
||||
|
||||
class WebhookConfig(BaseModel):
|
||||
"""Configuration for webhook notifications."""
|
||||
webhook_url: HttpUrl
|
||||
webhook_data_in_payload: bool = False
|
||||
webhook_headers: Optional[Dict[str, str]] = None
|
||||
|
||||
@field_validator("webhook_headers")
|
||||
@classmethod
|
||||
def _validate_headers(cls, v):
|
||||
# Reject unsafe outbound headers early (422). Mirrors
|
||||
# webhook.sanitize_webhook_headers; kept inline to avoid an import cycle.
|
||||
if not v:
|
||||
return v
|
||||
from webhook import sanitize_webhook_headers
|
||||
return sanitize_webhook_headers(v)
|
||||
|
||||
|
||||
class WebhookPayload(BaseModel):
|
||||
"""Payload sent to webhook endpoints."""
|
||||
task_id: str
|
||||
task_type: str # "crawl", "llm_extraction", etc.
|
||||
status: str # "completed" or "failed"
|
||||
timestamp: str # ISO 8601 format
|
||||
urls: List[str]
|
||||
error: Optional[str] = None
|
||||
data: Optional[Dict] = None # Included only if webhook_data_in_payload=True
|
||||
File diff suppressed because it is too large
Load Diff
Binary file not shown.
|
After Width: | Height: | Size: 5.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 11 KiB |
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,34 @@
|
||||
[supervisord]
|
||||
nodaemon=true ; Run supervisord in the foreground
|
||||
logfile=/dev/null ; Log supervisord output to stdout/stderr
|
||||
logfile_maxbytes=0
|
||||
|
||||
[program:redis]
|
||||
; Loopback-only and password-protected. REDIS_PASSWORD is exported by
|
||||
; entrypoint.sh from the mounted secret; the app must supply it to connect.
|
||||
command=/usr/bin/redis-server --loglevel notice --bind 127.0.0.1 ::1 --requirepass "%(ENV_REDIS_PASSWORD)s" --dir /var/lib/redis
|
||||
user=appuser ; Run redis as our non-root user
|
||||
autorestart=true
|
||||
priority=10
|
||||
stdout_logfile=/dev/stdout ; Redirect redis stdout to container stdout
|
||||
stdout_logfile_maxbytes=0
|
||||
stderr_logfile=/dev/stderr ; Redirect redis stderr to container stderr
|
||||
stderr_logfile_maxbytes=0
|
||||
|
||||
[program:gunicorn]
|
||||
; Bind is resolved by entrypoint.sh into GUNICORN_BIND: loopback unless a
|
||||
; credential (CRAWL4AI_API_TOKEN) is present, in which case the operator may
|
||||
; expose 0.0.0.0. gunicorn is the authoritative socket-level guard; the Python
|
||||
; _resolve_auth() check is the in-process guard - both must agree.
|
||||
command=/usr/local/bin/gunicorn --bind "%(ENV_GUNICORN_BIND)s" --workers 1 --threads 4 --timeout 1800 --graceful-timeout 30 --keep-alive 300 --log-level info --limit-request-line 8190 --limit-request-fields 100 --worker-class uvicorn.workers.UvicornWorker server:app
|
||||
directory=/app ; Working directory for the app
|
||||
user=appuser ; Run gunicorn as our non-root user
|
||||
autorestart=true
|
||||
priority=20
|
||||
environment=PYTHONUNBUFFERED=1 ; Ensure Python output is sent straight to logs
|
||||
stdout_logfile=/dev/stdout ; Redirect gunicorn stdout to container stdout
|
||||
stdout_logfile_maxbytes=0
|
||||
stderr_logfile=/dev/stderr ; Redirect gunicorn stderr to container stderr
|
||||
stderr_logfile_maxbytes=0
|
||||
|
||||
# Optional: Add filebeat or other logging agents here if needed
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Quick WebSocket test - Connect to monitor WebSocket and print updates
|
||||
"""
|
||||
import asyncio
|
||||
import websockets
|
||||
import json
|
||||
|
||||
async def test_websocket():
|
||||
uri = "ws://localhost:11235/monitor/ws"
|
||||
print(f"Connecting to {uri}...")
|
||||
|
||||
try:
|
||||
async with websockets.connect(uri) as websocket:
|
||||
print("✅ Connected!")
|
||||
|
||||
# Receive and print 5 updates
|
||||
for i in range(5):
|
||||
message = await websocket.recv()
|
||||
data = json.loads(message)
|
||||
print(f"\n📊 Update #{i+1}:")
|
||||
print(f" - Health: CPU {data['health']['container']['cpu_percent']}%, Memory {data['health']['container']['memory_percent']}%")
|
||||
print(f" - Active Requests: {len(data['requests']['active'])}")
|
||||
print(f" - Browsers: {len(data['browsers'])}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
return 1
|
||||
|
||||
print("\n✅ WebSocket test passed!")
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit(asyncio.run(test_websocket()))
|
||||
@@ -0,0 +1,168 @@
|
||||
"""
|
||||
Shared fixtures for the Crawl4AI Docker server *behavioral* security tests.
|
||||
|
||||
Unlike the legacy `test_security_*.py` suites (which grep the source text and
|
||||
pass even when the running server is wide open), these fixtures boot the real
|
||||
FastAPI application via Starlette's TestClient and exercise it as a client
|
||||
would. That makes the tests behavioral: they assert what the server actually
|
||||
*does*, not what the source happens to contain.
|
||||
|
||||
Design notes
|
||||
------------
|
||||
* The app is imported once per session. `deploy/docker` is put on `sys.path`
|
||||
first, because `server.py` does `from crawler_pool import ...` at module top,
|
||||
before its own `sys.path.append`.
|
||||
* The TestClient is created *without* the `with` context-manager form on
|
||||
purpose: that skips the FastAPI lifespan, so no Chromium is launched and no
|
||||
Redis connection is opened at startup. Authentication is decided by a
|
||||
dependency/middleware that runs before any route handler, so auth-posture
|
||||
assertions resolve before the app ever needs a browser or Redis.
|
||||
* `offline_dns` / `rebinding_dns` monkeypatch name resolution so the SSRF /
|
||||
egress behavioral tests (R3) run fully offline and can model DNS-rebinding
|
||||
(resolve-then-reconnect TOCTOU) deterministically.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import socket
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# deploy/docker (the dir that holds server.py, auth.py, ...) must be importable
|
||||
# before we import `server`, since server.py imports its siblings by bare name.
|
||||
DOCKER_DIR = Path(__file__).resolve().parents[1]
|
||||
if str(DOCKER_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(DOCKER_DIR))
|
||||
|
||||
|
||||
def pytest_configure(config):
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"posture: the secure-by-default acceptance gate (expected RED until R1-R7 land)",
|
||||
)
|
||||
config.addinivalue_line(
|
||||
"markers", "cve: regression test for a specific reported vulnerability"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def server_module():
|
||||
"""Import the Docker server app once for the whole test session.
|
||||
|
||||
Returns the imported `server` module so tests can introspect the loaded
|
||||
config, the redis-url builder, feature flags, etc.
|
||||
"""
|
||||
# Import fresh; if a previous test mutated it we still want the real module.
|
||||
if "server" in sys.modules:
|
||||
return sys.modules["server"]
|
||||
return importlib.import_module("server")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stock_client(server_module):
|
||||
"""A TestClient bound to the app as shipped (config.yml defaults).
|
||||
|
||||
No lifespan => no real browser, no Redis connect. Auth/headers/routing are
|
||||
all exercised normally because they sit in front of the route handlers.
|
||||
"""
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
# raise_server_exceptions=False so an un-gated handler that blows up on a
|
||||
# missing pool/redis surfaces as a 500 response (which our posture test
|
||||
# treats as "not 401" => still a finding) instead of bubbling out of the
|
||||
# test client as an exception.
|
||||
return TestClient(server_module.app, raise_server_exceptions=False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def effective_config(server_module):
|
||||
"""The configuration dict the running app actually loaded."""
|
||||
return server_module.config
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def effective_browser_args(effective_config):
|
||||
"""The Chromium launch flags the app will pass to every browser."""
|
||||
return list(effective_config["crawler"]["browser"].get("extra_args", []))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def effective_redis_url(server_module):
|
||||
"""The Redis URL the app builds from config + environment."""
|
||||
return server_module._build_redis_url(server_module.config)
|
||||
|
||||
|
||||
# ─────────────────────── offline DNS control ────────────────────────
|
||||
# These let SSRF/egress tests run without touching the network and let us
|
||||
# model DNS rebinding precisely.
|
||||
|
||||
class _FakeResolver:
|
||||
"""Drop-in for socket.getaddrinfo with a controllable host->IP map.
|
||||
|
||||
Unknown hosts resolve to a stable public-ish default so accidental real
|
||||
lookups never escape the test process.
|
||||
"""
|
||||
|
||||
DEFAULT_IP = "93.184.216.34" # documentation/example address space
|
||||
|
||||
def __init__(self):
|
||||
self.map = {} # host -> ip (single answer)
|
||||
|
||||
def set(self, host, ip):
|
||||
self.map[host] = ip
|
||||
|
||||
def getaddrinfo(self, host, port, *args, **kwargs):
|
||||
ip = self.map.get(host, self.DEFAULT_IP)
|
||||
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, port or 0))]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def offline_dns(monkeypatch):
|
||||
"""Patch socket.getaddrinfo with a controllable, offline resolver.
|
||||
|
||||
Usage:
|
||||
offline_dns.set("evil.example", "169.254.169.254")
|
||||
"""
|
||||
resolver = _FakeResolver()
|
||||
monkeypatch.setattr(socket, "getaddrinfo", resolver.getaddrinfo)
|
||||
return resolver
|
||||
|
||||
|
||||
class _RebindingResolver:
|
||||
"""Returns a *different* answer on the second+ lookup of the same host.
|
||||
|
||||
Models the resolve-then-discard TOCTOU: validation sees a public IP, the
|
||||
later real connection re-resolves to an internal IP. A correct egress
|
||||
broker resolves once and pins, so the second (internal) answer is never
|
||||
dialed.
|
||||
"""
|
||||
|
||||
def __init__(self, host, first_ip, second_ip):
|
||||
self.host = host
|
||||
self.first_ip = first_ip
|
||||
self.second_ip = second_ip
|
||||
self.calls = 0
|
||||
|
||||
def getaddrinfo(self, host, port, *args, **kwargs):
|
||||
if host == self.host:
|
||||
self.calls += 1
|
||||
ip = self.first_ip if self.calls == 1 else self.second_ip
|
||||
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", (ip, port or 0))]
|
||||
return [(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", port or 0))]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def rebinding_dns(monkeypatch):
|
||||
"""Factory that installs a DNS-rebinding resolver.
|
||||
|
||||
Usage:
|
||||
r = rebinding_dns("rebind.example", "93.184.216.34", "169.254.169.254")
|
||||
... # first lookup public, every later lookup internal
|
||||
"""
|
||||
def _install(host, first_ip, second_ip):
|
||||
resolver = _RebindingResolver(host, first_ip, second_ip)
|
||||
monkeypatch.setattr(socket, "getaddrinfo", resolver.getaddrinfo)
|
||||
return resolver
|
||||
|
||||
return _install
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Monitor Dashboard Demo Script
|
||||
Generates varied activity to showcase all monitoring features for video recording.
|
||||
"""
|
||||
import httpx
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import datetime
|
||||
|
||||
BASE_URL = "http://localhost:11235"
|
||||
|
||||
async def demo_dashboard():
|
||||
print("🎬 Monitor Dashboard Demo - Starting...\n")
|
||||
print(f"📊 Dashboard: {BASE_URL}/dashboard")
|
||||
print("=" * 60)
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
|
||||
# Phase 1: Simple requests (permanent browser)
|
||||
print("\n🔷 Phase 1: Testing permanent browser pool")
|
||||
print("-" * 60)
|
||||
for i in range(5):
|
||||
print(f" {i+1}/5 Request to /crawl (default config)...")
|
||||
try:
|
||||
r = await client.post(
|
||||
f"{BASE_URL}/crawl",
|
||||
json={"urls": [f"https://httpbin.org/html?req={i}"], "crawler_config": {}}
|
||||
)
|
||||
print(f" ✅ Status: {r.status_code}, Time: {r.elapsed.total_seconds():.2f}s")
|
||||
except Exception as e:
|
||||
print(f" ❌ Error: {e}")
|
||||
await asyncio.sleep(1) # Small delay between requests
|
||||
|
||||
# Phase 2: Create variant browsers (different configs)
|
||||
print("\n🔶 Phase 2: Testing cold→hot pool promotion")
|
||||
print("-" * 60)
|
||||
viewports = [
|
||||
{"width": 1920, "height": 1080},
|
||||
{"width": 1280, "height": 720},
|
||||
{"width": 800, "height": 600}
|
||||
]
|
||||
|
||||
for idx, viewport in enumerate(viewports):
|
||||
print(f" Viewport {viewport['width']}x{viewport['height']}:")
|
||||
for i in range(4): # 4 requests each to trigger promotion at 3
|
||||
try:
|
||||
r = await client.post(
|
||||
f"{BASE_URL}/crawl",
|
||||
json={
|
||||
"urls": [f"https://httpbin.org/json?v={idx}&r={i}"],
|
||||
"browser_config": {"viewport": viewport},
|
||||
"crawler_config": {}
|
||||
}
|
||||
)
|
||||
print(f" {i+1}/4 ✅ {r.status_code} - Should see cold→hot after 3 uses")
|
||||
except Exception as e:
|
||||
print(f" {i+1}/4 ❌ {e}")
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
# Phase 3: Concurrent burst (stress pool)
|
||||
print("\n🔷 Phase 3: Concurrent burst (10 parallel)")
|
||||
print("-" * 60)
|
||||
tasks = []
|
||||
for i in range(10):
|
||||
tasks.append(
|
||||
client.post(
|
||||
f"{BASE_URL}/crawl",
|
||||
json={"urls": [f"https://httpbin.org/delay/2?burst={i}"], "crawler_config": {}}
|
||||
)
|
||||
)
|
||||
|
||||
print(" Sending 10 concurrent requests...")
|
||||
start = time.time()
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
elapsed = time.time() - start
|
||||
|
||||
successes = sum(1 for r in results if not isinstance(r, Exception) and r.status_code == 200)
|
||||
print(f" ✅ {successes}/10 succeeded in {elapsed:.2f}s")
|
||||
|
||||
# Phase 4: Multi-endpoint coverage
|
||||
print("\n🔶 Phase 4: Testing multiple endpoints")
|
||||
print("-" * 60)
|
||||
endpoints = [
|
||||
("/md", {"url": "https://httpbin.org/html", "f": "fit", "c": "0"}),
|
||||
("/screenshot", {"url": "https://httpbin.org/html"}),
|
||||
("/pdf", {"url": "https://httpbin.org/html"}),
|
||||
]
|
||||
|
||||
for endpoint, payload in endpoints:
|
||||
print(f" Testing {endpoint}...")
|
||||
try:
|
||||
if endpoint == "/md":
|
||||
r = await client.post(f"{BASE_URL}{endpoint}", json=payload)
|
||||
else:
|
||||
r = await client.post(f"{BASE_URL}{endpoint}", json=payload)
|
||||
print(f" ✅ {r.status_code}")
|
||||
except Exception as e:
|
||||
print(f" ❌ {e}")
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Phase 5: Intentional error (to populate errors tab)
|
||||
print("\n🔷 Phase 5: Generating error examples")
|
||||
print("-" * 60)
|
||||
print(" Triggering invalid URL error...")
|
||||
try:
|
||||
r = await client.post(
|
||||
f"{BASE_URL}/crawl",
|
||||
json={"urls": ["invalid://bad-url"], "crawler_config": {}}
|
||||
)
|
||||
print(f" Response: {r.status_code}")
|
||||
except Exception as e:
|
||||
print(f" ✅ Error captured: {type(e).__name__}")
|
||||
|
||||
# Phase 6: Wait for janitor activity
|
||||
print("\n🔶 Phase 6: Waiting for janitor cleanup...")
|
||||
print("-" * 60)
|
||||
print(" Idle for 40s to allow janitor to clean cold pool browsers...")
|
||||
for i in range(40, 0, -10):
|
||||
print(f" {i}s remaining... (Check dashboard for cleanup events)")
|
||||
await asyncio.sleep(10)
|
||||
|
||||
# Phase 7: Final stats check
|
||||
print("\n🔷 Phase 7: Final dashboard state")
|
||||
print("-" * 60)
|
||||
|
||||
r = await client.get(f"{BASE_URL}/monitor/health")
|
||||
health = r.json()
|
||||
print(f" Memory: {health['container']['memory_percent']:.1f}%")
|
||||
print(f" Browsers: Perm={health['pool']['permanent']['active']}, "
|
||||
f"Hot={health['pool']['hot']['count']}, Cold={health['pool']['cold']['count']}")
|
||||
|
||||
r = await client.get(f"{BASE_URL}/monitor/endpoints/stats")
|
||||
stats = r.json()
|
||||
print(f"\n Endpoint Stats:")
|
||||
for endpoint, data in stats.items():
|
||||
print(f" {endpoint}: {data['count']} req, "
|
||||
f"{data['avg_latency_ms']:.0f}ms avg, "
|
||||
f"{data['success_rate_percent']:.1f}% success")
|
||||
|
||||
r = await client.get(f"{BASE_URL}/monitor/browsers")
|
||||
browsers = r.json()
|
||||
print(f"\n Pool Efficiency:")
|
||||
print(f" Total browsers: {browsers['summary']['total_count']}")
|
||||
print(f" Memory usage: {browsers['summary']['total_memory_mb']} MB")
|
||||
print(f" Reuse rate: {browsers['summary']['reuse_rate_percent']:.1f}%")
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("✅ Demo complete! Dashboard is now populated with rich data.")
|
||||
print(f"\n📹 Recording tip: Refresh {BASE_URL}/dashboard")
|
||||
print(" You should see:")
|
||||
print(" • Active & completed requests")
|
||||
print(" • Browser pool (permanent + hot/cold)")
|
||||
print(" • Janitor cleanup events")
|
||||
print(" • Endpoint analytics")
|
||||
print(" • Memory timeline")
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(demo_dashboard())
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n⚠️ Demo interrupted by user")
|
||||
except Exception as e:
|
||||
print(f"\n\n❌ Demo failed: {e}")
|
||||
@@ -0,0 +1,2 @@
|
||||
httpx>=0.25.0
|
||||
docker>=7.0.0
|
||||
Executable
+196
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Security Integration Tests for Crawl4AI Docker API.
|
||||
Tests that security fixes are working correctly against a running server.
|
||||
|
||||
Usage:
|
||||
python run_security_tests.py [base_url]
|
||||
|
||||
Example:
|
||||
python run_security_tests.py http://localhost:11235
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import re
|
||||
|
||||
# Colors for terminal output
|
||||
GREEN = '\033[0;32m'
|
||||
RED = '\033[0;31m'
|
||||
YELLOW = '\033[1;33m'
|
||||
NC = '\033[0m' # No Color
|
||||
|
||||
PASSED = 0
|
||||
FAILED = 0
|
||||
|
||||
|
||||
def run_curl(args: list) -> str:
|
||||
"""Run curl command and return output."""
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['curl', '-s'] + args,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=30
|
||||
)
|
||||
return result.stdout + result.stderr
|
||||
except subprocess.TimeoutExpired:
|
||||
return "TIMEOUT"
|
||||
except Exception as e:
|
||||
return str(e)
|
||||
|
||||
|
||||
def test_expect(name: str, expect_pattern: str, curl_args: list) -> bool:
|
||||
"""Run a test and check if output matches expected pattern."""
|
||||
global PASSED, FAILED
|
||||
|
||||
result = run_curl(curl_args)
|
||||
|
||||
if re.search(expect_pattern, result, re.IGNORECASE):
|
||||
print(f"{GREEN}✓{NC} {name}")
|
||||
PASSED += 1
|
||||
return True
|
||||
else:
|
||||
print(f"{RED}✗{NC} {name}")
|
||||
print(f" Expected pattern: {expect_pattern}")
|
||||
print(f" Got: {result[:200]}")
|
||||
FAILED += 1
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
global PASSED, FAILED
|
||||
|
||||
base_url = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:11235"
|
||||
|
||||
print("=" * 60)
|
||||
print("Crawl4AI Security Integration Tests")
|
||||
print(f"Target: {base_url}")
|
||||
print("=" * 60)
|
||||
print()
|
||||
|
||||
# Check server availability
|
||||
print("Checking server availability...")
|
||||
result = run_curl(['-o', '/dev/null', '-w', '%{http_code}', f'{base_url}/health'])
|
||||
if '200' not in result:
|
||||
print(f"{RED}ERROR: Server not reachable at {base_url}{NC}")
|
||||
print("Please start the server first.")
|
||||
sys.exit(1)
|
||||
print(f"{GREEN}Server is running{NC}")
|
||||
print()
|
||||
|
||||
# === Part A: Security Tests ===
|
||||
print("=== Part A: Security Tests ===")
|
||||
print("(Vulnerabilities must be BLOCKED)")
|
||||
print()
|
||||
|
||||
test_expect(
|
||||
"A1: Hooks disabled by default (403)",
|
||||
r"403|disabled|Hooks are disabled",
|
||||
['-X', 'POST', f'{base_url}/crawl',
|
||||
'-H', 'Content-Type: application/json',
|
||||
'-d', '{"urls":["https://example.com"],"hooks":{"code":{"on_page_context_created":"async def hook(page, context, **kwargs): return page"}}}']
|
||||
)
|
||||
|
||||
test_expect(
|
||||
"A2: file:// blocked on /execute_js (400)",
|
||||
r"400|must start with",
|
||||
['-X', 'POST', f'{base_url}/execute_js',
|
||||
'-H', 'Content-Type: application/json',
|
||||
'-d', '{"url":"file:///etc/passwd","scripts":["1"]}']
|
||||
)
|
||||
|
||||
test_expect(
|
||||
"A3: file:// blocked on /screenshot (400)",
|
||||
r"400|must start with",
|
||||
['-X', 'POST', f'{base_url}/screenshot',
|
||||
'-H', 'Content-Type: application/json',
|
||||
'-d', '{"url":"file:///etc/passwd"}']
|
||||
)
|
||||
|
||||
test_expect(
|
||||
"A4: file:// blocked on /pdf (400)",
|
||||
r"400|must start with",
|
||||
['-X', 'POST', f'{base_url}/pdf',
|
||||
'-H', 'Content-Type: application/json',
|
||||
'-d', '{"url":"file:///etc/passwd"}']
|
||||
)
|
||||
|
||||
test_expect(
|
||||
"A5: file:// blocked on /html (400)",
|
||||
r"400|must start with",
|
||||
['-X', 'POST', f'{base_url}/html',
|
||||
'-H', 'Content-Type: application/json',
|
||||
'-d', '{"url":"file:///etc/passwd"}']
|
||||
)
|
||||
|
||||
print()
|
||||
|
||||
# === Part B: Functionality Tests ===
|
||||
print("=== Part B: Functionality Tests ===")
|
||||
print("(Normal operations must WORK)")
|
||||
print()
|
||||
|
||||
test_expect(
|
||||
"B1: Basic crawl works",
|
||||
r"success.*true|results",
|
||||
['-X', 'POST', f'{base_url}/crawl',
|
||||
'-H', 'Content-Type: application/json',
|
||||
'-d', '{"urls":["https://example.com"]}']
|
||||
)
|
||||
|
||||
test_expect(
|
||||
"B2: /md works with https://",
|
||||
r"success.*true|markdown",
|
||||
['-X', 'POST', f'{base_url}/md',
|
||||
'-H', 'Content-Type: application/json',
|
||||
'-d', '{"url":"https://example.com"}']
|
||||
)
|
||||
|
||||
test_expect(
|
||||
"B3: Health endpoint works",
|
||||
r"ok",
|
||||
[f'{base_url}/health']
|
||||
)
|
||||
|
||||
print()
|
||||
|
||||
# === Part C: Edge Cases ===
|
||||
print("=== Part C: Edge Cases ===")
|
||||
print("(Malformed input must be REJECTED)")
|
||||
print()
|
||||
|
||||
test_expect(
|
||||
"C1: javascript: URL rejected (400)",
|
||||
r"400|must start with",
|
||||
['-X', 'POST', f'{base_url}/execute_js',
|
||||
'-H', 'Content-Type: application/json',
|
||||
'-d', '{"url":"javascript:alert(1)","scripts":["1"]}']
|
||||
)
|
||||
|
||||
test_expect(
|
||||
"C2: data: URL rejected (400)",
|
||||
r"400|must start with",
|
||||
['-X', 'POST', f'{base_url}/execute_js',
|
||||
'-H', 'Content-Type: application/json',
|
||||
'-d', '{"url":"data:text/html,<h1>test</h1>","scripts":["1"]}']
|
||||
)
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("Results")
|
||||
print("=" * 60)
|
||||
print(f"Passed: {GREEN}{PASSED}{NC}")
|
||||
print(f"Failed: {RED}{FAILED}{NC}")
|
||||
print()
|
||||
|
||||
if FAILED > 0:
|
||||
print(f"{RED}SOME TESTS FAILED{NC}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(f"{GREEN}ALL TESTS PASSED{NC}")
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Executable
+138
@@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test 1: Basic Container Health + Single Endpoint
|
||||
- Starts container
|
||||
- Hits /health endpoint 10 times
|
||||
- Reports success rate and basic latency
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
import docker
|
||||
import httpx
|
||||
|
||||
# Config
|
||||
IMAGE = "crawl4ai-local:latest"
|
||||
CONTAINER_NAME = "crawl4ai-test"
|
||||
PORT = 11235
|
||||
REQUESTS = 10
|
||||
|
||||
async def test_endpoint(url: str, count: int):
|
||||
"""Hit endpoint multiple times, return stats."""
|
||||
results = []
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
for i in range(count):
|
||||
start = time.time()
|
||||
try:
|
||||
resp = await client.get(url)
|
||||
elapsed = (time.time() - start) * 1000 # ms
|
||||
results.append({
|
||||
"success": resp.status_code == 200,
|
||||
"latency_ms": elapsed,
|
||||
"status": resp.status_code
|
||||
})
|
||||
print(f" [{i+1}/{count}] ✓ {resp.status_code} - {elapsed:.0f}ms")
|
||||
except Exception as e:
|
||||
results.append({
|
||||
"success": False,
|
||||
"latency_ms": None,
|
||||
"error": str(e)
|
||||
})
|
||||
print(f" [{i+1}/{count}] ✗ Error: {e}")
|
||||
return results
|
||||
|
||||
def start_container(client, image: str, name: str, port: int):
|
||||
"""Start container, return container object."""
|
||||
# Clean up existing
|
||||
try:
|
||||
old = client.containers.get(name)
|
||||
print(f"🧹 Stopping existing container '{name}'...")
|
||||
old.stop()
|
||||
old.remove()
|
||||
except docker.errors.NotFound:
|
||||
pass
|
||||
|
||||
print(f"🚀 Starting container '{name}' from image '{image}'...")
|
||||
container = client.containers.run(
|
||||
image,
|
||||
name=name,
|
||||
ports={f"{port}/tcp": port},
|
||||
detach=True,
|
||||
shm_size="1g",
|
||||
environment={"PYTHON_ENV": "production"}
|
||||
)
|
||||
|
||||
# Wait for health
|
||||
print(f"⏳ Waiting for container to be healthy...")
|
||||
for _ in range(30): # 30s timeout
|
||||
time.sleep(1)
|
||||
container.reload()
|
||||
if container.status == "running":
|
||||
try:
|
||||
# Quick health check
|
||||
import requests
|
||||
resp = requests.get(f"http://localhost:{port}/health", timeout=2)
|
||||
if resp.status_code == 200:
|
||||
print(f"✅ Container healthy!")
|
||||
return container
|
||||
except:
|
||||
pass
|
||||
raise TimeoutError("Container failed to start")
|
||||
|
||||
def stop_container(container):
|
||||
"""Stop and remove container."""
|
||||
print(f"🛑 Stopping container...")
|
||||
container.stop()
|
||||
container.remove()
|
||||
print(f"✅ Container removed")
|
||||
|
||||
async def main():
|
||||
print("="*60)
|
||||
print("TEST 1: Basic Container Health + Single Endpoint")
|
||||
print("="*60)
|
||||
|
||||
client = docker.from_env()
|
||||
container = None
|
||||
|
||||
try:
|
||||
# Start container
|
||||
container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
|
||||
|
||||
# Test /health endpoint
|
||||
print(f"\n📊 Testing /health endpoint ({REQUESTS} requests)...")
|
||||
url = f"http://localhost:{PORT}/health"
|
||||
results = await test_endpoint(url, REQUESTS)
|
||||
|
||||
# Calculate stats
|
||||
successes = sum(1 for r in results if r["success"])
|
||||
success_rate = (successes / len(results)) * 100
|
||||
latencies = [r["latency_ms"] for r in results if r["latency_ms"] is not None]
|
||||
avg_latency = sum(latencies) / len(latencies) if latencies else 0
|
||||
|
||||
# Print results
|
||||
print(f"\n{'='*60}")
|
||||
print(f"RESULTS:")
|
||||
print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(results)})")
|
||||
print(f" Avg Latency: {avg_latency:.0f}ms")
|
||||
if latencies:
|
||||
print(f" Min Latency: {min(latencies):.0f}ms")
|
||||
print(f" Max Latency: {max(latencies):.0f}ms")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# Pass/Fail
|
||||
if success_rate >= 100:
|
||||
print(f"✅ TEST PASSED")
|
||||
return 0
|
||||
else:
|
||||
print(f"❌ TEST FAILED (expected 100% success rate)")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ TEST ERROR: {e}")
|
||||
return 1
|
||||
finally:
|
||||
if container:
|
||||
stop_container(container)
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(main())
|
||||
exit(exit_code)
|
||||
Executable
+205
@@ -0,0 +1,205 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test 2: Docker Stats Monitoring
|
||||
- Extends Test 1 with real-time container stats
|
||||
- Monitors memory % and CPU during requests
|
||||
- Reports baseline, peak, and final memory
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
import docker
|
||||
import httpx
|
||||
from threading import Thread, Event
|
||||
|
||||
# Config
|
||||
IMAGE = "crawl4ai-local:latest"
|
||||
CONTAINER_NAME = "crawl4ai-test"
|
||||
PORT = 11235
|
||||
REQUESTS = 20 # More requests to see memory usage
|
||||
|
||||
# Stats tracking
|
||||
stats_history = []
|
||||
stop_monitoring = Event()
|
||||
|
||||
def monitor_stats(container):
|
||||
"""Background thread to collect container stats."""
|
||||
for stat in container.stats(decode=True, stream=True):
|
||||
if stop_monitoring.is_set():
|
||||
break
|
||||
|
||||
try:
|
||||
# Extract memory stats
|
||||
mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024) # MB
|
||||
mem_limit = stat['memory_stats'].get('limit', 1) / (1024 * 1024)
|
||||
mem_percent = (mem_usage / mem_limit * 100) if mem_limit > 0 else 0
|
||||
|
||||
# Extract CPU stats (handle missing fields on Mac)
|
||||
cpu_percent = 0
|
||||
try:
|
||||
cpu_delta = stat['cpu_stats']['cpu_usage']['total_usage'] - \
|
||||
stat['precpu_stats']['cpu_usage']['total_usage']
|
||||
system_delta = stat['cpu_stats'].get('system_cpu_usage', 0) - \
|
||||
stat['precpu_stats'].get('system_cpu_usage', 0)
|
||||
if system_delta > 0:
|
||||
num_cpus = stat['cpu_stats'].get('online_cpus', 1)
|
||||
cpu_percent = (cpu_delta / system_delta * num_cpus * 100.0)
|
||||
except (KeyError, ZeroDivisionError):
|
||||
pass
|
||||
|
||||
stats_history.append({
|
||||
'timestamp': time.time(),
|
||||
'memory_mb': mem_usage,
|
||||
'memory_percent': mem_percent,
|
||||
'cpu_percent': cpu_percent
|
||||
})
|
||||
except Exception as e:
|
||||
# Skip malformed stats
|
||||
pass
|
||||
|
||||
time.sleep(0.5) # Sample every 500ms
|
||||
|
||||
async def test_endpoint(url: str, count: int):
|
||||
"""Hit endpoint, return stats."""
|
||||
results = []
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
for i in range(count):
|
||||
start = time.time()
|
||||
try:
|
||||
resp = await client.get(url)
|
||||
elapsed = (time.time() - start) * 1000
|
||||
results.append({
|
||||
"success": resp.status_code == 200,
|
||||
"latency_ms": elapsed,
|
||||
})
|
||||
if (i + 1) % 5 == 0: # Print every 5 requests
|
||||
print(f" [{i+1}/{count}] ✓ {resp.status_code} - {elapsed:.0f}ms")
|
||||
except Exception as e:
|
||||
results.append({"success": False, "error": str(e)})
|
||||
print(f" [{i+1}/{count}] ✗ Error: {e}")
|
||||
return results
|
||||
|
||||
def start_container(client, image: str, name: str, port: int):
|
||||
"""Start container."""
|
||||
try:
|
||||
old = client.containers.get(name)
|
||||
print(f"🧹 Stopping existing container '{name}'...")
|
||||
old.stop()
|
||||
old.remove()
|
||||
except docker.errors.NotFound:
|
||||
pass
|
||||
|
||||
print(f"🚀 Starting container '{name}'...")
|
||||
container = client.containers.run(
|
||||
image,
|
||||
name=name,
|
||||
ports={f"{port}/tcp": port},
|
||||
detach=True,
|
||||
shm_size="1g",
|
||||
mem_limit="4g", # Set explicit memory limit
|
||||
)
|
||||
|
||||
print(f"⏳ Waiting for health...")
|
||||
for _ in range(30):
|
||||
time.sleep(1)
|
||||
container.reload()
|
||||
if container.status == "running":
|
||||
try:
|
||||
import requests
|
||||
resp = requests.get(f"http://localhost:{port}/health", timeout=2)
|
||||
if resp.status_code == 200:
|
||||
print(f"✅ Container healthy!")
|
||||
return container
|
||||
except:
|
||||
pass
|
||||
raise TimeoutError("Container failed to start")
|
||||
|
||||
def stop_container(container):
|
||||
"""Stop container."""
|
||||
print(f"🛑 Stopping container...")
|
||||
container.stop()
|
||||
container.remove()
|
||||
|
||||
async def main():
|
||||
print("="*60)
|
||||
print("TEST 2: Docker Stats Monitoring")
|
||||
print("="*60)
|
||||
|
||||
client = docker.from_env()
|
||||
container = None
|
||||
monitor_thread = None
|
||||
|
||||
try:
|
||||
# Start container
|
||||
container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
|
||||
|
||||
# Start stats monitoring in background
|
||||
print(f"\n📊 Starting stats monitor...")
|
||||
stop_monitoring.clear()
|
||||
stats_history.clear()
|
||||
monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
|
||||
monitor_thread.start()
|
||||
|
||||
# Wait a bit for baseline
|
||||
await asyncio.sleep(2)
|
||||
baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
|
||||
print(f"📏 Baseline memory: {baseline_mem:.1f} MB")
|
||||
|
||||
# Test /health endpoint
|
||||
print(f"\n🔄 Running {REQUESTS} requests to /health...")
|
||||
url = f"http://localhost:{PORT}/health"
|
||||
results = await test_endpoint(url, REQUESTS)
|
||||
|
||||
# Wait a bit to capture peak
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Stop monitoring
|
||||
stop_monitoring.set()
|
||||
if monitor_thread:
|
||||
monitor_thread.join(timeout=2)
|
||||
|
||||
# Calculate stats
|
||||
successes = sum(1 for r in results if r.get("success"))
|
||||
success_rate = (successes / len(results)) * 100
|
||||
latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
|
||||
avg_latency = sum(latencies) / len(latencies) if latencies else 0
|
||||
|
||||
# Memory stats
|
||||
memory_samples = [s['memory_mb'] for s in stats_history]
|
||||
peak_mem = max(memory_samples) if memory_samples else 0
|
||||
final_mem = memory_samples[-1] if memory_samples else 0
|
||||
mem_delta = final_mem - baseline_mem
|
||||
|
||||
# Print results
|
||||
print(f"\n{'='*60}")
|
||||
print(f"RESULTS:")
|
||||
print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(results)})")
|
||||
print(f" Avg Latency: {avg_latency:.0f}ms")
|
||||
print(f"\n Memory Stats:")
|
||||
print(f" Baseline: {baseline_mem:.1f} MB")
|
||||
print(f" Peak: {peak_mem:.1f} MB")
|
||||
print(f" Final: {final_mem:.1f} MB")
|
||||
print(f" Delta: {mem_delta:+.1f} MB")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# Pass/Fail
|
||||
if success_rate >= 100 and mem_delta < 100: # No significant memory growth
|
||||
print(f"✅ TEST PASSED")
|
||||
return 0
|
||||
else:
|
||||
if success_rate < 100:
|
||||
print(f"❌ TEST FAILED (success rate < 100%)")
|
||||
if mem_delta >= 100:
|
||||
print(f"⚠️ WARNING: Memory grew by {mem_delta:.1f} MB")
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ TEST ERROR: {e}")
|
||||
return 1
|
||||
finally:
|
||||
stop_monitoring.set()
|
||||
if container:
|
||||
stop_container(container)
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(main())
|
||||
exit(exit_code)
|
||||
Executable
+229
@@ -0,0 +1,229 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test 3: Pool Validation - Permanent Browser Reuse
|
||||
- Tests /html endpoint (should use permanent browser)
|
||||
- Monitors container logs for pool hit markers
|
||||
- Validates browser reuse rate
|
||||
- Checks memory after browser creation
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
import docker
|
||||
import httpx
|
||||
from threading import Thread, Event
|
||||
|
||||
# Config
|
||||
IMAGE = "crawl4ai-local:latest"
|
||||
CONTAINER_NAME = "crawl4ai-test"
|
||||
PORT = 11235
|
||||
REQUESTS = 30
|
||||
|
||||
# Stats tracking
|
||||
stats_history = []
|
||||
stop_monitoring = Event()
|
||||
|
||||
def monitor_stats(container):
|
||||
"""Background stats collector."""
|
||||
for stat in container.stats(decode=True, stream=True):
|
||||
if stop_monitoring.is_set():
|
||||
break
|
||||
try:
|
||||
mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
|
||||
stats_history.append({
|
||||
'timestamp': time.time(),
|
||||
'memory_mb': mem_usage,
|
||||
})
|
||||
except:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
|
||||
def count_log_markers(container):
|
||||
"""Extract pool usage markers from logs."""
|
||||
logs = container.logs().decode('utf-8')
|
||||
|
||||
permanent_hits = logs.count("🔥 Using permanent browser")
|
||||
hot_hits = logs.count("♨️ Using hot pool browser")
|
||||
cold_hits = logs.count("❄️ Using cold pool browser")
|
||||
new_created = logs.count("🆕 Creating new browser")
|
||||
|
||||
return {
|
||||
'permanent_hits': permanent_hits,
|
||||
'hot_hits': hot_hits,
|
||||
'cold_hits': cold_hits,
|
||||
'new_created': new_created,
|
||||
'total_hits': permanent_hits + hot_hits + cold_hits
|
||||
}
|
||||
|
||||
async def test_endpoint(url: str, count: int):
|
||||
"""Hit endpoint multiple times."""
|
||||
results = []
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
for i in range(count):
|
||||
start = time.time()
|
||||
try:
|
||||
resp = await client.post(url, json={"url": "https://httpbin.org/html"})
|
||||
elapsed = (time.time() - start) * 1000
|
||||
results.append({
|
||||
"success": resp.status_code == 200,
|
||||
"latency_ms": elapsed,
|
||||
})
|
||||
if (i + 1) % 10 == 0:
|
||||
print(f" [{i+1}/{count}] ✓ {resp.status_code} - {elapsed:.0f}ms")
|
||||
except Exception as e:
|
||||
results.append({"success": False, "error": str(e)})
|
||||
print(f" [{i+1}/{count}] ✗ Error: {e}")
|
||||
return results
|
||||
|
||||
def start_container(client, image: str, name: str, port: int):
|
||||
"""Start container."""
|
||||
try:
|
||||
old = client.containers.get(name)
|
||||
print(f"🧹 Stopping existing container...")
|
||||
old.stop()
|
||||
old.remove()
|
||||
except docker.errors.NotFound:
|
||||
pass
|
||||
|
||||
print(f"🚀 Starting container...")
|
||||
container = client.containers.run(
|
||||
image,
|
||||
name=name,
|
||||
ports={f"{port}/tcp": port},
|
||||
detach=True,
|
||||
shm_size="1g",
|
||||
mem_limit="4g",
|
||||
)
|
||||
|
||||
print(f"⏳ Waiting for health...")
|
||||
for _ in range(30):
|
||||
time.sleep(1)
|
||||
container.reload()
|
||||
if container.status == "running":
|
||||
try:
|
||||
import requests
|
||||
resp = requests.get(f"http://localhost:{port}/health", timeout=2)
|
||||
if resp.status_code == 200:
|
||||
print(f"✅ Container healthy!")
|
||||
return container
|
||||
except:
|
||||
pass
|
||||
raise TimeoutError("Container failed to start")
|
||||
|
||||
def stop_container(container):
|
||||
"""Stop container."""
|
||||
print(f"🛑 Stopping container...")
|
||||
container.stop()
|
||||
container.remove()
|
||||
|
||||
async def main():
|
||||
print("="*60)
|
||||
print("TEST 3: Pool Validation - Permanent Browser Reuse")
|
||||
print("="*60)
|
||||
|
||||
client = docker.from_env()
|
||||
container = None
|
||||
monitor_thread = None
|
||||
|
||||
try:
|
||||
# Start container
|
||||
container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
|
||||
|
||||
# Wait for permanent browser initialization
|
||||
print(f"\n⏳ Waiting for permanent browser init (3s)...")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Start stats monitoring
|
||||
print(f"📊 Starting stats monitor...")
|
||||
stop_monitoring.clear()
|
||||
stats_history.clear()
|
||||
monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
|
||||
monitor_thread.start()
|
||||
|
||||
await asyncio.sleep(1)
|
||||
baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
|
||||
print(f"📏 Baseline (with permanent browser): {baseline_mem:.1f} MB")
|
||||
|
||||
# Test /html endpoint (uses permanent browser for default config)
|
||||
print(f"\n🔄 Running {REQUESTS} requests to /html...")
|
||||
url = f"http://localhost:{PORT}/html"
|
||||
results = await test_endpoint(url, REQUESTS)
|
||||
|
||||
# Wait a bit
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Stop monitoring
|
||||
stop_monitoring.set()
|
||||
if monitor_thread:
|
||||
monitor_thread.join(timeout=2)
|
||||
|
||||
# Analyze logs for pool markers
|
||||
print(f"\n📋 Analyzing pool usage...")
|
||||
pool_stats = count_log_markers(container)
|
||||
|
||||
# Calculate request stats
|
||||
successes = sum(1 for r in results if r.get("success"))
|
||||
success_rate = (successes / len(results)) * 100
|
||||
latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
|
||||
avg_latency = sum(latencies) / len(latencies) if latencies else 0
|
||||
|
||||
# Memory stats
|
||||
memory_samples = [s['memory_mb'] for s in stats_history]
|
||||
peak_mem = max(memory_samples) if memory_samples else 0
|
||||
final_mem = memory_samples[-1] if memory_samples else 0
|
||||
mem_delta = final_mem - baseline_mem
|
||||
|
||||
# Calculate reuse rate
|
||||
total_requests = len(results)
|
||||
total_pool_hits = pool_stats['total_hits']
|
||||
reuse_rate = (total_pool_hits / total_requests * 100) if total_requests > 0 else 0
|
||||
|
||||
# Print results
|
||||
print(f"\n{'='*60}")
|
||||
print(f"RESULTS:")
|
||||
print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(results)})")
|
||||
print(f" Avg Latency: {avg_latency:.0f}ms")
|
||||
print(f"\n Pool Stats:")
|
||||
print(f" 🔥 Permanent Hits: {pool_stats['permanent_hits']}")
|
||||
print(f" ♨️ Hot Pool Hits: {pool_stats['hot_hits']}")
|
||||
print(f" ❄️ Cold Pool Hits: {pool_stats['cold_hits']}")
|
||||
print(f" 🆕 New Created: {pool_stats['new_created']}")
|
||||
print(f" 📊 Reuse Rate: {reuse_rate:.1f}%")
|
||||
print(f"\n Memory Stats:")
|
||||
print(f" Baseline: {baseline_mem:.1f} MB")
|
||||
print(f" Peak: {peak_mem:.1f} MB")
|
||||
print(f" Final: {final_mem:.1f} MB")
|
||||
print(f" Delta: {mem_delta:+.1f} MB")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# Pass/Fail
|
||||
passed = True
|
||||
if success_rate < 100:
|
||||
print(f"❌ FAIL: Success rate {success_rate:.1f}% < 100%")
|
||||
passed = False
|
||||
if reuse_rate < 80:
|
||||
print(f"❌ FAIL: Reuse rate {reuse_rate:.1f}% < 80% (expected high permanent browser usage)")
|
||||
passed = False
|
||||
if pool_stats['permanent_hits'] < (total_requests * 0.8):
|
||||
print(f"⚠️ WARNING: Only {pool_stats['permanent_hits']} permanent hits out of {total_requests} requests")
|
||||
if mem_delta > 200:
|
||||
print(f"⚠️ WARNING: Memory grew by {mem_delta:.1f} MB (possible browser leak)")
|
||||
|
||||
if passed:
|
||||
print(f"✅ TEST PASSED")
|
||||
return 0
|
||||
else:
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ TEST ERROR: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
finally:
|
||||
stop_monitoring.set()
|
||||
if container:
|
||||
stop_container(container)
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(main())
|
||||
exit(exit_code)
|
||||
Executable
+236
@@ -0,0 +1,236 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test 4: Concurrent Load Testing
|
||||
- Tests pool under concurrent load
|
||||
- Escalates: 10 → 50 → 100 concurrent requests
|
||||
- Validates latency distribution (P50, P95, P99)
|
||||
- Monitors memory stability
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
import docker
|
||||
import httpx
|
||||
from threading import Thread, Event
|
||||
from collections import defaultdict
|
||||
|
||||
# Config
|
||||
IMAGE = "crawl4ai-local:latest"
|
||||
CONTAINER_NAME = "crawl4ai-test"
|
||||
PORT = 11235
|
||||
LOAD_LEVELS = [
|
||||
{"name": "Light", "concurrent": 10, "requests": 20},
|
||||
{"name": "Medium", "concurrent": 50, "requests": 100},
|
||||
{"name": "Heavy", "concurrent": 100, "requests": 200},
|
||||
]
|
||||
|
||||
# Stats
|
||||
stats_history = []
|
||||
stop_monitoring = Event()
|
||||
|
||||
def monitor_stats(container):
|
||||
"""Background stats collector."""
|
||||
for stat in container.stats(decode=True, stream=True):
|
||||
if stop_monitoring.is_set():
|
||||
break
|
||||
try:
|
||||
mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
|
||||
stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
|
||||
except:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
|
||||
def count_log_markers(container):
|
||||
"""Extract pool markers."""
|
||||
logs = container.logs().decode('utf-8')
|
||||
return {
|
||||
'permanent': logs.count("🔥 Using permanent browser"),
|
||||
'hot': logs.count("♨️ Using hot pool browser"),
|
||||
'cold': logs.count("❄️ Using cold pool browser"),
|
||||
'new': logs.count("🆕 Creating new browser"),
|
||||
}
|
||||
|
||||
async def hit_endpoint(client, url, payload, semaphore):
|
||||
"""Single request with concurrency control."""
|
||||
async with semaphore:
|
||||
start = time.time()
|
||||
try:
|
||||
resp = await client.post(url, json=payload, timeout=60.0)
|
||||
elapsed = (time.time() - start) * 1000
|
||||
return {"success": resp.status_code == 200, "latency_ms": elapsed}
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e)}
|
||||
|
||||
async def run_concurrent_test(url, payload, concurrent, total_requests):
|
||||
"""Run concurrent requests."""
|
||||
semaphore = asyncio.Semaphore(concurrent)
|
||||
async with httpx.AsyncClient() as client:
|
||||
tasks = [hit_endpoint(client, url, payload, semaphore) for _ in range(total_requests)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
return results
|
||||
|
||||
def calculate_percentiles(latencies):
|
||||
"""Calculate P50, P95, P99."""
|
||||
if not latencies:
|
||||
return 0, 0, 0
|
||||
sorted_lat = sorted(latencies)
|
||||
n = len(sorted_lat)
|
||||
return (
|
||||
sorted_lat[int(n * 0.50)],
|
||||
sorted_lat[int(n * 0.95)],
|
||||
sorted_lat[int(n * 0.99)],
|
||||
)
|
||||
|
||||
def start_container(client, image, name, port):
|
||||
"""Start container."""
|
||||
try:
|
||||
old = client.containers.get(name)
|
||||
print(f"🧹 Stopping existing container...")
|
||||
old.stop()
|
||||
old.remove()
|
||||
except docker.errors.NotFound:
|
||||
pass
|
||||
|
||||
print(f"🚀 Starting container...")
|
||||
container = client.containers.run(
|
||||
image, name=name, ports={f"{port}/tcp": port},
|
||||
detach=True, shm_size="1g", mem_limit="4g",
|
||||
)
|
||||
|
||||
print(f"⏳ Waiting for health...")
|
||||
for _ in range(30):
|
||||
time.sleep(1)
|
||||
container.reload()
|
||||
if container.status == "running":
|
||||
try:
|
||||
import requests
|
||||
if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
|
||||
print(f"✅ Container healthy!")
|
||||
return container
|
||||
except:
|
||||
pass
|
||||
raise TimeoutError("Container failed to start")
|
||||
|
||||
async def main():
|
||||
print("="*60)
|
||||
print("TEST 4: Concurrent Load Testing")
|
||||
print("="*60)
|
||||
|
||||
client = docker.from_env()
|
||||
container = None
|
||||
monitor_thread = None
|
||||
|
||||
try:
|
||||
container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
|
||||
|
||||
print(f"\n⏳ Waiting for permanent browser init (3s)...")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Start monitoring
|
||||
stop_monitoring.clear()
|
||||
stats_history.clear()
|
||||
monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
|
||||
monitor_thread.start()
|
||||
|
||||
await asyncio.sleep(1)
|
||||
baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
|
||||
print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
|
||||
|
||||
url = f"http://localhost:{PORT}/html"
|
||||
payload = {"url": "https://httpbin.org/html"}
|
||||
|
||||
all_results = []
|
||||
level_stats = []
|
||||
|
||||
# Run load levels
|
||||
for level in LOAD_LEVELS:
|
||||
print(f"{'='*60}")
|
||||
print(f"🔄 {level['name']} Load: {level['concurrent']} concurrent, {level['requests']} total")
|
||||
print(f"{'='*60}")
|
||||
|
||||
start_time = time.time()
|
||||
results = await run_concurrent_test(url, payload, level['concurrent'], level['requests'])
|
||||
duration = time.time() - start_time
|
||||
|
||||
successes = sum(1 for r in results if r.get("success"))
|
||||
success_rate = (successes / len(results)) * 100
|
||||
latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
|
||||
p50, p95, p99 = calculate_percentiles(latencies)
|
||||
avg_lat = sum(latencies) / len(latencies) if latencies else 0
|
||||
|
||||
print(f" Duration: {duration:.1f}s")
|
||||
print(f" Success: {success_rate:.1f}% ({successes}/{len(results)})")
|
||||
print(f" Avg Latency: {avg_lat:.0f}ms")
|
||||
print(f" P50/P95/P99: {p50:.0f}ms / {p95:.0f}ms / {p99:.0f}ms")
|
||||
|
||||
level_stats.append({
|
||||
'name': level['name'],
|
||||
'concurrent': level['concurrent'],
|
||||
'success_rate': success_rate,
|
||||
'avg_latency': avg_lat,
|
||||
'p50': p50, 'p95': p95, 'p99': p99,
|
||||
})
|
||||
all_results.extend(results)
|
||||
|
||||
await asyncio.sleep(2) # Cool down between levels
|
||||
|
||||
# Stop monitoring
|
||||
await asyncio.sleep(1)
|
||||
stop_monitoring.set()
|
||||
if monitor_thread:
|
||||
monitor_thread.join(timeout=2)
|
||||
|
||||
# Final stats
|
||||
pool_stats = count_log_markers(container)
|
||||
memory_samples = [s['memory_mb'] for s in stats_history]
|
||||
peak_mem = max(memory_samples) if memory_samples else 0
|
||||
final_mem = memory_samples[-1] if memory_samples else 0
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"FINAL RESULTS:")
|
||||
print(f"{'='*60}")
|
||||
print(f" Total Requests: {len(all_results)}")
|
||||
print(f"\n Pool Utilization:")
|
||||
print(f" 🔥 Permanent: {pool_stats['permanent']}")
|
||||
print(f" ♨️ Hot: {pool_stats['hot']}")
|
||||
print(f" ❄️ Cold: {pool_stats['cold']}")
|
||||
print(f" 🆕 New: {pool_stats['new']}")
|
||||
print(f"\n Memory:")
|
||||
print(f" Baseline: {baseline_mem:.1f} MB")
|
||||
print(f" Peak: {peak_mem:.1f} MB")
|
||||
print(f" Final: {final_mem:.1f} MB")
|
||||
print(f" Delta: {final_mem - baseline_mem:+.1f} MB")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# Pass/Fail
|
||||
passed = True
|
||||
for ls in level_stats:
|
||||
if ls['success_rate'] < 99:
|
||||
print(f"❌ FAIL: {ls['name']} success rate {ls['success_rate']:.1f}% < 99%")
|
||||
passed = False
|
||||
if ls['p99'] > 10000: # 10s threshold
|
||||
print(f"⚠️ WARNING: {ls['name']} P99 latency {ls['p99']:.0f}ms very high")
|
||||
|
||||
if final_mem - baseline_mem > 300:
|
||||
print(f"⚠️ WARNING: Memory grew {final_mem - baseline_mem:.1f} MB")
|
||||
|
||||
if passed:
|
||||
print(f"✅ TEST PASSED")
|
||||
return 0
|
||||
else:
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ TEST ERROR: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
finally:
|
||||
stop_monitoring.set()
|
||||
if container:
|
||||
print(f"🛑 Stopping container...")
|
||||
container.stop()
|
||||
container.remove()
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(main())
|
||||
exit(exit_code)
|
||||
Executable
+267
@@ -0,0 +1,267 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test 5: Pool Stress - Mixed Configs
|
||||
- Tests hot/cold pool with different browser configs
|
||||
- Uses different viewports to create config variants
|
||||
- Validates cold → hot promotion after 3 uses
|
||||
- Monitors pool tier distribution
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
import docker
|
||||
import httpx
|
||||
from threading import Thread, Event
|
||||
import random
|
||||
|
||||
# Config
|
||||
IMAGE = "crawl4ai-local:latest"
|
||||
CONTAINER_NAME = "crawl4ai-test"
|
||||
PORT = 11235
|
||||
REQUESTS_PER_CONFIG = 5 # 5 requests per config variant
|
||||
|
||||
# Different viewport configs to test pool tiers
|
||||
VIEWPORT_CONFIGS = [
|
||||
None, # Default (permanent browser)
|
||||
{"width": 1920, "height": 1080}, # Desktop
|
||||
{"width": 1024, "height": 768}, # Tablet
|
||||
{"width": 375, "height": 667}, # Mobile
|
||||
]
|
||||
|
||||
# Stats
|
||||
stats_history = []
|
||||
stop_monitoring = Event()
|
||||
|
||||
def monitor_stats(container):
|
||||
"""Background stats collector."""
|
||||
for stat in container.stats(decode=True, stream=True):
|
||||
if stop_monitoring.is_set():
|
||||
break
|
||||
try:
|
||||
mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
|
||||
stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
|
||||
except:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
|
||||
def analyze_pool_logs(container):
|
||||
"""Extract detailed pool stats from logs."""
|
||||
logs = container.logs().decode('utf-8')
|
||||
|
||||
permanent = logs.count("🔥 Using permanent browser")
|
||||
hot = logs.count("♨️ Using hot pool browser")
|
||||
cold = logs.count("❄️ Using cold pool browser")
|
||||
new = logs.count("🆕 Creating new browser")
|
||||
promotions = logs.count("⬆️ Promoting to hot pool")
|
||||
|
||||
return {
|
||||
'permanent': permanent,
|
||||
'hot': hot,
|
||||
'cold': cold,
|
||||
'new': new,
|
||||
'promotions': promotions,
|
||||
'total': permanent + hot + cold
|
||||
}
|
||||
|
||||
async def crawl_with_viewport(client, url, viewport):
|
||||
"""Single request with specific viewport."""
|
||||
payload = {
|
||||
"urls": ["https://httpbin.org/html"],
|
||||
"browser_config": {},
|
||||
"crawler_config": {}
|
||||
}
|
||||
|
||||
# Add viewport if specified
|
||||
if viewport:
|
||||
payload["browser_config"] = {
|
||||
"type": "BrowserConfig",
|
||||
"params": {
|
||||
"viewport": {"type": "dict", "value": viewport},
|
||||
"headless": True,
|
||||
"text_mode": True,
|
||||
"extra_args": [
|
||||
"--no-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-gpu",
|
||||
"--disable-software-rasterizer",
|
||||
"--disable-web-security",
|
||||
"--allow-insecure-localhost",
|
||||
"--ignore-certificate-errors"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
start = time.time()
|
||||
try:
|
||||
resp = await client.post(url, json=payload, timeout=60.0)
|
||||
elapsed = (time.time() - start) * 1000
|
||||
return {"success": resp.status_code == 200, "latency_ms": elapsed, "viewport": viewport}
|
||||
except Exception as e:
|
||||
return {"success": False, "error": str(e), "viewport": viewport}
|
||||
|
||||
def start_container(client, image, name, port):
|
||||
"""Start container."""
|
||||
try:
|
||||
old = client.containers.get(name)
|
||||
print(f"🧹 Stopping existing container...")
|
||||
old.stop()
|
||||
old.remove()
|
||||
except docker.errors.NotFound:
|
||||
pass
|
||||
|
||||
print(f"🚀 Starting container...")
|
||||
container = client.containers.run(
|
||||
image, name=name, ports={f"{port}/tcp": port},
|
||||
detach=True, shm_size="1g", mem_limit="4g",
|
||||
)
|
||||
|
||||
print(f"⏳ Waiting for health...")
|
||||
for _ in range(30):
|
||||
time.sleep(1)
|
||||
container.reload()
|
||||
if container.status == "running":
|
||||
try:
|
||||
import requests
|
||||
if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
|
||||
print(f"✅ Container healthy!")
|
||||
return container
|
||||
except:
|
||||
pass
|
||||
raise TimeoutError("Container failed to start")
|
||||
|
||||
async def main():
|
||||
print("="*60)
|
||||
print("TEST 5: Pool Stress - Mixed Configs")
|
||||
print("="*60)
|
||||
|
||||
client = docker.from_env()
|
||||
container = None
|
||||
monitor_thread = None
|
||||
|
||||
try:
|
||||
container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
|
||||
|
||||
print(f"\n⏳ Waiting for permanent browser init (3s)...")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Start monitoring
|
||||
stop_monitoring.clear()
|
||||
stats_history.clear()
|
||||
monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
|
||||
monitor_thread.start()
|
||||
|
||||
await asyncio.sleep(1)
|
||||
baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
|
||||
print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
|
||||
|
||||
url = f"http://localhost:{PORT}/crawl"
|
||||
|
||||
print(f"Testing {len(VIEWPORT_CONFIGS)} different configs:")
|
||||
for i, vp in enumerate(VIEWPORT_CONFIGS):
|
||||
vp_str = "Default" if vp is None else f"{vp['width']}x{vp['height']}"
|
||||
print(f" {i+1}. {vp_str}")
|
||||
print()
|
||||
|
||||
# Run requests: repeat each config REQUESTS_PER_CONFIG times
|
||||
all_results = []
|
||||
config_sequence = []
|
||||
|
||||
for _ in range(REQUESTS_PER_CONFIG):
|
||||
for viewport in VIEWPORT_CONFIGS:
|
||||
config_sequence.append(viewport)
|
||||
|
||||
# Shuffle to mix configs
|
||||
random.shuffle(config_sequence)
|
||||
|
||||
print(f"🔄 Running {len(config_sequence)} requests with mixed configs...")
|
||||
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
for i, viewport in enumerate(config_sequence):
|
||||
result = await crawl_with_viewport(http_client, url, viewport)
|
||||
all_results.append(result)
|
||||
|
||||
if (i + 1) % 5 == 0:
|
||||
vp_str = "default" if result['viewport'] is None else f"{result['viewport']['width']}x{result['viewport']['height']}"
|
||||
status = "✓" if result.get('success') else "✗"
|
||||
lat = f"{result.get('latency_ms', 0):.0f}ms" if 'latency_ms' in result else "error"
|
||||
print(f" [{i+1}/{len(config_sequence)}] {status} {vp_str} - {lat}")
|
||||
|
||||
# Stop monitoring
|
||||
await asyncio.sleep(2)
|
||||
stop_monitoring.set()
|
||||
if monitor_thread:
|
||||
monitor_thread.join(timeout=2)
|
||||
|
||||
# Analyze results
|
||||
pool_stats = analyze_pool_logs(container)
|
||||
|
||||
successes = sum(1 for r in all_results if r.get("success"))
|
||||
success_rate = (successes / len(all_results)) * 100
|
||||
latencies = [r["latency_ms"] for r in all_results if "latency_ms" in r]
|
||||
avg_lat = sum(latencies) / len(latencies) if latencies else 0
|
||||
|
||||
memory_samples = [s['memory_mb'] for s in stats_history]
|
||||
peak_mem = max(memory_samples) if memory_samples else 0
|
||||
final_mem = memory_samples[-1] if memory_samples else 0
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"RESULTS:")
|
||||
print(f"{'='*60}")
|
||||
print(f" Requests: {len(all_results)}")
|
||||
print(f" Success Rate: {success_rate:.1f}% ({successes}/{len(all_results)})")
|
||||
print(f" Avg Latency: {avg_lat:.0f}ms")
|
||||
print(f"\n Pool Statistics:")
|
||||
print(f" 🔥 Permanent: {pool_stats['permanent']}")
|
||||
print(f" ♨️ Hot: {pool_stats['hot']}")
|
||||
print(f" ❄️ Cold: {pool_stats['cold']}")
|
||||
print(f" 🆕 New: {pool_stats['new']}")
|
||||
print(f" ⬆️ Promotions: {pool_stats['promotions']}")
|
||||
print(f" 📊 Reuse: {(pool_stats['total'] / len(all_results) * 100):.1f}%")
|
||||
print(f"\n Memory:")
|
||||
print(f" Baseline: {baseline_mem:.1f} MB")
|
||||
print(f" Peak: {peak_mem:.1f} MB")
|
||||
print(f" Final: {final_mem:.1f} MB")
|
||||
print(f" Delta: {final_mem - baseline_mem:+.1f} MB")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# Pass/Fail
|
||||
passed = True
|
||||
|
||||
if success_rate < 99:
|
||||
print(f"❌ FAIL: Success rate {success_rate:.1f}% < 99%")
|
||||
passed = False
|
||||
|
||||
# Should see promotions since we repeat each config 5 times
|
||||
if pool_stats['promotions'] < (len(VIEWPORT_CONFIGS) - 1): # -1 for default
|
||||
print(f"⚠️ WARNING: Only {pool_stats['promotions']} promotions (expected ~{len(VIEWPORT_CONFIGS)-1})")
|
||||
|
||||
# Should have created some browsers for different configs
|
||||
if pool_stats['new'] == 0:
|
||||
print(f"⚠️ NOTE: No new browsers created (all used default?)")
|
||||
|
||||
if pool_stats['permanent'] == len(all_results):
|
||||
print(f"⚠️ NOTE: All requests used permanent browser (configs not varying enough?)")
|
||||
|
||||
if final_mem - baseline_mem > 500:
|
||||
print(f"⚠️ WARNING: Memory grew {final_mem - baseline_mem:.1f} MB")
|
||||
|
||||
if passed:
|
||||
print(f"✅ TEST PASSED")
|
||||
return 0
|
||||
else:
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ TEST ERROR: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
finally:
|
||||
stop_monitoring.set()
|
||||
if container:
|
||||
print(f"🛑 Stopping container...")
|
||||
container.stop()
|
||||
container.remove()
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(main())
|
||||
exit(exit_code)
|
||||
Executable
+234
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test 6: Multi-Endpoint Testing
|
||||
- Tests multiple endpoints together: /html, /screenshot, /pdf, /crawl
|
||||
- Validates each endpoint works correctly
|
||||
- Monitors success rates per endpoint
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
import docker
|
||||
import httpx
|
||||
from threading import Thread, Event
|
||||
|
||||
# Config
|
||||
IMAGE = "crawl4ai-local:latest"
|
||||
CONTAINER_NAME = "crawl4ai-test"
|
||||
PORT = 11235
|
||||
REQUESTS_PER_ENDPOINT = 10
|
||||
|
||||
# Stats
|
||||
stats_history = []
|
||||
stop_monitoring = Event()
|
||||
|
||||
def monitor_stats(container):
|
||||
"""Background stats collector."""
|
||||
for stat in container.stats(decode=True, stream=True):
|
||||
if stop_monitoring.is_set():
|
||||
break
|
||||
try:
|
||||
mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
|
||||
stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
|
||||
except:
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
|
||||
async def test_html(client, base_url, count):
|
||||
"""Test /html endpoint."""
|
||||
url = f"{base_url}/html"
|
||||
results = []
|
||||
for _ in range(count):
|
||||
start = time.time()
|
||||
try:
|
||||
resp = await client.post(url, json={"url": "https://httpbin.org/html"}, timeout=30.0)
|
||||
elapsed = (time.time() - start) * 1000
|
||||
results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
|
||||
except Exception as e:
|
||||
results.append({"success": False, "error": str(e)})
|
||||
return results
|
||||
|
||||
async def test_screenshot(client, base_url, count):
|
||||
"""Test /screenshot endpoint."""
|
||||
url = f"{base_url}/screenshot"
|
||||
results = []
|
||||
for _ in range(count):
|
||||
start = time.time()
|
||||
try:
|
||||
resp = await client.post(url, json={"url": "https://httpbin.org/html"}, timeout=30.0)
|
||||
elapsed = (time.time() - start) * 1000
|
||||
results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
|
||||
except Exception as e:
|
||||
results.append({"success": False, "error": str(e)})
|
||||
return results
|
||||
|
||||
async def test_pdf(client, base_url, count):
|
||||
"""Test /pdf endpoint."""
|
||||
url = f"{base_url}/pdf"
|
||||
results = []
|
||||
for _ in range(count):
|
||||
start = time.time()
|
||||
try:
|
||||
resp = await client.post(url, json={"url": "https://httpbin.org/html"}, timeout=30.0)
|
||||
elapsed = (time.time() - start) * 1000
|
||||
results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
|
||||
except Exception as e:
|
||||
results.append({"success": False, "error": str(e)})
|
||||
return results
|
||||
|
||||
async def test_crawl(client, base_url, count):
|
||||
"""Test /crawl endpoint."""
|
||||
url = f"{base_url}/crawl"
|
||||
results = []
|
||||
payload = {
|
||||
"urls": ["https://httpbin.org/html"],
|
||||
"browser_config": {},
|
||||
"crawler_config": {}
|
||||
}
|
||||
for _ in range(count):
|
||||
start = time.time()
|
||||
try:
|
||||
resp = await client.post(url, json=payload, timeout=30.0)
|
||||
elapsed = (time.time() - start) * 1000
|
||||
results.append({"success": resp.status_code == 200, "latency_ms": elapsed})
|
||||
except Exception as e:
|
||||
results.append({"success": False, "error": str(e)})
|
||||
return results
|
||||
|
||||
def start_container(client, image, name, port):
|
||||
"""Start container."""
|
||||
try:
|
||||
old = client.containers.get(name)
|
||||
print(f"🧹 Stopping existing container...")
|
||||
old.stop()
|
||||
old.remove()
|
||||
except docker.errors.NotFound:
|
||||
pass
|
||||
|
||||
print(f"🚀 Starting container...")
|
||||
container = client.containers.run(
|
||||
image, name=name, ports={f"{port}/tcp": port},
|
||||
detach=True, shm_size="1g", mem_limit="4g",
|
||||
)
|
||||
|
||||
print(f"⏳ Waiting for health...")
|
||||
for _ in range(30):
|
||||
time.sleep(1)
|
||||
container.reload()
|
||||
if container.status == "running":
|
||||
try:
|
||||
import requests
|
||||
if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
|
||||
print(f"✅ Container healthy!")
|
||||
return container
|
||||
except:
|
||||
pass
|
||||
raise TimeoutError("Container failed to start")
|
||||
|
||||
async def main():
|
||||
print("="*60)
|
||||
print("TEST 6: Multi-Endpoint Testing")
|
||||
print("="*60)
|
||||
|
||||
client = docker.from_env()
|
||||
container = None
|
||||
monitor_thread = None
|
||||
|
||||
try:
|
||||
container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
|
||||
|
||||
print(f"\n⏳ Waiting for permanent browser init (3s)...")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Start monitoring
|
||||
stop_monitoring.clear()
|
||||
stats_history.clear()
|
||||
monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
|
||||
monitor_thread.start()
|
||||
|
||||
await asyncio.sleep(1)
|
||||
baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
|
||||
print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
|
||||
|
||||
base_url = f"http://localhost:{PORT}"
|
||||
|
||||
# Test each endpoint
|
||||
endpoints = {
|
||||
"/html": test_html,
|
||||
"/screenshot": test_screenshot,
|
||||
"/pdf": test_pdf,
|
||||
"/crawl": test_crawl,
|
||||
}
|
||||
|
||||
all_endpoint_stats = {}
|
||||
|
||||
async with httpx.AsyncClient() as http_client:
|
||||
for endpoint_name, test_func in endpoints.items():
|
||||
print(f"🔄 Testing {endpoint_name} ({REQUESTS_PER_ENDPOINT} requests)...")
|
||||
results = await test_func(http_client, base_url, REQUESTS_PER_ENDPOINT)
|
||||
|
||||
successes = sum(1 for r in results if r.get("success"))
|
||||
success_rate = (successes / len(results)) * 100
|
||||
latencies = [r["latency_ms"] for r in results if "latency_ms" in r]
|
||||
avg_lat = sum(latencies) / len(latencies) if latencies else 0
|
||||
|
||||
all_endpoint_stats[endpoint_name] = {
|
||||
'success_rate': success_rate,
|
||||
'avg_latency': avg_lat,
|
||||
'total': len(results),
|
||||
'successes': successes
|
||||
}
|
||||
|
||||
print(f" ✓ Success: {success_rate:.1f}% ({successes}/{len(results)}), Avg: {avg_lat:.0f}ms")
|
||||
|
||||
# Stop monitoring
|
||||
await asyncio.sleep(1)
|
||||
stop_monitoring.set()
|
||||
if monitor_thread:
|
||||
monitor_thread.join(timeout=2)
|
||||
|
||||
# Final stats
|
||||
memory_samples = [s['memory_mb'] for s in stats_history]
|
||||
peak_mem = max(memory_samples) if memory_samples else 0
|
||||
final_mem = memory_samples[-1] if memory_samples else 0
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"RESULTS:")
|
||||
print(f"{'='*60}")
|
||||
for endpoint, stats in all_endpoint_stats.items():
|
||||
print(f" {endpoint:12} Success: {stats['success_rate']:5.1f}% Avg: {stats['avg_latency']:6.0f}ms")
|
||||
|
||||
print(f"\n Memory:")
|
||||
print(f" Baseline: {baseline_mem:.1f} MB")
|
||||
print(f" Peak: {peak_mem:.1f} MB")
|
||||
print(f" Final: {final_mem:.1f} MB")
|
||||
print(f" Delta: {final_mem - baseline_mem:+.1f} MB")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# Pass/Fail
|
||||
passed = True
|
||||
for endpoint, stats in all_endpoint_stats.items():
|
||||
if stats['success_rate'] < 100:
|
||||
print(f"❌ FAIL: {endpoint} success rate {stats['success_rate']:.1f}% < 100%")
|
||||
passed = False
|
||||
|
||||
if passed:
|
||||
print(f"✅ TEST PASSED")
|
||||
return 0
|
||||
else:
|
||||
return 1
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ TEST ERROR: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
finally:
|
||||
stop_monitoring.set()
|
||||
if container:
|
||||
print(f"🛑 Stopping container...")
|
||||
container.stop()
|
||||
container.remove()
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(main())
|
||||
exit(exit_code)
|
||||
Executable
+199
@@ -0,0 +1,199 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test 7: Cleanup Verification (Janitor)
|
||||
- Creates load spike then goes idle
|
||||
- Verifies memory returns to near baseline
|
||||
- Tests janitor cleanup of idle browsers
|
||||
- Monitors memory recovery time
|
||||
"""
|
||||
import asyncio
|
||||
import time
|
||||
import docker
|
||||
import httpx
|
||||
from threading import Thread, Event
|
||||
|
||||
# Config
|
||||
IMAGE = "crawl4ai-local:latest"
|
||||
CONTAINER_NAME = "crawl4ai-test"
|
||||
PORT = 11235
|
||||
SPIKE_REQUESTS = 20 # Create some browsers
|
||||
IDLE_TIME = 90 # Wait 90s for janitor (runs every 60s)
|
||||
|
||||
# Stats
|
||||
stats_history = []
|
||||
stop_monitoring = Event()
|
||||
|
||||
def monitor_stats(container):
|
||||
"""Background stats collector."""
|
||||
for stat in container.stats(decode=True, stream=True):
|
||||
if stop_monitoring.is_set():
|
||||
break
|
||||
try:
|
||||
mem_usage = stat['memory_stats'].get('usage', 0) / (1024 * 1024)
|
||||
stats_history.append({'timestamp': time.time(), 'memory_mb': mem_usage})
|
||||
except:
|
||||
pass
|
||||
time.sleep(1) # Sample every 1s for this test
|
||||
|
||||
def start_container(client, image, name, port):
|
||||
"""Start container."""
|
||||
try:
|
||||
old = client.containers.get(name)
|
||||
print(f"🧹 Stopping existing container...")
|
||||
old.stop()
|
||||
old.remove()
|
||||
except docker.errors.NotFound:
|
||||
pass
|
||||
|
||||
print(f"🚀 Starting container...")
|
||||
container = client.containers.run(
|
||||
image, name=name, ports={f"{port}/tcp": port},
|
||||
detach=True, shm_size="1g", mem_limit="4g",
|
||||
)
|
||||
|
||||
print(f"⏳ Waiting for health...")
|
||||
for _ in range(30):
|
||||
time.sleep(1)
|
||||
container.reload()
|
||||
if container.status == "running":
|
||||
try:
|
||||
import requests
|
||||
if requests.get(f"http://localhost:{port}/health", timeout=2).status_code == 200:
|
||||
print(f"✅ Container healthy!")
|
||||
return container
|
||||
except:
|
||||
pass
|
||||
raise TimeoutError("Container failed to start")
|
||||
|
||||
async def main():
|
||||
print("="*60)
|
||||
print("TEST 7: Cleanup Verification (Janitor)")
|
||||
print("="*60)
|
||||
|
||||
client = docker.from_env()
|
||||
container = None
|
||||
monitor_thread = None
|
||||
|
||||
try:
|
||||
container = start_container(client, IMAGE, CONTAINER_NAME, PORT)
|
||||
|
||||
print(f"\n⏳ Waiting for permanent browser init (3s)...")
|
||||
await asyncio.sleep(3)
|
||||
|
||||
# Start monitoring
|
||||
stop_monitoring.clear()
|
||||
stats_history.clear()
|
||||
monitor_thread = Thread(target=monitor_stats, args=(container,), daemon=True)
|
||||
monitor_thread.start()
|
||||
|
||||
await asyncio.sleep(2)
|
||||
baseline_mem = stats_history[-1]['memory_mb'] if stats_history else 0
|
||||
print(f"📏 Baseline: {baseline_mem:.1f} MB\n")
|
||||
|
||||
# Create load spike with different configs to populate pool
|
||||
print(f"🔥 Creating load spike ({SPIKE_REQUESTS} requests with varied configs)...")
|
||||
url = f"http://localhost:{PORT}/crawl"
|
||||
|
||||
viewports = [
|
||||
{"width": 1920, "height": 1080},
|
||||
{"width": 1024, "height": 768},
|
||||
{"width": 375, "height": 667},
|
||||
]
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0) as http_client:
|
||||
tasks = []
|
||||
for i in range(SPIKE_REQUESTS):
|
||||
vp = viewports[i % len(viewports)]
|
||||
payload = {
|
||||
"urls": ["https://httpbin.org/html"],
|
||||
"browser_config": {
|
||||
"type": "BrowserConfig",
|
||||
"params": {
|
||||
"viewport": {"type": "dict", "value": vp},
|
||||
"headless": True,
|
||||
"text_mode": True,
|
||||
"extra_args": [
|
||||
"--no-sandbox", "--disable-dev-shm-usage",
|
||||
"--disable-gpu", "--disable-software-rasterizer",
|
||||
"--disable-web-security", "--allow-insecure-localhost",
|
||||
"--ignore-certificate-errors"
|
||||
]
|
||||
}
|
||||
},
|
||||
"crawler_config": {}
|
||||
}
|
||||
tasks.append(http_client.post(url, json=payload))
|
||||
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
successes = sum(1 for r in results if hasattr(r, 'status_code') and r.status_code == 200)
|
||||
print(f" ✓ Spike completed: {successes}/{len(results)} successful")
|
||||
|
||||
# Measure peak
|
||||
await asyncio.sleep(2)
|
||||
peak_mem = max([s['memory_mb'] for s in stats_history]) if stats_history else baseline_mem
|
||||
print(f" 📊 Peak memory: {peak_mem:.1f} MB (+{peak_mem - baseline_mem:.1f} MB)")
|
||||
|
||||
# Now go idle and wait for janitor
|
||||
print(f"\n⏸️ Going idle for {IDLE_TIME}s (janitor cleanup)...")
|
||||
print(f" (Janitor runs every 60s, checking for idle browsers)")
|
||||
|
||||
for elapsed in range(0, IDLE_TIME, 10):
|
||||
await asyncio.sleep(10)
|
||||
current_mem = stats_history[-1]['memory_mb'] if stats_history else 0
|
||||
print(f" [{elapsed+10:3d}s] Memory: {current_mem:.1f} MB")
|
||||
|
||||
# Stop monitoring
|
||||
stop_monitoring.set()
|
||||
if monitor_thread:
|
||||
monitor_thread.join(timeout=2)
|
||||
|
||||
# Analyze memory recovery
|
||||
final_mem = stats_history[-1]['memory_mb'] if stats_history else 0
|
||||
recovery_mb = peak_mem - final_mem
|
||||
recovery_pct = (recovery_mb / (peak_mem - baseline_mem) * 100) if (peak_mem - baseline_mem) > 0 else 0
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"RESULTS:")
|
||||
print(f"{'='*60}")
|
||||
print(f" Memory Journey:")
|
||||
print(f" Baseline: {baseline_mem:.1f} MB")
|
||||
print(f" Peak: {peak_mem:.1f} MB (+{peak_mem - baseline_mem:.1f} MB)")
|
||||
print(f" Final: {final_mem:.1f} MB (+{final_mem - baseline_mem:.1f} MB)")
|
||||
print(f" Recovered: {recovery_mb:.1f} MB ({recovery_pct:.1f}%)")
|
||||
print(f"{'='*60}")
|
||||
|
||||
# Pass/Fail
|
||||
passed = True
|
||||
|
||||
# Should have created some memory pressure
|
||||
if peak_mem - baseline_mem < 100:
|
||||
print(f"⚠️ WARNING: Peak increase only {peak_mem - baseline_mem:.1f} MB (expected more browsers)")
|
||||
|
||||
# Should recover most memory (within 100MB of baseline)
|
||||
if final_mem - baseline_mem > 100:
|
||||
print(f"⚠️ WARNING: Memory didn't recover well (still +{final_mem - baseline_mem:.1f} MB above baseline)")
|
||||
else:
|
||||
print(f"✅ Good memory recovery!")
|
||||
|
||||
# Baseline + 50MB tolerance
|
||||
if final_mem - baseline_mem < 50:
|
||||
print(f"✅ Excellent cleanup (within 50MB of baseline)")
|
||||
|
||||
print(f"✅ TEST PASSED")
|
||||
return 0
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ TEST ERROR: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
return 1
|
||||
finally:
|
||||
stop_monitoring.set()
|
||||
if container:
|
||||
print(f"🛑 Stopping container...")
|
||||
container.stop()
|
||||
container.remove()
|
||||
|
||||
if __name__ == "__main__":
|
||||
exit_code = asyncio.run(main())
|
||||
exit(exit_code)
|
||||
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Quick test to generate monitor dashboard activity"""
|
||||
import httpx
|
||||
import asyncio
|
||||
|
||||
async def test_dashboard():
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
print("📊 Generating dashboard activity...")
|
||||
|
||||
# Test 1: Simple crawl
|
||||
print("\n1️⃣ Running simple crawl...")
|
||||
r1 = await client.post(
|
||||
"http://localhost:11235/crawl",
|
||||
json={"urls": ["https://httpbin.org/html"], "crawler_config": {}}
|
||||
)
|
||||
print(f" Status: {r1.status_code}")
|
||||
|
||||
# Test 2: Multiple URLs
|
||||
print("\n2️⃣ Running multi-URL crawl...")
|
||||
r2 = await client.post(
|
||||
"http://localhost:11235/crawl",
|
||||
json={
|
||||
"urls": [
|
||||
"https://httpbin.org/html",
|
||||
"https://httpbin.org/json"
|
||||
],
|
||||
"crawler_config": {}
|
||||
}
|
||||
)
|
||||
print(f" Status: {r2.status_code}")
|
||||
|
||||
# Test 3: Check monitor health
|
||||
print("\n3️⃣ Checking monitor health...")
|
||||
r3 = await client.get("http://localhost:11235/monitor/health")
|
||||
health = r3.json()
|
||||
print(f" Memory: {health['container']['memory_percent']}%")
|
||||
print(f" Browsers: {health['pool']['permanent']['active']}")
|
||||
|
||||
# Test 4: Check requests
|
||||
print("\n4️⃣ Checking request log...")
|
||||
r4 = await client.get("http://localhost:11235/monitor/requests")
|
||||
reqs = r4.json()
|
||||
print(f" Active: {len(reqs['active'])}")
|
||||
print(f" Completed: {len(reqs['completed'])}")
|
||||
|
||||
# Test 5: Check endpoint stats
|
||||
print("\n5️⃣ Checking endpoint stats...")
|
||||
r5 = await client.get("http://localhost:11235/monitor/endpoints/stats")
|
||||
stats = r5.json()
|
||||
for endpoint, data in stats.items():
|
||||
print(f" {endpoint}: {data['count']} requests, {data['avg_latency_ms']}ms avg")
|
||||
|
||||
print("\n✅ Dashboard should now show activity!")
|
||||
print(f"\n🌐 Open: http://localhost:11235/dashboard")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_dashboard())
|
||||
@@ -0,0 +1,312 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Adversarial security tests for the 4 vulns reported 2026-04-13.
|
||||
Self-contained: copies validation logic to avoid import issues with dns.resolver etc.
|
||||
|
||||
Vuln 1: Arbitrary File Write via /screenshot + /pdf output_path
|
||||
Vuln 2: Monitor Auth Bypass (structural source check)
|
||||
Vuln 3: Stored XSS in Monitor Dashboard (server-side + client-side)
|
||||
Vuln 4: SSRF via Webhook URL
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
import ipaddress
|
||||
import socket
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Local copies of security utilities (to avoid dns.resolver import in utils.py)
|
||||
# ============================================================================
|
||||
|
||||
ALLOWED_OUTPUT_DIR = os.environ.get("CRAWL4AI_OUTPUT_DIR", "/tmp/crawl4ai-outputs")
|
||||
|
||||
|
||||
def validate_output_path(user_path):
|
||||
safe_path = os.path.normpath(user_path).lstrip(os.sep)
|
||||
abs_path = os.path.abspath(os.path.join(ALLOWED_OUTPUT_DIR, safe_path))
|
||||
abs_allowed = os.path.abspath(ALLOWED_OUTPUT_DIR) + os.sep
|
||||
if not abs_path.startswith(abs_allowed):
|
||||
raise ValueError(f"output_path must resolve within {ALLOWED_OUTPUT_DIR}")
|
||||
return abs_path
|
||||
|
||||
|
||||
_BLOCKED_NETWORKS = [
|
||||
ipaddress.ip_network("0.0.0.0/8"),
|
||||
ipaddress.ip_network("10.0.0.0/8"),
|
||||
ipaddress.ip_network("100.64.0.0/10"),
|
||||
ipaddress.ip_network("127.0.0.0/8"),
|
||||
ipaddress.ip_network("169.254.0.0/16"),
|
||||
ipaddress.ip_network("172.16.0.0/12"),
|
||||
ipaddress.ip_network("192.0.0.0/24"),
|
||||
ipaddress.ip_network("192.168.0.0/16"),
|
||||
ipaddress.ip_network("198.18.0.0/15"),
|
||||
ipaddress.ip_network("::1/128"),
|
||||
ipaddress.ip_network("fc00::/7"),
|
||||
ipaddress.ip_network("fe80::/10"),
|
||||
]
|
||||
_BLOCKED_HOSTNAMES = {
|
||||
"localhost", "metadata.google.internal", "metadata",
|
||||
"kubernetes.default", "kubernetes.default.svc",
|
||||
}
|
||||
|
||||
|
||||
def validate_webhook_url(url):
|
||||
parsed = urlparse(str(url))
|
||||
hostname = parsed.hostname
|
||||
if not hostname:
|
||||
raise ValueError("Webhook URL must have a valid hostname")
|
||||
hostname_lower = hostname.lower()
|
||||
if hostname_lower in _BLOCKED_HOSTNAMES:
|
||||
raise ValueError(f"Webhook URL hostname '{hostname}' is blocked")
|
||||
if hostname_lower.startswith("host.docker.internal"):
|
||||
raise ValueError(f"Webhook URL hostname '{hostname}' is blocked")
|
||||
try:
|
||||
resolved = socket.getaddrinfo(hostname, None)
|
||||
except socket.gaierror:
|
||||
raise ValueError(f"Cannot resolve webhook hostname '{hostname}'")
|
||||
for _, _, _, _, sockaddr in resolved:
|
||||
ip = ipaddress.ip_address(sockaddr[0])
|
||||
for network in _BLOCKED_NETWORKS:
|
||||
if ip in network:
|
||||
raise ValueError(f"Webhook URL resolves to blocked address: {ip}")
|
||||
|
||||
|
||||
DEPLOY_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# VULN 1: Arbitrary File Write - Path Traversal
|
||||
# ============================================================================
|
||||
|
||||
class TestOutputPathRemoved(unittest.TestCase):
|
||||
"""output_path is gone; the server owns paths via the artifact store.
|
||||
|
||||
The old string-only validate_output_path was bypassable (symlink/TOCTOU,
|
||||
sibling-prefix '...-evil') -> arbitrary write -> RCE. The fix is to never
|
||||
accept a caller path at all. Behavioral artifact-store coverage (O_NOFOLLOW,
|
||||
O_EXCL, hex id, TTL, quota) lives in test_security_artifact_store.py.
|
||||
"""
|
||||
|
||||
def test_screenshot_request_has_no_output_path(self):
|
||||
sys.path.insert(0, DEPLOY_DIR)
|
||||
from schemas import ScreenshotRequest
|
||||
self.assertNotIn("output_path", ScreenshotRequest.model_fields)
|
||||
|
||||
def test_pdf_request_has_no_output_path(self):
|
||||
sys.path.insert(0, DEPLOY_DIR)
|
||||
from schemas import PDFRequest
|
||||
self.assertNotIn("output_path", PDFRequest.model_fields)
|
||||
|
||||
def test_validate_output_path_deleted(self):
|
||||
sys.path.insert(0, DEPLOY_DIR)
|
||||
import utils
|
||||
self.assertFalse(
|
||||
hasattr(utils, "validate_output_path"),
|
||||
"validate_output_path must be deleted (caller paths no longer accepted)",
|
||||
)
|
||||
|
||||
|
||||
class TestPydanticPathValidator(unittest.TestCase):
|
||||
"""output_path (and its traversal validator) are gone entirely.
|
||||
|
||||
Traversal rejection used to be the mitigation; the real fix is that no
|
||||
caller path is accepted at all, so there is nothing to traverse. The
|
||||
sandboxed artifact store owns all paths now.
|
||||
"""
|
||||
|
||||
def test_no_output_path_field_on_request_models(self):
|
||||
sys.path.insert(0, DEPLOY_DIR)
|
||||
from schemas import ScreenshotRequest, PDFRequest
|
||||
self.assertNotIn("output_path", ScreenshotRequest.model_fields)
|
||||
self.assertNotIn("output_path", PDFRequest.model_fields)
|
||||
|
||||
def test_traversal_validator_removed(self):
|
||||
# No reject_traversal validator should remain registered on the models.
|
||||
sys.path.insert(0, DEPLOY_DIR)
|
||||
from schemas import ScreenshotRequest, PDFRequest
|
||||
for model in (ScreenshotRequest, PDFRequest):
|
||||
names = set(getattr(model, "__pydantic_decorators__").field_validators)
|
||||
self.assertNotIn("reject_traversal", names)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# VULN 2: Monitor Auth Bypass (structural check)
|
||||
# ============================================================================
|
||||
|
||||
class TestMonitorAuthStructural(unittest.TestCase):
|
||||
|
||||
def test_monitor_router_has_auth(self):
|
||||
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
|
||||
source = f.read()
|
||||
# Find the line with monitor_router
|
||||
for line in source.splitlines():
|
||||
if "monitor_router" in line and "include_router" in line:
|
||||
self.assertIn("dependencies=", line,
|
||||
"Monitor router must have dependencies=[Depends(token_dep)]")
|
||||
return
|
||||
self.fail("Could not find monitor_router include_router line")
|
||||
|
||||
def test_websocket_auth_enforced_by_gate(self):
|
||||
"""The monitor WS is gated by the AuthGateMiddleware (behavioral).
|
||||
|
||||
Auth moved out of the route's bespoke, timing-unsafe env-token check
|
||||
and into the outermost ASGI gate, which closes an unauthenticated
|
||||
WebSocket before it is accepted.
|
||||
"""
|
||||
import sys
|
||||
sys.path.insert(0, DEPLOY_DIR)
|
||||
import server
|
||||
from starlette.testclient import TestClient
|
||||
|
||||
client = TestClient(server.app)
|
||||
# Unauthenticated connect must be rejected by the gate.
|
||||
with self.assertRaises(Exception):
|
||||
with client.websocket_connect("/monitor/ws") as ws:
|
||||
ws.receive_text()
|
||||
|
||||
# The old in-route check must be gone (it read a different secret than
|
||||
# the REST path and compared with a timing-unsafe `!=`).
|
||||
with open(os.path.join(DEPLOY_DIR, "monitor_routes.py")) as f:
|
||||
source = f.read()
|
||||
self.assertNotIn('os.environ.get("CRAWL4AI_API_TOKEN"', source,
|
||||
"bespoke WS token check must be removed; the gate owns WS auth")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# VULN 3: Stored XSS (server-side + client-side)
|
||||
# ============================================================================
|
||||
|
||||
class TestXSSPrevention(unittest.TestCase):
|
||||
|
||||
def test_html_escape_blocks_script_tags(self):
|
||||
import html
|
||||
payload = '<script>alert(document.cookie)</script>'
|
||||
escaped = html.escape(payload)
|
||||
self.assertNotIn("<script>", escaped)
|
||||
self.assertIn("<script>", escaped)
|
||||
|
||||
def test_html_escape_blocks_img_tag(self):
|
||||
import html
|
||||
payload = '"><img src=x onerror=alert(1)>'
|
||||
escaped = html.escape(payload)
|
||||
self.assertNotIn("<img", escaped)
|
||||
self.assertIn("<img", escaped)
|
||||
|
||||
def test_monitor_py_uses_html_escape(self):
|
||||
with open(os.path.join(DEPLOY_DIR, "monitor.py")) as f:
|
||||
source = f.read()
|
||||
self.assertIn("import html", source)
|
||||
self.assertIn("html.escape(url", source)
|
||||
self.assertIn("html.escape(str(error)", source)
|
||||
|
||||
def test_index_html_has_escape_function(self):
|
||||
html_path = os.path.join(DEPLOY_DIR, "static", "monitor", "index.html")
|
||||
with open(html_path) as f:
|
||||
source = f.read()
|
||||
self.assertIn("function escapeHtml(", source)
|
||||
|
||||
def test_index_html_no_raw_url_injection(self):
|
||||
html_path = os.path.join(DEPLOY_DIR, "static", "monitor", "index.html")
|
||||
with open(html_path) as f:
|
||||
source = f.read()
|
||||
self.assertNotIn("${req.url}", source, "Raw ${req.url} found")
|
||||
self.assertNotIn("${err.url}", source, "Raw ${err.url} found")
|
||||
self.assertNotIn("${err.error}", source, "Raw ${err.error} found")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# VULN 4: SSRF via Webhook URL
|
||||
# ============================================================================
|
||||
|
||||
class TestSSRFBlocked(unittest.TestCase):
|
||||
|
||||
def test_localhost_ip(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_webhook_url("http://127.0.0.1:9999/steal")
|
||||
|
||||
def test_localhost_hostname(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_webhook_url("http://localhost:9999/steal")
|
||||
|
||||
def test_loopback_127_x(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_webhook_url("http://127.0.0.2:9999/steal")
|
||||
|
||||
def test_10_network(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_webhook_url("http://10.0.0.1:8080/admin")
|
||||
|
||||
def test_172_16_network(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_webhook_url("http://172.16.0.1:8080/admin")
|
||||
|
||||
def test_192_168_network(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_webhook_url("http://192.168.1.1:8080/admin")
|
||||
|
||||
def test_aws_metadata_ip(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_webhook_url("http://169.254.169.254/latest/meta-data/")
|
||||
|
||||
def test_gcp_metadata_hostname(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_webhook_url("http://metadata.google.internal/computeMetadata/v1/")
|
||||
|
||||
def test_docker_internal(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_webhook_url("http://host.docker.internal:9998/steal")
|
||||
|
||||
def test_kubernetes_default(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_webhook_url("http://kubernetes.default/api/v1/secrets")
|
||||
|
||||
def test_empty_hostname(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_webhook_url("http:///steal")
|
||||
|
||||
def test_valid_external_url(self):
|
||||
validate_webhook_url("https://webhook.site/test-uuid")
|
||||
validate_webhook_url("https://hooks.slack.com/services/T00/B00/xxx")
|
||||
|
||||
def test_valid_external_with_port(self):
|
||||
"""External URL with port -- may fail DNS in CI, that's expected."""
|
||||
try:
|
||||
validate_webhook_url("https://google.com:443/webhook")
|
||||
except ValueError as e:
|
||||
if "Cannot resolve" in str(e):
|
||||
self.skipTest("DNS resolution unavailable in this environment")
|
||||
raise
|
||||
|
||||
|
||||
class TestWebhookSourceChecks(unittest.TestCase):
|
||||
|
||||
def test_redirects_not_auto_followed(self):
|
||||
# Redirects are followed manually and re-validated per hop, never
|
||||
# auto-followed (which would skip the SSRF re-check).
|
||||
with open(os.path.join(DEPLOY_DIR, "webhook.py")) as f:
|
||||
source = f.read()
|
||||
self.assertIn("allow_redirects=False", source)
|
||||
self.assertIn("check_redirect", source) # each hop re-validated
|
||||
|
||||
def test_webhook_validates_at_send_time(self):
|
||||
# Validation happens at send time via the egress broker (resolve_and_pin
|
||||
# rejects non-global targets); the connection is also pinned to that IP.
|
||||
with open(os.path.join(DEPLOY_DIR, "webhook.py")) as f:
|
||||
source = f.read()
|
||||
self.assertIn("resolve_and_pin", source)
|
||||
|
||||
def test_job_validates_webhook(self):
|
||||
with open(os.path.join(DEPLOY_DIR, "job.py")) as f:
|
||||
source = f.read()
|
||||
self.assertIn("validate_webhook_url", source)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("=" * 70)
|
||||
print("Crawl4AI Security Tests - 2026-04-13 Vulnerabilities")
|
||||
print("=" * 70)
|
||||
print()
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Adversarial security tests for Batch 2 vulns reported 2026-04-14 (by111/August829).
|
||||
Self-contained tests that verify fixes at the code/source level.
|
||||
|
||||
B2-V1: /execute_js disabled by default + SSRF block
|
||||
B2-V2: Hardcoded JWT secret removed
|
||||
B2-V3: eval() in /config/dump replaced with JSON
|
||||
B2-V4: Hook sandbox __builtins__ escape fixed
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import ast
|
||||
import unittest
|
||||
import builtins
|
||||
import types
|
||||
from unittest import mock
|
||||
|
||||
DEPLOY_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# B2-V2: Hardcoded JWT Secret
|
||||
# ============================================================================
|
||||
|
||||
class TestJWTSecretHardened(unittest.TestCase):
|
||||
"""Verify the hardcoded 'mysecret' default is gone from auth.py."""
|
||||
|
||||
def test_no_mysecret_as_default(self):
|
||||
"""auth.py must not use 'mysecret' as a fallback default for SECRET_KEY."""
|
||||
with open(os.path.join(DEPLOY_DIR, "auth.py")) as f:
|
||||
source = f.read()
|
||||
# The old dangerous pattern: os.environ.get("SECRET_KEY", "mysecret")
|
||||
self.assertNotIn('get("SECRET_KEY", "mysecret")', source,
|
||||
"auth.py must not use 'mysecret' as env var default")
|
||||
|
||||
def test_weak_secret_validation_exists(self):
|
||||
"""auth.py must reject weak and too-short SECRET_KEY values (behavioral)."""
|
||||
import importlib
|
||||
import sys
|
||||
sys.path.insert(0, DEPLOY_DIR)
|
||||
auth = importlib.import_module("auth")
|
||||
|
||||
# A known-weak secret must be rejected.
|
||||
with mock.patch.dict(os.environ, {"SECRET_KEY": "mysecret"}):
|
||||
with self.assertRaises(RuntimeError):
|
||||
auth.resolve_secret_key(required=True)
|
||||
|
||||
# A too-short secret (< 32 chars) must be rejected.
|
||||
with mock.patch.dict(os.environ, {"SECRET_KEY": "short"}):
|
||||
with self.assertRaises(RuntimeError):
|
||||
auth.resolve_secret_key(required=True)
|
||||
|
||||
# A strong secret must be accepted and returned unchanged.
|
||||
strong = "a" * 32
|
||||
with mock.patch.dict(os.environ, {"SECRET_KEY": strong}):
|
||||
self.assertEqual(auth.resolve_secret_key(required=True), strong)
|
||||
|
||||
def test_mysecret_in_weak_list(self):
|
||||
"""'mysecret' must be in the weak secrets blocklist."""
|
||||
with open(os.path.join(DEPLOY_DIR, "auth.py")) as f:
|
||||
source = f.read()
|
||||
# Parse the source to find _WEAK_SECRETS set
|
||||
self.assertIn("mysecret", source,
|
||||
"'mysecret' must be listed in _WEAK_SECRETS blocklist")
|
||||
|
||||
def test_auto_generation_exists(self):
|
||||
"""auth.py must auto-generate key when none is set."""
|
||||
with open(os.path.join(DEPLOY_DIR, "auth.py")) as f:
|
||||
source = f.read()
|
||||
self.assertIn("token_hex", source,
|
||||
"auth.py must use secrets.token_hex for auto-generation")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# B2-V3: eval() removed from /config/dump
|
||||
# ============================================================================
|
||||
|
||||
class TestConfigDumpNoEval(unittest.TestCase):
|
||||
"""Verify eval() is completely removed from the /config/dump path."""
|
||||
|
||||
def test_no_safe_eval_config(self):
|
||||
"""_safe_eval_config function must be removed from server.py."""
|
||||
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
|
||||
source = f.read()
|
||||
self.assertNotIn("def _safe_eval_config", source,
|
||||
"_safe_eval_config must be deleted (replaced with JSON input)")
|
||||
|
||||
def test_config_from_json_exists(self):
|
||||
"""_config_from_json function must exist."""
|
||||
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
|
||||
source = f.read()
|
||||
self.assertIn("def _config_from_json", source,
|
||||
"_config_from_json must replace _safe_eval_config")
|
||||
|
||||
def test_config_dump_has_auth(self):
|
||||
"""config_dump endpoint must require authentication."""
|
||||
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
|
||||
source = f.read()
|
||||
# Find the config_dump function and check it has token_dep
|
||||
idx = source.index("config_dump")
|
||||
# Look backwards for the decorator/function definition area
|
||||
nearby = source[max(0, idx-200):idx+200]
|
||||
self.assertIn("token_dep", nearby,
|
||||
"/config/dump must require token_dep authentication")
|
||||
|
||||
def test_no_eval_in_config_path(self):
|
||||
"""No eval() call should exist in the config dump code path."""
|
||||
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
|
||||
source = f.read()
|
||||
# The old allowlist constants should be gone
|
||||
self.assertNotIn("_SAFE_CONFIG_ALLOWED_NAMES", source,
|
||||
"Old eval allowlist constants should be removed")
|
||||
self.assertNotIn("_SAFE_CONFIG_ALLOWED_ATTRS", source,
|
||||
"Old eval allowlist constants should be removed")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# B2-V1: /execute_js disabled by default
|
||||
# ============================================================================
|
||||
|
||||
class TestExecuteJsDisabled(unittest.TestCase):
|
||||
"""Verify /execute_js is disabled by default with proper guards."""
|
||||
|
||||
def test_execute_js_flag_exists(self):
|
||||
"""EXECUTE_JS_ENABLED flag must exist in server.py."""
|
||||
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
|
||||
source = f.read()
|
||||
self.assertIn("EXECUTE_JS_ENABLED", source)
|
||||
|
||||
def test_execute_js_disabled_by_default(self):
|
||||
"""EXECUTE_JS_ENABLED must default to false."""
|
||||
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
|
||||
source = f.read()
|
||||
# Find the line that sets EXECUTE_JS_ENABLED
|
||||
for line in source.splitlines():
|
||||
if "EXECUTE_JS_ENABLED" in line and "os.environ" in line:
|
||||
self.assertIn('"false"', line,
|
||||
"EXECUTE_JS_ENABLED must default to 'false'")
|
||||
return
|
||||
self.fail("Could not find EXECUTE_JS_ENABLED env var line")
|
||||
|
||||
def test_execute_js_checks_flag(self):
|
||||
"""execute_js endpoint must check EXECUTE_JS_ENABLED."""
|
||||
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
|
||||
source = f.read()
|
||||
idx = source.index("async def execute_js")
|
||||
func_body = source[idx:idx+3000]
|
||||
self.assertIn("EXECUTE_JS_ENABLED", func_body,
|
||||
"execute_js must check EXECUTE_JS_ENABLED flag")
|
||||
|
||||
def test_execute_js_has_ssrf_check(self):
|
||||
"""execute_js must validate URL against SSRF blocklist."""
|
||||
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
|
||||
source = f.read()
|
||||
idx = source.index("async def execute_js")
|
||||
func_body = source[idx:idx+3000]
|
||||
self.assertIn("validate_webhook_url", func_body,
|
||||
"execute_js must validate URL against SSRF blocklist")
|
||||
|
||||
def test_disable_web_security_removed_from_defaults(self):
|
||||
"""--disable-web-security must not be in default browser args."""
|
||||
with open(os.path.join(DEPLOY_DIR, "utils.py")) as f:
|
||||
source = f.read()
|
||||
# Find the DEFAULT_CONFIG extra_args
|
||||
tree = ast.parse(source)
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Constant) and node.value == "--disable-web-security":
|
||||
self.fail("--disable-web-security must not be in DEFAULT_CONFIG extra_args")
|
||||
|
||||
def test_disable_web_security_removed_from_config_yml(self):
|
||||
"""--disable-web-security must not be active in config.yml."""
|
||||
with open(os.path.join(DEPLOY_DIR, "config.yml")) as f:
|
||||
for line in f:
|
||||
stripped = line.strip()
|
||||
if stripped == '- "--disable-web-security"':
|
||||
self.fail("--disable-web-security must not be an active entry in config.yml")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# B2-V4: exec-based hook system removed (replaced by declarative registry)
|
||||
# ============================================================================
|
||||
|
||||
class TestHookExecPathRemoved(unittest.TestCase):
|
||||
"""The exec()/compile() hook system is gone; hooks are now declarative.
|
||||
|
||||
No sandbox survives attacker Python in-process, so hook_manager.py was
|
||||
deleted and replaced by hook_registry.py (a fixed set of server-authored
|
||||
actions selected by name with schema-validated scalar params).
|
||||
"""
|
||||
|
||||
def test_hook_manager_module_deleted(self):
|
||||
self.assertFalse(
|
||||
os.path.exists(os.path.join(DEPLOY_DIR, "hook_manager.py")),
|
||||
"hook_manager.py (the exec-based hook system) must be deleted",
|
||||
)
|
||||
|
||||
def test_no_exec_compile_eval_in_docker_layer(self):
|
||||
"""The RCE primitives must be absent from the docker server code.
|
||||
|
||||
Uses AST so it flags only bare-builtin calls (exec/eval/compile), not
|
||||
attribute calls like re.compile(...) or the words in comments/docstrings.
|
||||
"""
|
||||
import ast
|
||||
import glob
|
||||
offenders = []
|
||||
for path in glob.glob(os.path.join(DEPLOY_DIR, "*.py")):
|
||||
with open(path) as f:
|
||||
tree = ast.parse(f.read(), filename=path)
|
||||
for node in ast.walk(tree):
|
||||
if (
|
||||
isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Name)
|
||||
and node.func.id in {"exec", "eval", "compile"}
|
||||
):
|
||||
offenders.append((os.path.basename(path), node.func.id, node.lineno))
|
||||
self.assertEqual(offenders, [], f"bare exec/eval/compile call found: {offenders}")
|
||||
|
||||
def test_registry_rejects_unknown_action(self):
|
||||
sys.path.insert(0, DEPLOY_DIR)
|
||||
from hook_registry import build_declarative_hooks, HookValidationError
|
||||
with self.assertRaises(HookValidationError):
|
||||
build_declarative_hooks([{"action": "run_python", "params": {"code": "x"}}])
|
||||
|
||||
def test_registry_rejects_bad_params(self):
|
||||
sys.path.insert(0, DEPLOY_DIR)
|
||||
from hook_registry import build_declarative_hooks, HookValidationError
|
||||
with self.assertRaises(HookValidationError):
|
||||
build_declarative_hooks([{"action": "block_resources", "params": {"resource_types": ["script"]}}])
|
||||
|
||||
def test_registry_builds_valid_hooks(self):
|
||||
sys.path.insert(0, DEPLOY_DIR)
|
||||
from hook_registry import build_declarative_hooks
|
||||
hooks = build_declarative_hooks([
|
||||
{"action": "block_resources", "params": {"resource_types": ["image"]}},
|
||||
{"action": "scroll_to_bottom", "params": {"max_steps": 5}},
|
||||
])
|
||||
# block_resources -> on_page_context_created, scroll -> before_retrieve_html
|
||||
self.assertIn("on_page_context_created", hooks)
|
||||
self.assertIn("before_retrieve_html", hooks)
|
||||
for fn in hooks.values():
|
||||
self.assertTrue(callable(fn))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,108 @@
|
||||
"""
|
||||
R4 artifact-store behavioral tests: the server owns the directory, names and
|
||||
bytes. Writes are O_EXCL|O_NOFOLLOW 0600; retrieval requires a 32-hex id,
|
||||
refuses symlinks/non-regular files, and enforces a TTL.
|
||||
"""
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path, monkeypatch):
|
||||
"""Point the artifact store at an isolated temp dir and reload it."""
|
||||
monkeypatch.setenv("CRAWL4AI_ARTIFACT_DIR", str(tmp_path / "art"))
|
||||
monkeypatch.setenv("CRAWL4AI_MAX_ARTIFACT_BYTES", "1024")
|
||||
monkeypatch.setenv("CRAWL4AI_ARTIFACT_QUOTA_BYTES", "4096")
|
||||
monkeypatch.setenv("CRAWL4AI_ARTIFACT_TTL_SECONDS", "3600")
|
||||
import importlib
|
||||
import artifacts
|
||||
importlib.reload(artifacts)
|
||||
artifacts.init_store()
|
||||
yield artifacts
|
||||
importlib.reload(artifacts) # restore defaults for other tests
|
||||
|
||||
|
||||
pytestmark = pytest.mark.posture
|
||||
|
||||
|
||||
class TestWrite:
|
||||
def test_write_returns_hex_id_and_meta(self, store):
|
||||
meta = store.write_artifact("png", b"\x89PNG data")
|
||||
assert len(meta["artifact_id"]) == 32 and all(c in "0123456789abcdef" for c in meta["artifact_id"])
|
||||
assert meta["mime"] == "image/png" and meta["size"] == len(b"\x89PNG data")
|
||||
|
||||
def test_dir_is_0700_and_file_0600(self, store):
|
||||
meta = store.write_artifact("pdf", b"%PDF-1.4")
|
||||
assert oct(os.stat(store.ARTIFACT_DIR).st_mode)[-3:] == "700"
|
||||
path, _ = store.resolve_artifact(meta["artifact_id"])
|
||||
assert oct(os.stat(path).st_mode)[-3:] == "600"
|
||||
|
||||
def test_oversize_rejected(self, store):
|
||||
with pytest.raises(store.ArtifactTooLarge):
|
||||
store.write_artifact("png", b"x" * 2048) # cap is 1024
|
||||
|
||||
def test_quota_enforced(self, store):
|
||||
for _ in range(4):
|
||||
store.write_artifact("png", b"x" * 1000)
|
||||
with pytest.raises(store.QuotaExceeded):
|
||||
store.write_artifact("png", b"x" * 1000) # would exceed 4096
|
||||
|
||||
|
||||
class TestResolve:
|
||||
def test_roundtrip(self, store):
|
||||
meta = store.write_artifact("png", b"hello")
|
||||
path, mime = store.resolve_artifact(meta["artifact_id"])
|
||||
assert mime == "image/png"
|
||||
with open(path, "rb") as f:
|
||||
assert f.read() == b"hello"
|
||||
|
||||
@pytest.mark.parametrize("bad", ["../etc/passwd", "..", "g" * 32, "abc", "/etc/passwd", "a" * 31])
|
||||
def test_non_hex_or_traversal_404(self, store, bad):
|
||||
with pytest.raises(store.ArtifactNotFound):
|
||||
store.resolve_artifact(bad)
|
||||
|
||||
def test_symlink_not_followed(self, store):
|
||||
# Plant a symlink named like a valid artifact pointing at a secret.
|
||||
secret = os.path.join(store.ARTIFACT_DIR, "secret.txt")
|
||||
with open(secret, "wb") as f:
|
||||
f.write(b"TOPSECRET")
|
||||
fake_id = "a" * 32
|
||||
link = os.path.join(store.ARTIFACT_DIR, fake_id + ".png")
|
||||
os.symlink(secret, link)
|
||||
with pytest.raises(store.ArtifactNotFound):
|
||||
store.resolve_artifact(fake_id) # lstat sees a symlink -> refuse
|
||||
|
||||
def test_ttl_expired_404_and_reaped(self, store):
|
||||
meta = store.write_artifact("png", b"old")
|
||||
path, _ = store.resolve_artifact(meta["artifact_id"])
|
||||
old = time.time() - 7200 # 2h ago, TTL is 1h
|
||||
os.utime(path, (old, old))
|
||||
with pytest.raises(store.ArtifactNotFound):
|
||||
store.resolve_artifact(meta["artifact_id"])
|
||||
assert not os.path.exists(path) # resolve reaped it
|
||||
|
||||
|
||||
class TestWriteIsExclusiveAndNoFollow:
|
||||
def test_write_uses_oexcl_nofollow(self, store):
|
||||
import inspect
|
||||
src = inspect.getsource(store.write_artifact)
|
||||
assert "O_EXCL" in src and "O_NOFOLLOW" in src
|
||||
|
||||
|
||||
class TestJanitor:
|
||||
def test_janitor_removes_symlinks_and_expired(self, store):
|
||||
meta = store.write_artifact("png", b"keep")
|
||||
# expired regular file
|
||||
old_meta = store.write_artifact("pdf", b"old")
|
||||
op, _ = store.resolve_artifact(old_meta["artifact_id"])
|
||||
past = time.time() - 7200
|
||||
os.utime(op, (past, past))
|
||||
# a planted symlink
|
||||
os.symlink("/etc/passwd", os.path.join(store.ARTIFACT_DIR, "deadbeef" * 4 + ".png"))
|
||||
reaped = store.janitor()
|
||||
assert reaped >= 2
|
||||
# the fresh one survives
|
||||
store.resolve_artifact(meta["artifact_id"])
|
||||
@@ -0,0 +1,116 @@
|
||||
"""
|
||||
Behavioral authorization tests for R1 (beyond the blanket default-deny gate):
|
||||
|
||||
* admin scope - monitor destructive actions require an admin principal;
|
||||
a valid data-scope token is allowed past the gate but
|
||||
rejected (403) by require_admin.
|
||||
* MCP no-laundering - the MCP tool proxy authenticates its internal loopback
|
||||
call (it no longer relies on the endpoints being open).
|
||||
* job ownership - a task records its owner; a different requester gets 404
|
||||
(not 403), and an admin can read any task.
|
||||
|
||||
These exercise the running app, not its source text.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.posture
|
||||
|
||||
from auth import create_access_token # noqa: E402
|
||||
|
||||
|
||||
def _bearer(token: str) -> dict:
|
||||
return {"Authorization": f"Bearer {token}"}
|
||||
|
||||
|
||||
# ───────────────────────────── admin scope ─────────────────────────────
|
||||
class TestAdminScope:
|
||||
def test_data_scope_cannot_reset_stats(self, stock_client):
|
||||
data_tok = create_access_token({"sub": "user@x.com"}, scope="data")
|
||||
r = stock_client.post("/monitor/stats/reset", headers=_bearer(data_tok))
|
||||
assert r.status_code == 403, f"data-scope reached admin action: {r.status_code}"
|
||||
|
||||
def test_data_scope_cannot_kill_browser(self, stock_client):
|
||||
data_tok = create_access_token({"sub": "user@x.com"}, scope="data")
|
||||
r = stock_client.post(
|
||||
"/monitor/actions/kill_browser", json={"sig": "abc"}, headers=_bearer(data_tok)
|
||||
)
|
||||
assert r.status_code == 403
|
||||
|
||||
def test_admin_scope_passes_require_admin(self, stock_client):
|
||||
admin_tok = create_access_token({"sub": "ops@x.com"}, scope="admin")
|
||||
r = stock_client.post("/monitor/stats/reset", headers=_bearer(admin_tok))
|
||||
# Past require_admin: not a 401/403. (500 acceptable here: no monitor
|
||||
# singleton without a lifespan — the point is authz let it through.)
|
||||
assert r.status_code not in (401, 403), f"admin blocked: {r.status_code}"
|
||||
|
||||
def test_unauthenticated_admin_action_is_401(self, stock_client):
|
||||
r = stock_client.post("/monitor/stats/reset")
|
||||
assert r.status_code == 401
|
||||
|
||||
|
||||
# ────────────────────────── MCP no-laundering ──────────────────────────
|
||||
class TestMcpNoLaundering:
|
||||
def test_mcp_proxy_attaches_service_credential(self):
|
||||
"""The internal loopback proxy must carry a valid, data-scope token."""
|
||||
import mcp_bridge
|
||||
from auth import decode_token
|
||||
|
||||
headers = mcp_bridge._service_auth_headers()
|
||||
assert "Authorization" in headers
|
||||
scheme, _, token = headers["Authorization"].partition(" ")
|
||||
assert scheme == "Bearer" and token
|
||||
claims = decode_token(token)
|
||||
assert claims["scope"] == "data", "MCP service token must not be admin-scoped"
|
||||
|
||||
def test_mcp_base_url_is_loopback(self, server_module):
|
||||
"""The MCP proxy must target loopback, never the 0.0.0.0 bind address."""
|
||||
import inspect
|
||||
src = inspect.getsource(server_module)
|
||||
assert "http://127.0.0.1:" in src
|
||||
assert 'base_url=f"http://{config' not in src
|
||||
|
||||
|
||||
# ─────────────────────────── job ownership ─────────────────────────────
|
||||
class _FakeRedis:
|
||||
"""Minimal async Redis stub holding one task hash."""
|
||||
|
||||
def __init__(self, task):
|
||||
# Real redis returns bytes keys/values; decode_redis_hash expects that.
|
||||
self._task = {k.encode(): str(v).encode() for k, v in task.items()}
|
||||
|
||||
async def hgetall(self, key):
|
||||
return dict(self._task)
|
||||
|
||||
async def delete(self, key):
|
||||
self._task = {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestJobOwnership:
|
||||
async def _status(self, task, **kw):
|
||||
import api
|
||||
redis = _FakeRedis(task)
|
||||
return await api.handle_task_status(
|
||||
redis, "crawl_abc", base_url="http://t/", keep=True, **kw
|
||||
)
|
||||
|
||||
async def test_owner_can_read_own_task(self):
|
||||
task = {"status": "completed", "created_at": "2026-01-01T00:00:00",
|
||||
"url": "[]", "result": "{}", "owner": "alice@x.com"}
|
||||
resp = await self._status(task, requester="alice@x.com", is_admin=False)
|
||||
assert resp.status_code == 200
|
||||
|
||||
async def test_other_requester_gets_404_not_403(self):
|
||||
from fastapi import HTTPException
|
||||
task = {"status": "completed", "created_at": "2026-01-01T00:00:00",
|
||||
"url": "[]", "result": "{}", "owner": "alice@x.com"}
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
await self._status(task, requester="mallory@x.com", is_admin=False)
|
||||
assert exc.value.status_code == 404 # not 403: don't reveal existence
|
||||
|
||||
async def test_admin_can_read_any_task(self):
|
||||
task = {"status": "completed", "created_at": "2026-01-01T00:00:00",
|
||||
"url": "[]", "result": "{}", "owner": "alice@x.com"}
|
||||
resp = await self._status(task, requester="ops@x.com", is_admin=True)
|
||||
assert resp.status_code == 200
|
||||
@@ -0,0 +1,124 @@
|
||||
"""
|
||||
R7 container/deployment posture - static parse of the deployment artifacts.
|
||||
|
||||
These assert the hardening is declared in the Dockerfile / docker-compose.yml /
|
||||
supervisord.conf. The RUNTIME acceptance gate (the hardened image builds,
|
||||
Chromium starts without --no-sandbox under userns/seccomp, the read-only rootfs
|
||||
+ tmpfs work, redis truly requires auth) needs an actual `docker build` + boot
|
||||
and is tracked as build-gated; it is out of scope for this offline suite.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.posture
|
||||
|
||||
REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
|
||||
DOCKER_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
|
||||
|
||||
|
||||
def _read(path):
|
||||
with open(path) as f:
|
||||
return f.read()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def dockerfile():
|
||||
return _read(os.path.join(REPO_ROOT, "Dockerfile"))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def compose():
|
||||
return _read(os.path.join(REPO_ROOT, "docker-compose.yml"))
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def supervisord():
|
||||
return _read(os.path.join(DOCKER_DIR, "supervisord.conf"))
|
||||
|
||||
|
||||
class TestDockerfile:
|
||||
def test_no_redis_expose(self, dockerfile):
|
||||
# No active EXPOSE 6379 line (commented references are fine).
|
||||
for line in dockerfile.splitlines():
|
||||
stripped = line.strip()
|
||||
if stripped.startswith("#"):
|
||||
continue
|
||||
assert not re.match(r"EXPOSE\s+.*\b6379\b", stripped), \
|
||||
"redis port 6379 must not be EXPOSEd"
|
||||
|
||||
def test_app_dir_root_owned_readonly(self, dockerfile):
|
||||
assert "chown -R root:root ${APP_HOME}" in dockerfile
|
||||
assert "chmod -R a-w ${APP_HOME}" in dockerfile
|
||||
|
||||
def test_artifact_dir_created_0700(self, dockerfile):
|
||||
assert "/var/lib/crawl4ai/outputs" in dockerfile
|
||||
assert "chmod 700 /var/lib/crawl4ai/outputs" in dockerfile
|
||||
|
||||
def test_runs_as_non_root(self, dockerfile):
|
||||
assert re.search(r"^USER\s+appuser", dockerfile, re.MULTILINE)
|
||||
|
||||
|
||||
class TestSupervisord:
|
||||
def test_redis_requires_password(self, supervisord):
|
||||
assert "--requirepass" in supervisord
|
||||
|
||||
def test_redis_bound_loopback(self, supervisord):
|
||||
assert "--bind 127.0.0.1" in supervisord
|
||||
|
||||
def test_gunicorn_bind_is_env_driven(self, supervisord):
|
||||
# entrypoint.sh resolves GUNICORN_BIND (loopback unless a credential).
|
||||
assert "ENV_GUNICORN_BIND" in supervisord
|
||||
|
||||
def test_gunicorn_request_limits(self, supervisord):
|
||||
assert "--limit-request-line" in supervisord
|
||||
|
||||
|
||||
class TestCompose:
|
||||
def test_cap_drop_all(self, compose):
|
||||
assert "cap_drop" in compose and "ALL" in compose
|
||||
|
||||
def test_no_new_privileges(self, compose):
|
||||
assert "no-new-privileges:true" in compose
|
||||
|
||||
def test_read_only_rootfs(self, compose):
|
||||
assert re.search(r"read_only:\s*true", compose)
|
||||
|
||||
def test_no_host_dev_shm_bind(self, compose):
|
||||
assert "/dev/shm:/dev/shm" not in compose
|
||||
assert "shm_size" in compose
|
||||
|
||||
def test_pids_limit(self, compose):
|
||||
assert "pids_limit" in compose
|
||||
|
||||
def test_read_only_runtime_tmpfs_are_appuser_owned(self, compose):
|
||||
assert "/var/lib/redis:uid=999,gid=999,mode=0700" in compose
|
||||
assert "/var/lib/crawl4ai/outputs:uid=999,gid=999,mode=0700" in compose
|
||||
assert "/home/appuser/.crawl4ai:uid=999,gid=999,mode=0700" in compose
|
||||
assert "/home/appuser/.gunicorn:uid=999,gid=999,mode=0700" in compose
|
||||
|
||||
def test_playwright_cache_is_not_shadowed(self, compose):
|
||||
assert "/home/appuser/.cache\n" not in compose
|
||||
assert "/home/appuser/.cache/url_seeder:uid=999,gid=999,mode=0700" in compose
|
||||
|
||||
|
||||
class TestEntrypoint:
|
||||
def test_entrypoint_exists_and_resolves_bind(self):
|
||||
src = _read(os.path.join(DOCKER_DIR, "entrypoint.sh"))
|
||||
assert "GUNICORN_BIND" in src and "REDIS_PASSWORD" in src
|
||||
assert "127.0.0.1" in src # loopback default when no credential
|
||||
|
||||
|
||||
class TestSandboxOptOut:
|
||||
def test_default_keeps_no_sandbox(self, server_module, monkeypatch):
|
||||
monkeypatch.setattr(server_module, "CHROMIUM_SANDBOX", False)
|
||||
assert "--no-sandbox" in server_module._browser_extra_args()
|
||||
|
||||
def test_opt_in_drops_no_sandbox(self, server_module, monkeypatch):
|
||||
# CRAWL4AI_CHROMIUM_SANDBOX=true -> run the renderer sandboxed.
|
||||
monkeypatch.setattr(server_module, "CHROMIUM_SANDBOX", True)
|
||||
assert "--no-sandbox" not in server_module._browser_extra_args()
|
||||
# other flags are preserved
|
||||
assert "--disable-dev-shm-usage" in server_module._browser_extra_args()
|
||||
@@ -0,0 +1,254 @@
|
||||
"""
|
||||
HEADLINE SECURITY POSTURE GATE — test_security_default_posture
|
||||
|
||||
This is the acceptance gate for the Docker-server security hardening
|
||||
(root causes R1-R7). It boots the server *with the shipped config.yml* and
|
||||
asserts the secure-by-default end state:
|
||||
|
||||
- every mutating + read endpoint, static mount, and MCP transport requires
|
||||
auth out of the box (R1); only the health check and UI shell pages
|
||||
(dashboard/playground) are public,
|
||||
- the token endpoint will not freely mint credentials when no api_token /
|
||||
secret is configured (R1),
|
||||
- strong security headers + a strict CSP are present on every response even
|
||||
when `security.enabled` is false (R6),
|
||||
- the effective browser launch flags carry no `--no-sandbox` and no
|
||||
TLS-verification-disabling flags (R3/R7),
|
||||
- Redis requires a password and security is on by default (R7),
|
||||
- dangerous request->code features are off by default (R2).
|
||||
|
||||
IT IS EXPECTED TO BE RED TODAY. Each failing assertion names one gap the
|
||||
hardening must close. As R1-R7 land, methods flip green; when the whole class
|
||||
is green, the default Docker deploy is safe and CI can gate on it.
|
||||
|
||||
Run: pytest deploy/docker/tests/test_security_default_posture.py -v
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from redis_config import build_rate_limit_storage_uri, build_redis_url
|
||||
|
||||
pytestmark = pytest.mark.posture
|
||||
|
||||
HEALTH = "/health"
|
||||
|
||||
# (method, path, json-body-or-None) for every endpoint that must NOT be
|
||||
# reachable without authentication. Bodies are intentionally minimal/invalid so
|
||||
# that in the *current* (open) state the request fails fast at validation
|
||||
# rather than launching a real crawl — the assertion (== 401) still fails,
|
||||
# which is the RED signal we want, with no browser/network side effects.
|
||||
PROTECTED_ENDPOINTS = [
|
||||
# mutating / action endpoints
|
||||
("post", "/crawl", {}),
|
||||
("post", "/crawl/stream", {}),
|
||||
("post", "/md", {}),
|
||||
("post", "/html", {}),
|
||||
("post", "/screenshot", {}),
|
||||
("post", "/pdf", {}),
|
||||
("post", "/execute_js", {}),
|
||||
("post", "/config/dump", {}),
|
||||
("post", "/crawl/job", {}),
|
||||
("post", "/llm/job", {}),
|
||||
# monitor mutating actions (must additionally require *admin* scope; here we
|
||||
# only assert they are not anonymously reachable). Pools are empty without a
|
||||
# lifespan, so probing these performs no destructive action.
|
||||
("post", "/monitor/actions/cleanup", {}),
|
||||
("post", "/monitor/actions/kill_browser", {}),
|
||||
("post", "/monitor/actions/restart_browser", {}),
|
||||
("post", "/monitor/stats/reset", {}),
|
||||
# read / info endpoints that currently have NO auth dependency at all
|
||||
("get", "/schema", None),
|
||||
("get", "/hooks/info", None),
|
||||
("get", "/ask", None),
|
||||
("get", "/metrics", None),
|
||||
("get", "/llm/example.com", None), # no `q` -> fast 400 today, must be 401
|
||||
("get", "/monitor/health", None),
|
||||
("get", "/monitor/requests", None),
|
||||
("get", "/monitor/browsers", None),
|
||||
("get", "/monitor/endpoints/stats", None),
|
||||
("get", "/monitor/timeline", None),
|
||||
# MCP transport (must be gated; today it is open AND launders credentials)
|
||||
("get", "/mcp/schema", None),
|
||||
]
|
||||
|
||||
# UI shell pages that are intentionally public — they serve only static
|
||||
# HTML/CSS/JS and contain no data. All data routes behind them remain gated.
|
||||
PUBLIC_UI_PATHS = ["/dashboard/", "/playground/"]
|
||||
|
||||
|
||||
def _call(client, method, path, body):
|
||||
fn = getattr(client, method)
|
||||
if body is not None:
|
||||
return fn(path, json=body)
|
||||
return fn(path)
|
||||
|
||||
|
||||
class TestDefaultDeployIsSafe:
|
||||
"""The shipped Docker image must be secure with an empty/default config."""
|
||||
|
||||
# ───────────────────────── R1: authentication ─────────────────────────
|
||||
|
||||
def test_health_check_is_public(self, stock_client):
|
||||
"""The health endpoint is the one intentionally public route."""
|
||||
r = stock_client.get(HEALTH)
|
||||
assert r.status_code == 200
|
||||
|
||||
@pytest.mark.parametrize("path", PUBLIC_UI_PATHS, ids=PUBLIC_UI_PATHS)
|
||||
def test_ui_shell_is_public(self, stock_client, path):
|
||||
"""UI shells serve static HTML only — no data — so they load without auth."""
|
||||
r = stock_client.get(path)
|
||||
assert r.status_code == 200, (
|
||||
f"GET {path} returned {r.status_code}; expected 200. "
|
||||
f"UI shell pages must be publicly accessible."
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"method,path,body",
|
||||
PROTECTED_ENDPOINTS,
|
||||
ids=[f"{m.upper()} {p}" for m, p, _ in PROTECTED_ENDPOINTS],
|
||||
)
|
||||
def test_endpoint_requires_auth_by_default(self, stock_client, method, path, body):
|
||||
"""Without a credential, every non-health route must refuse (401)."""
|
||||
r = _call(stock_client, method, path, body)
|
||||
assert r.status_code == 401, (
|
||||
f"{method.upper()} {path} returned {r.status_code} unauthenticated; "
|
||||
f"expected 401. Endpoint is reachable without a credential."
|
||||
)
|
||||
|
||||
def test_token_endpoint_does_not_freely_mint(self, stock_client, server_module, monkeypatch):
|
||||
"""With no api_token configured, /token must not hand a JWT to anyone.
|
||||
|
||||
Force the email-domain MX check to pass so the test exercises the real
|
||||
vulnerability (anonymous token minting) rather than passing for the
|
||||
wrong reason when offline.
|
||||
"""
|
||||
monkeypatch.setattr(server_module, "verify_email_domain", lambda email: True)
|
||||
r = stock_client.post("/token", json={"email": "attacker@example.com"})
|
||||
if r.status_code == 200:
|
||||
assert "access_token" not in r.json(), (
|
||||
"/token minted a credential for an anonymous caller with no "
|
||||
"api_token configured — anyone can self-issue a valid JWT."
|
||||
)
|
||||
|
||||
# ─────────────────────── R6: headers / CSP / framing ──────────────────
|
||||
|
||||
def test_security_headers_present_by_default(self, stock_client):
|
||||
"""Baseline security headers must be emitted even when security.enabled is false."""
|
||||
r = stock_client.get(HEALTH)
|
||||
h = {k.lower(): v for k, v in r.headers.items()}
|
||||
assert h.get("x-content-type-options") == "nosniff"
|
||||
assert h.get("x-frame-options", "").upper() == "DENY"
|
||||
assert "content-security-policy" in h, "no CSP header on a default response"
|
||||
|
||||
def test_csp_is_strict(self, stock_client):
|
||||
"""CSP must lock script execution to self and forbid inline/unsafe + framing."""
|
||||
r = stock_client.get(HEALTH)
|
||||
csp = r.headers.get("content-security-policy", "")
|
||||
assert "script-src 'self'" in csp, f"CSP missing strict script-src: {csp!r}"
|
||||
assert "frame-ancestors 'none'" in csp, f"CSP allows framing: {csp!r}"
|
||||
assert "unsafe-inline" not in csp, f"CSP permits unsafe-inline: {csp!r}"
|
||||
|
||||
# ─────────────────── R3 / R7: browser & TLS launch flags ───────────────
|
||||
|
||||
@pytest.mark.xfail(
|
||||
reason="build-gated: --no-sandbox stays until the container runs with an "
|
||||
"unprivileged userns or a verified seccomp profile and a docker build "
|
||||
"confirms Chromium still starts. Tracked, not yet done.",
|
||||
strict=False,
|
||||
)
|
||||
def test_no_no_sandbox_flag(self, effective_browser_args):
|
||||
"""Chromium must not run with --no-sandbox by default (renderer escape)."""
|
||||
assert "--no-sandbox" not in effective_browser_args, (
|
||||
"default config launches Chromium with --no-sandbox"
|
||||
)
|
||||
|
||||
def test_tls_verification_not_disabled(self, effective_browser_args):
|
||||
"""TLS verification must not be disabled by default."""
|
||||
assert "--ignore-certificate-errors" not in effective_browser_args
|
||||
assert "--allow-insecure-localhost" not in effective_browser_args
|
||||
|
||||
# ───────────────────────── R7: deploy posture ─────────────────────────
|
||||
|
||||
def test_security_enabled_by_default(self, effective_config):
|
||||
"""The security middleware block must be on out of the box."""
|
||||
assert effective_config["security"]["enabled"] is True
|
||||
|
||||
def test_redis_is_not_network_exposed(self, effective_redis_url, effective_config):
|
||||
"""Redis must not be open on the network: loopback host or a password.
|
||||
|
||||
The container runs redis loopback-only with --requirepass (supervisord)
|
||||
and no EXPOSE (asserted in the container-posture suite); a password also
|
||||
satisfies this for an external redis.
|
||||
"""
|
||||
rc = effective_config.get("redis", {})
|
||||
host = str(rc.get("host", "localhost")).lower()
|
||||
pw = rc.get("password", "") or ""
|
||||
loopback = host in ("localhost", "127.0.0.1", "::1")
|
||||
assert loopback or pw, (
|
||||
"redis is neither loopback-bound nor password-protected"
|
||||
)
|
||||
|
||||
def test_redis_url_includes_acl_username_and_encoded_password(self, monkeypatch):
|
||||
"""Redis URLs must authenticate during HELLO when Redis is password-protected."""
|
||||
monkeypatch.setenv("REDIS_PASSWORD", "p@ss/w:rd")
|
||||
cfg = {
|
||||
"redis": {
|
||||
"host": "localhost",
|
||||
"port": 6379,
|
||||
"db": 0,
|
||||
}
|
||||
}
|
||||
url = build_redis_url(cfg)
|
||||
assert url.startswith("redis://default:")
|
||||
assert "p%40ss%2Fw%3Ard" in url
|
||||
assert url.endswith("@localhost:6379/0")
|
||||
|
||||
def test_rate_limit_redis_storage_reuses_redis_password(self, monkeypatch):
|
||||
"""SlowAPI must not connect to protected Redis without credentials."""
|
||||
monkeypatch.setenv("REDIS_PASSWORD", "abc123")
|
||||
cfg = {
|
||||
"redis": {"host": "localhost", "port": 6379, "db": 0},
|
||||
"rate_limiting": {"storage_uri": "redis://localhost:6379/0"},
|
||||
}
|
||||
url = build_rate_limit_storage_uri(cfg)
|
||||
assert url.startswith("redis://default:")
|
||||
assert "abc123" in url
|
||||
assert url.endswith("@localhost:6379/0")
|
||||
|
||||
def test_rate_limit_storage_preserves_explicit_auth(self, monkeypatch):
|
||||
"""Do not rewrite operator-provided rate-limit Redis credentials."""
|
||||
monkeypatch.setenv("REDIS_PASSWORD", "container-secret")
|
||||
storage_uri = "redis://" + "alice" + ":" + "custom" + "@redis:6379/2"
|
||||
cfg = {
|
||||
"redis": {"host": "localhost", "port": 6379, "db": 0},
|
||||
"rate_limiting": {"storage_uri": storage_uri},
|
||||
}
|
||||
assert build_rate_limit_storage_uri(cfg) == storage_uri
|
||||
|
||||
def test_rate_limit_storage_keeps_memory_backend(self, monkeypatch):
|
||||
"""Non-Redis rate-limit backends must not be rewritten."""
|
||||
monkeypatch.setenv("REDIS_PASSWORD", "abc123")
|
||||
cfg = {
|
||||
"redis": {"host": "localhost", "port": 6379, "db": 0},
|
||||
"rate_limiting": {"storage_uri": "memory://"},
|
||||
}
|
||||
assert build_rate_limit_storage_uri(cfg) == "memory://"
|
||||
|
||||
def test_trusted_hosts_not_wildcard_when_exposed(self, effective_config):
|
||||
"""A wildcard trusted_hosts on a non-loopback bind silently disables the host guard."""
|
||||
host = effective_config["app"]["host"]
|
||||
trusted = effective_config["security"]["trusted_hosts"]
|
||||
exposed = host not in ("127.0.0.1", "localhost", "::1")
|
||||
assert not (exposed and trusted == ["*"]), (
|
||||
f"app binds {host} but trusted_hosts is {trusted} (host guard disabled)"
|
||||
)
|
||||
|
||||
# ───────────────────── R2: dangerous features off by default ───────────
|
||||
|
||||
def test_hooks_disabled_by_default(self, server_module):
|
||||
"""User-supplied hook code (exec path) must be off unless explicitly enabled."""
|
||||
assert server_module.HOOKS_ENABLED is False
|
||||
|
||||
def test_execute_js_disabled_by_default(self, server_module):
|
||||
"""Arbitrary JS execution endpoint must be off unless explicitly enabled."""
|
||||
assert server_module.EXECUTE_JS_ENABLED is False
|
||||
@@ -0,0 +1,100 @@
|
||||
"""Path-traversal regression tests for crawler file downloads.
|
||||
|
||||
Y4tacker reported an arbitrary-file-write -> RCE: a download filename taken
|
||||
from an attacker-controlled Content-Disposition header (HTTP strategy) or the
|
||||
browser's suggested filename (Playwright strategy) was joined to the downloads
|
||||
dir with no confinement, so an absolute path or ``../`` sequence escaped it.
|
||||
|
||||
Both call sites now route through ``_safe_download_filepath``, which reduces
|
||||
the name to a bare basename and re-checks containment. These tests exercise
|
||||
that helper directly (offline, no network/browser).
|
||||
"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
|
||||
from crawl4ai.async_crawler_strategy import _safe_download_filepath, _nofollow_opener
|
||||
|
||||
pytestmark = pytest.mark.posture
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def downloads_dir():
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
yield d
|
||||
|
||||
|
||||
class TestDownloadPathConfinement:
|
||||
def test_plain_filename_kept(self, downloads_dir):
|
||||
p = _safe_download_filepath(downloads_dir, "report.pdf")
|
||||
assert p == os.path.join(os.path.realpath(downloads_dir), "report.pdf")
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"evil",
|
||||
[
|
||||
"/etc/cron.d/evil",
|
||||
"/tmp/pwned_absolute.sh",
|
||||
"../../../.bashrc",
|
||||
"../../.ssh/authorized_keys",
|
||||
"../../../../etc/passwd",
|
||||
"..",
|
||||
".",
|
||||
"",
|
||||
"foo/../../bar",
|
||||
],
|
||||
)
|
||||
def test_escape_attempts_confined(self, downloads_dir, evil):
|
||||
"""No input may produce a path outside the downloads root."""
|
||||
root = os.path.realpath(downloads_dir)
|
||||
p = _safe_download_filepath(downloads_dir, evil)
|
||||
assert os.path.commonpath([root, p]) == root
|
||||
# The resolved file sits directly inside the root (basename only).
|
||||
assert os.path.dirname(p) == root
|
||||
|
||||
def test_absolute_path_does_not_win(self, downloads_dir):
|
||||
p = _safe_download_filepath(downloads_dir, "/etc/cron.d/evil")
|
||||
assert not p.startswith("/etc/")
|
||||
assert p.startswith(os.path.realpath(downloads_dir))
|
||||
|
||||
def test_symlink_escape_rejected(self, downloads_dir):
|
||||
"""If the basename is an existing symlink in the root pointing out,
|
||||
realpath resolves outside the root and the write must be rejected."""
|
||||
root = os.path.realpath(downloads_dir)
|
||||
outside_dir = tempfile.mkdtemp()
|
||||
outside_file = os.path.join(outside_dir, "target")
|
||||
link = os.path.join(root, "evil")
|
||||
os.symlink(outside_file, link)
|
||||
# filename "evil" -> root/evil is a symlink to outside -> rejected.
|
||||
with pytest.raises(ValueError):
|
||||
_safe_download_filepath(downloads_dir, "evil")
|
||||
|
||||
|
||||
class TestNoFollowOpener:
|
||||
"""The write must refuse to follow a symlink swapped in after path
|
||||
confinement (the TOCTOU window between realpath-check and open)."""
|
||||
|
||||
def test_open_refuses_symlink_target(self, downloads_dir):
|
||||
import os
|
||||
outside = tempfile.mkdtemp()
|
||||
outside_file = os.path.join(outside, "secret")
|
||||
with open(outside_file, "w") as f:
|
||||
f.write("original")
|
||||
# Attacker swaps the confined target for a symlink pointing outside.
|
||||
target = os.path.join(downloads_dir, "report.pdf")
|
||||
os.symlink(outside_file, target)
|
||||
with pytest.raises(OSError): # ELOOP from O_NOFOLLOW
|
||||
with open(target, "wb", opener=_nofollow_opener) as f:
|
||||
f.write(b"pwned")
|
||||
# The outside file was not overwritten through the symlink.
|
||||
with open(outside_file) as f:
|
||||
assert f.read() == "original"
|
||||
|
||||
def test_open_writes_normal_file(self, downloads_dir):
|
||||
import os
|
||||
target = os.path.join(downloads_dir, "ok.bin")
|
||||
with open(target, "wb", opener=_nofollow_opener) as f:
|
||||
f.write(b"data")
|
||||
with open(target, "rb") as f:
|
||||
assert f.read() == b"data"
|
||||
@@ -0,0 +1,132 @@
|
||||
"""
|
||||
R3 browser egress-proxy tests (real loopback sockets, fully offline).
|
||||
|
||||
The pinning proxy is what actually stops DNS rebinding on the browser path:
|
||||
Chromium is pointed at it, so it asks the proxy to CONNECT host:port; the proxy
|
||||
resolves-and-pins (egress_broker.resolve_and_pin) and dials only the pinned,
|
||||
global IP. We drive it with a raw asyncio client + a fake upstream, and stub
|
||||
resolve_and_pin so a "public" host pins to the loopback upstream while an
|
||||
"internal" host is refused. (The not-is_global rule itself is covered in
|
||||
test_security_ssrf_egress.py.)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
import egress_proxy
|
||||
from egress_broker import EgressBlocked, PinnedTarget
|
||||
from egress_proxy import PinningProxy
|
||||
|
||||
pytestmark = pytest.mark.posture
|
||||
|
||||
|
||||
async def _fake_upstream():
|
||||
async def handle(reader, writer):
|
||||
await reader.read(65536)
|
||||
writer.write(b"UPSTREAM-OK")
|
||||
await writer.drain()
|
||||
writer.close()
|
||||
server = await asyncio.start_server(handle, "127.0.0.1", 0)
|
||||
return server, server.sockets[0].getsockname()[1]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestPinningProxy:
|
||||
async def test_connect_to_global_host_tunnels(self, monkeypatch):
|
||||
up, up_port = await _fake_upstream()
|
||||
|
||||
# Pin "good.example" to the loopback upstream (stand-in for a global IP).
|
||||
def fake_pin(url):
|
||||
return PinnedTarget("https", "good.example", up_port, "127.0.0.1")
|
||||
monkeypatch.setattr(egress_proxy, "resolve_and_pin", fake_pin)
|
||||
|
||||
proxy = PinningProxy()
|
||||
await proxy.start()
|
||||
try:
|
||||
r, w = await asyncio.open_connection(proxy.bound_host, proxy.bound_port)
|
||||
w.write(f"CONNECT good.example:{up_port} HTTP/1.1\r\n\r\n".encode())
|
||||
await w.drain()
|
||||
status = await asyncio.wait_for(r.readline(), timeout=5)
|
||||
assert b"200" in status
|
||||
await r.readline() # blank line after the 200
|
||||
w.write(b"hello")
|
||||
await w.drain()
|
||||
body = await asyncio.wait_for(r.read(100), timeout=5)
|
||||
assert b"UPSTREAM-OK" in body
|
||||
w.close()
|
||||
finally:
|
||||
await proxy.stop()
|
||||
up.close()
|
||||
|
||||
async def test_connect_to_internal_host_blocked(self, monkeypatch):
|
||||
def fake_pin(url):
|
||||
raise EgressBlocked()
|
||||
monkeypatch.setattr(egress_proxy, "resolve_and_pin", fake_pin)
|
||||
|
||||
proxy = PinningProxy()
|
||||
await proxy.start()
|
||||
try:
|
||||
r, w = await asyncio.open_connection(proxy.bound_host, proxy.bound_port)
|
||||
w.write(b"CONNECT evil.example:443 HTTP/1.1\r\n\r\n")
|
||||
await w.drain()
|
||||
status = await asyncio.wait_for(r.readline(), timeout=5)
|
||||
assert b"403" in status
|
||||
w.close()
|
||||
finally:
|
||||
await proxy.stop()
|
||||
|
||||
async def test_proxy_dials_pinned_ip_not_requested_host(self, monkeypatch):
|
||||
# resolve_and_pin returns a pinned ip distinct from the CONNECT host;
|
||||
# assert the proxy dials the pinned ip.
|
||||
dialed = {}
|
||||
up, up_port = await _fake_upstream()
|
||||
|
||||
def fake_pin(url):
|
||||
return PinnedTarget("https", "rebind.example", up_port, "127.0.0.1")
|
||||
monkeypatch.setattr(egress_proxy, "resolve_and_pin", fake_pin)
|
||||
|
||||
real_open = asyncio.open_connection
|
||||
|
||||
async def spy_open(host, port, *a, **k):
|
||||
dialed["host"], dialed["port"] = host, port
|
||||
return await real_open(host, port, *a, **k)
|
||||
# patch only the name the proxy module uses
|
||||
monkeypatch.setattr(egress_proxy.asyncio, "open_connection", spy_open)
|
||||
|
||||
proxy = PinningProxy()
|
||||
await proxy.start()
|
||||
try:
|
||||
r, w = await real_open(proxy.bound_host, proxy.bound_port)
|
||||
w.write(f"CONNECT rebind.example:{up_port} HTTP/1.1\r\n\r\n".encode())
|
||||
await w.drain()
|
||||
await asyncio.wait_for(r.readline(), timeout=5)
|
||||
assert dialed.get("host") == "127.0.0.1" # the pinned ip
|
||||
w.close()
|
||||
finally:
|
||||
await proxy.stop()
|
||||
up.close()
|
||||
|
||||
async def test_malformed_connect_400(self):
|
||||
proxy = PinningProxy()
|
||||
await proxy.start()
|
||||
try:
|
||||
r, w = await asyncio.open_connection(proxy.bound_host, proxy.bound_port)
|
||||
w.write(b"CONNECT not-a-host-port HTTP/1.1\r\n\r\n")
|
||||
await w.drain()
|
||||
status = await asyncio.wait_for(r.readline(), timeout=5)
|
||||
assert b"400" in status
|
||||
w.close()
|
||||
finally:
|
||||
await proxy.stop()
|
||||
|
||||
|
||||
class TestEnforceEgressWiring:
|
||||
def test_enforce_egress_sets_proxy(self, monkeypatch):
|
||||
import egress_broker
|
||||
from crawl4ai import BrowserConfig
|
||||
monkeypatch.setattr(egress_broker, "_EGRESS_PROXY_URL", "http://127.0.0.1:9999")
|
||||
b = BrowserConfig()
|
||||
egress_broker.enforce_egress(b)
|
||||
assert b.proxy_config is not None
|
||||
assert b.proxy_config.server == "http://127.0.0.1:9999"
|
||||
@@ -0,0 +1,274 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Unit tests for security fixes.
|
||||
These tests verify the security fixes at the code level without needing a running server.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Add parent directory to path to import modules
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
import unittest
|
||||
|
||||
|
||||
class TestURLValidation(unittest.TestCase):
|
||||
"""Test URL scheme validation helper."""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test fixtures."""
|
||||
self.ALLOWED_URL_SCHEMES = ("http://", "https://")
|
||||
self.ALLOWED_URL_SCHEMES_WITH_RAW = ("http://", "https://", "raw:", "raw://")
|
||||
|
||||
def validate_url_scheme(self, url: str, allow_raw: bool = False) -> bool:
|
||||
"""Local version of validate_url_scheme for testing."""
|
||||
allowed = self.ALLOWED_URL_SCHEMES_WITH_RAW if allow_raw else self.ALLOWED_URL_SCHEMES
|
||||
return url.startswith(allowed)
|
||||
|
||||
# === SECURITY TESTS: These URLs must be BLOCKED ===
|
||||
|
||||
def test_file_url_blocked(self):
|
||||
"""file:// URLs must be blocked (LFI vulnerability)."""
|
||||
self.assertFalse(self.validate_url_scheme("file:///etc/passwd"))
|
||||
self.assertFalse(self.validate_url_scheme("file:///etc/passwd", allow_raw=True))
|
||||
|
||||
def test_file_url_blocked_windows(self):
|
||||
"""file:// URLs with Windows paths must be blocked."""
|
||||
self.assertFalse(self.validate_url_scheme("file:///C:/Windows/System32/config/sam"))
|
||||
|
||||
def test_javascript_url_blocked(self):
|
||||
"""javascript: URLs must be blocked (XSS)."""
|
||||
self.assertFalse(self.validate_url_scheme("javascript:alert(1)"))
|
||||
|
||||
def test_data_url_blocked(self):
|
||||
"""data: URLs must be blocked."""
|
||||
self.assertFalse(self.validate_url_scheme("data:text/html,<script>alert(1)</script>"))
|
||||
|
||||
def test_ftp_url_blocked(self):
|
||||
"""ftp: URLs must be blocked."""
|
||||
self.assertFalse(self.validate_url_scheme("ftp://example.com/file"))
|
||||
|
||||
def test_empty_url_blocked(self):
|
||||
"""Empty URLs must be blocked."""
|
||||
self.assertFalse(self.validate_url_scheme(""))
|
||||
|
||||
def test_relative_url_blocked(self):
|
||||
"""Relative URLs must be blocked."""
|
||||
self.assertFalse(self.validate_url_scheme("/etc/passwd"))
|
||||
self.assertFalse(self.validate_url_scheme("../../../etc/passwd"))
|
||||
|
||||
# === FUNCTIONALITY TESTS: These URLs must be ALLOWED ===
|
||||
|
||||
def test_http_url_allowed(self):
|
||||
"""http:// URLs must be allowed."""
|
||||
self.assertTrue(self.validate_url_scheme("http://example.com"))
|
||||
self.assertTrue(self.validate_url_scheme("http://localhost:8080"))
|
||||
|
||||
def test_https_url_allowed(self):
|
||||
"""https:// URLs must be allowed."""
|
||||
self.assertTrue(self.validate_url_scheme("https://example.com"))
|
||||
self.assertTrue(self.validate_url_scheme("https://example.com/path?query=1"))
|
||||
|
||||
def test_raw_url_allowed_when_enabled(self):
|
||||
"""raw: URLs must be allowed when allow_raw=True."""
|
||||
self.assertTrue(self.validate_url_scheme("raw:<html></html>", allow_raw=True))
|
||||
self.assertTrue(self.validate_url_scheme("raw://<html></html>", allow_raw=True))
|
||||
|
||||
def test_raw_url_blocked_when_disabled(self):
|
||||
"""raw: URLs must be blocked when allow_raw=False."""
|
||||
self.assertFalse(self.validate_url_scheme("raw:<html></html>", allow_raw=False))
|
||||
self.assertFalse(self.validate_url_scheme("raw://<html></html>", allow_raw=False))
|
||||
|
||||
|
||||
class TestHookBuiltins(unittest.TestCase):
|
||||
"""Hooks are declarative now - there is no builtins sandbox to harden.
|
||||
|
||||
The exec()-based hook system was removed (no in-process sandbox survives
|
||||
attacker Python). Hooks are a fixed registry of server-authored actions, so
|
||||
the relevant guarantee is simply that no action runs arbitrary code.
|
||||
"""
|
||||
|
||||
def test_registry_actions_are_fixed_and_codeless(self):
|
||||
from hook_registry import HOOK_REGISTRY
|
||||
# No action accepts/executes raw code.
|
||||
for action in HOOK_REGISTRY:
|
||||
self.assertNotIn("code", action.lower())
|
||||
self.assertNotIn("eval", action.lower())
|
||||
self.assertNotIn("exec", action.lower())
|
||||
# The documented safe actions are present.
|
||||
self.assertTrue({"block_resources", "add_cookies", "set_headers",
|
||||
"scroll_to_bottom", "wait_for_timeout"}.issubset(HOOK_REGISTRY))
|
||||
|
||||
|
||||
class TestHooksEnabled(unittest.TestCase):
|
||||
"""Test HOOKS_ENABLED environment variable logic."""
|
||||
|
||||
def test_hooks_disabled_by_default(self):
|
||||
"""Hooks must be disabled by default."""
|
||||
original = os.environ.pop("CRAWL4AI_HOOKS_ENABLED", None)
|
||||
try:
|
||||
hooks_enabled = os.environ.get("CRAWL4AI_HOOKS_ENABLED", "false").lower() == "true"
|
||||
self.assertFalse(hooks_enabled)
|
||||
finally:
|
||||
if original is not None:
|
||||
os.environ["CRAWL4AI_HOOKS_ENABLED"] = original
|
||||
|
||||
def test_hooks_enabled_when_true(self):
|
||||
"""Hooks must be enabled when CRAWL4AI_HOOKS_ENABLED=true."""
|
||||
original = os.environ.get("CRAWL4AI_HOOKS_ENABLED")
|
||||
try:
|
||||
os.environ["CRAWL4AI_HOOKS_ENABLED"] = "true"
|
||||
hooks_enabled = os.environ.get("CRAWL4AI_HOOKS_ENABLED", "false").lower() == "true"
|
||||
self.assertTrue(hooks_enabled)
|
||||
finally:
|
||||
if original is not None:
|
||||
os.environ["CRAWL4AI_HOOKS_ENABLED"] = original
|
||||
else:
|
||||
os.environ.pop("CRAWL4AI_HOOKS_ENABLED", None)
|
||||
|
||||
def test_hooks_disabled_when_false(self):
|
||||
"""Hooks must be disabled when CRAWL4AI_HOOKS_ENABLED=false."""
|
||||
original = os.environ.get("CRAWL4AI_HOOKS_ENABLED")
|
||||
try:
|
||||
os.environ["CRAWL4AI_HOOKS_ENABLED"] = "false"
|
||||
hooks_enabled = os.environ.get("CRAWL4AI_HOOKS_ENABLED", "false").lower() == "true"
|
||||
self.assertFalse(hooks_enabled)
|
||||
finally:
|
||||
if original is not None:
|
||||
os.environ["CRAWL4AI_HOOKS_ENABLED"] = original
|
||||
else:
|
||||
os.environ.pop("CRAWL4AI_HOOKS_ENABLED", None)
|
||||
|
||||
|
||||
class TestComputedFieldExpressionDisabled(unittest.TestCase):
|
||||
"""Test that computed field 'expression' key is completely disabled.
|
||||
|
||||
eval() on untrusted input is fundamentally unsafe - no sandbox survives.
|
||||
The expression path is now disabled; only 'function' key works.
|
||||
"""
|
||||
|
||||
def test_expression_returns_default(self):
|
||||
"""expression key must return default value, not evaluate."""
|
||||
import logging
|
||||
# Import the actual class
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
|
||||
|
||||
schema = {
|
||||
"baseSelector": "div",
|
||||
"fields": [
|
||||
{"name": "price", "selector": "span", "type": "text"},
|
||||
{
|
||||
"name": "computed",
|
||||
"type": "computed",
|
||||
"expression": "price * 2",
|
||||
"default": "DISABLED",
|
||||
},
|
||||
],
|
||||
}
|
||||
strategy = JsonCssExtractionStrategy(schema)
|
||||
result = strategy._compute_field({"price": 100}, schema["fields"][1])
|
||||
self.assertEqual(result, "DISABLED")
|
||||
|
||||
def test_expression_does_not_execute_code(self):
|
||||
"""expression must NEVER execute - even harmless code."""
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
|
||||
|
||||
schema = {
|
||||
"baseSelector": "div",
|
||||
"fields": [{"name": "x", "selector": "span", "type": "text"}],
|
||||
}
|
||||
strategy = JsonCssExtractionStrategy(schema)
|
||||
|
||||
# These should all return default, never execute
|
||||
dangerous_expressions = [
|
||||
"__import__('os').system('id')",
|
||||
"open('/etc/passwd').read()",
|
||||
"().__class__.__bases__[0].__subclasses__()",
|
||||
]
|
||||
for expr in dangerous_expressions:
|
||||
field = {"name": "test", "type": "computed", "expression": expr, "default": None}
|
||||
result = strategy._compute_field({}, field)
|
||||
self.assertIsNone(result, f"Expression should not execute: {expr}")
|
||||
|
||||
def test_function_key_still_works(self):
|
||||
"""function key with Python callable must still work."""
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
|
||||
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
|
||||
|
||||
schema = {
|
||||
"baseSelector": "div",
|
||||
"fields": [{"name": "price", "selector": "span", "type": "text"}],
|
||||
}
|
||||
strategy = JsonCssExtractionStrategy(schema)
|
||||
field = {
|
||||
"name": "computed",
|
||||
"type": "computed",
|
||||
"function": lambda item: item["price"] * 2,
|
||||
}
|
||||
result = strategy._compute_field({"price": 100}, field)
|
||||
self.assertEqual(result, 200)
|
||||
|
||||
|
||||
class TestDeserializationAllowlist(unittest.TestCase):
|
||||
"""Test that the deserialization allowlist blocks non-allowlisted types."""
|
||||
|
||||
def setUp(self):
|
||||
self.allowed_types = {
|
||||
"BrowserConfig", "CrawlerRunConfig", "HTTPCrawlerConfig",
|
||||
"LLMConfig", "ProxyConfig", "GeolocationConfig",
|
||||
"SeedingConfig", "VirtualScrollConfig", "LinkPreviewConfig",
|
||||
"JsonCssExtractionStrategy", "JsonXPathExtractionStrategy",
|
||||
"JsonLxmlExtractionStrategy", "LLMExtractionStrategy",
|
||||
"CosineStrategy", "RegexExtractionStrategy",
|
||||
"DefaultMarkdownGenerator",
|
||||
"PruningContentFilter", "BM25ContentFilter", "LLMContentFilter",
|
||||
"LXMLWebScrapingStrategy",
|
||||
"RegexChunking",
|
||||
"BFSDeepCrawlStrategy", "DFSDeepCrawlStrategy", "BestFirstCrawlingStrategy",
|
||||
"FilterChain", "URLPatternFilter", "DomainFilter",
|
||||
"ContentTypeFilter", "URLFilter", "SEOFilter", "ContentRelevanceFilter",
|
||||
"KeywordRelevanceScorer", "URLScorer", "CompositeScorer",
|
||||
"DomainAuthorityScorer", "FreshnessScorer", "PathDepthScorer",
|
||||
"CacheMode", "MatchMode", "DisplayMode",
|
||||
"MemoryAdaptiveDispatcher", "SemaphoreDispatcher",
|
||||
"DefaultTableExtraction", "NoTableExtraction", "LLMTableExtraction",
|
||||
"RoundRobinProxyStrategy",
|
||||
}
|
||||
|
||||
def test_arbitrary_class_not_in_allowlist(self):
|
||||
self.assertNotIn("AsyncWebCrawler", self.allowed_types)
|
||||
|
||||
def test_crawler_hub_not_in_allowlist(self):
|
||||
self.assertNotIn("CrawlerHub", self.allowed_types)
|
||||
|
||||
def test_browser_profiler_not_in_allowlist(self):
|
||||
self.assertNotIn("BrowserProfiler", self.allowed_types)
|
||||
|
||||
def test_docker_client_not_in_allowlist(self):
|
||||
self.assertNotIn("Crawl4aiDockerClient", self.allowed_types)
|
||||
|
||||
def test_allowlist_has_core_config_types(self):
|
||||
required = {"BrowserConfig", "CrawlerRunConfig", "LLMConfig", "ProxyConfig"}
|
||||
self.assertTrue(required.issubset(self.allowed_types))
|
||||
|
||||
def test_allowlist_has_extraction_strategies(self):
|
||||
required = {
|
||||
"JsonCssExtractionStrategy", "LLMExtractionStrategy",
|
||||
"RegexExtractionStrategy",
|
||||
}
|
||||
self.assertTrue(required.issubset(self.allowed_types))
|
||||
|
||||
def test_allowlist_has_enums(self):
|
||||
required = {"CacheMode", "MatchMode", "DisplayMode"}
|
||||
self.assertTrue(required.issubset(self.allowed_types))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
print("=" * 60)
|
||||
print("Crawl4AI Security Fixes - Unit Tests")
|
||||
print("=" * 60)
|
||||
print()
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,124 @@
|
||||
"""
|
||||
R6 headers / CSP / CORS behavioral tests.
|
||||
|
||||
Security headers are emitted unconditionally (independent of security.enabled).
|
||||
The strict CSP applies to the API / error surface; the still-inline dashboard /
|
||||
playground keep the baseline headers but not the strict CSP until they are
|
||||
externalized (CSP-compat refactor - tracked separately). CORS is deny-by-default.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.posture
|
||||
|
||||
|
||||
class TestBaselineHeaders:
|
||||
def test_present_on_api_response(self, stock_client):
|
||||
r = stock_client.get("/health")
|
||||
h = {k.lower(): v for k, v in r.headers.items()}
|
||||
assert h.get("x-content-type-options") == "nosniff"
|
||||
assert h.get("x-frame-options", "").upper() == "DENY"
|
||||
assert h.get("referrer-policy") == "no-referrer"
|
||||
assert h.get("cross-origin-opener-policy") == "same-origin"
|
||||
|
||||
def test_present_even_with_security_disabled(self, stock_client, server_module, monkeypatch):
|
||||
# Headers must be emitted independent of the security.enabled flag (the
|
||||
# old middleware only added them when enabled). Force it off and verify.
|
||||
monkeypatch.setitem(server_module.config["security"], "enabled", False)
|
||||
r = stock_client.get("/health")
|
||||
h = {k.lower() for k in r.headers}
|
||||
assert "content-security-policy" in h
|
||||
assert "x-content-type-options" in h
|
||||
|
||||
def test_baseline_on_dashboard_mount(self, stock_client, server_module):
|
||||
from auth import create_access_token
|
||||
h = {"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'}, scope='admin')}"}
|
||||
r = stock_client.get("/dashboard/", headers=h)
|
||||
hh = {k.lower(): v for k, v in r.headers.items()}
|
||||
assert hh.get("x-content-type-options") == "nosniff"
|
||||
assert hh.get("x-frame-options", "").upper() == "DENY"
|
||||
|
||||
|
||||
class TestStrictCsp:
|
||||
def test_api_csp_is_strict(self, stock_client):
|
||||
csp = stock_client.get("/health").headers.get("content-security-policy", "")
|
||||
assert "script-src 'self'" in csp
|
||||
assert "frame-ancestors 'none'" in csp
|
||||
assert "default-src 'none'" in csp
|
||||
assert "unsafe-inline" not in csp
|
||||
|
||||
def test_ui_mount_not_given_strict_csp(self, stock_client, server_module):
|
||||
# The inline-script UI must not get the strict CSP yet (it would break).
|
||||
from auth import create_access_token
|
||||
h = {"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'}, scope='admin')}"}
|
||||
csp = stock_client.get("/playground/", headers=h).headers.get("content-security-policy")
|
||||
assert csp is None
|
||||
|
||||
|
||||
class TestCorsDenyByDefault:
|
||||
def test_no_cors_allow_origin_by_default(self, stock_client):
|
||||
r = stock_client.get("/health", headers={"Origin": "https://evil.example"})
|
||||
assert "access-control-allow-origin" not in {k.lower() for k in r.headers}
|
||||
|
||||
|
||||
class TestErrorSanitization:
|
||||
def test_5xx_is_generic_with_correlation_id(self, stock_client, server_module):
|
||||
from auth import create_access_token
|
||||
h = {"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'}, scope='admin')}"}
|
||||
# /monitor/stats/reset has no monitor singleton without a lifespan -> the
|
||||
# handler raises -> our central handler returns a sanitized 500.
|
||||
r = stock_client.post("/monitor/stats/reset", headers=h)
|
||||
assert r.status_code >= 500
|
||||
body = r.json()
|
||||
assert body.get("error") == "Internal server error"
|
||||
assert "correlation_id" in body
|
||||
# No internal detail (exception text / paths) leaked.
|
||||
assert "Traceback" not in r.text and "/home/" not in r.text
|
||||
|
||||
def test_4xx_detail_preserved(self, stock_client):
|
||||
# /token is public; with no api_token configured it raises a 4xx, which
|
||||
# the central handler passes through with its developer-facing detail.
|
||||
r = stock_client.post("/token", json={"email": "x@y.com"})
|
||||
assert 400 <= r.status_code < 500
|
||||
assert "detail" in r.json() and r.json()["detail"]
|
||||
|
||||
|
||||
class TestWebhookHeaderSanitization:
|
||||
def test_crlf_in_value_rejected(self):
|
||||
from webhook import sanitize_webhook_headers
|
||||
with pytest.raises(ValueError):
|
||||
sanitize_webhook_headers({"X-Foo": "bar\r\nInjected: 1"})
|
||||
|
||||
def test_hop_by_hop_denied(self):
|
||||
from webhook import sanitize_webhook_headers
|
||||
for bad in ("Host", "Content-Length", "Transfer-Encoding", "Authorization"):
|
||||
with pytest.raises(ValueError):
|
||||
sanitize_webhook_headers({bad: "x"})
|
||||
|
||||
def test_bad_name_rejected(self):
|
||||
from webhook import sanitize_webhook_headers
|
||||
with pytest.raises(ValueError):
|
||||
sanitize_webhook_headers({"X Foo": "bar"})
|
||||
|
||||
def test_good_headers_pass(self):
|
||||
from webhook import sanitize_webhook_headers
|
||||
assert sanitize_webhook_headers({"X-Trace-Id": "abc123"}) == {"X-Trace-Id": "abc123"}
|
||||
|
||||
def test_schema_validator_rejects_early(self):
|
||||
from schemas import WebhookConfig
|
||||
import pydantic
|
||||
with pytest.raises(pydantic.ValidationError):
|
||||
WebhookConfig(webhook_url="https://example.com/cb",
|
||||
webhook_headers={"Host": "evil"})
|
||||
|
||||
|
||||
class TestCRLFSafeLogging:
|
||||
def test_crlf_stripped_from_log_message(self):
|
||||
import logging
|
||||
from utils import CRLFSafeFilter
|
||||
rec = logging.LogRecord("t", logging.INFO, __file__, 1,
|
||||
"url=http://x/\r\nINJECTED admin login", None, None)
|
||||
CRLFSafeFilter().filter(rec)
|
||||
msg = rec.getMessage()
|
||||
assert "\r" not in msg and "\n" not in msg
|
||||
assert "INJECTED" in msg # content kept, just de-fanged
|
||||
@@ -0,0 +1,78 @@
|
||||
"""
|
||||
R3 LLM-broker tests: a request selects a provider BY NAME only; base_url and
|
||||
api_token are always server-derived, so the configured provider key can never
|
||||
be redirected to an attacker endpoint (the reported credential-exfil gadget).
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from llm_broker import resolve_llm, allowed_provider_families, LLMProviderNotAllowed
|
||||
|
||||
pytestmark = pytest.mark.posture
|
||||
|
||||
CONF = {"llm": {"provider": "openai/gpt-4o-mini"}}
|
||||
|
||||
|
||||
class TestProviderAllowlist:
|
||||
def test_default_provider_allowed(self):
|
||||
out = resolve_llm(CONF, None)
|
||||
assert out["provider"] == "openai/gpt-4o-mini"
|
||||
|
||||
def test_same_family_allowed(self):
|
||||
out = resolve_llm(CONF, "openai/gpt-4o")
|
||||
assert out["provider"] == "openai/gpt-4o"
|
||||
|
||||
def test_other_family_rejected(self):
|
||||
with pytest.raises(LLMProviderNotAllowed):
|
||||
resolve_llm(CONF, "anthropic/claude-3-opus")
|
||||
|
||||
def test_explicit_allowlist_widens(self):
|
||||
conf = {"llm": {"provider": "openai/gpt-4o-mini",
|
||||
"allowed_providers": ["anthropic/claude-3-opus"]}}
|
||||
assert resolve_llm(conf, "anthropic/claude-3-opus")["provider"].startswith("anthropic/")
|
||||
|
||||
|
||||
class TestBaseUrlIsServerOnly:
|
||||
def test_resolve_has_no_request_base_url_param(self):
|
||||
import inspect
|
||||
params = list(inspect.signature(resolve_llm).parameters)
|
||||
# only (config, requested_provider) - no way to inject base_url/api_token
|
||||
assert params == ["config", "requested_provider"]
|
||||
|
||||
def test_base_url_is_canonical_from_env(self, monkeypatch):
|
||||
monkeypatch.setenv("OPENAI_BASE_URL", "https://api.openai.example")
|
||||
out = resolve_llm(CONF, "openai/gpt-4o-mini")
|
||||
assert out["base_url"] == "https://api.openai.example"
|
||||
|
||||
def test_no_base_url_when_unset(self, monkeypatch):
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("LLM_BASE_URL", raising=False)
|
||||
out = resolve_llm(CONF, "openai/gpt-4o-mini")
|
||||
assert out["base_url"] is None
|
||||
|
||||
|
||||
class TestRequestSurfaceRemoved:
|
||||
def test_markdown_request_has_no_base_url(self):
|
||||
from schemas import MarkdownRequest
|
||||
assert "base_url" not in MarkdownRequest.model_fields
|
||||
|
||||
def test_llm_job_payload_has_no_base_url(self):
|
||||
from job import LlmJobPayload
|
||||
assert "base_url" not in LlmJobPayload.model_fields
|
||||
|
||||
def test_llm_endpoint_has_no_base_url_query(self, server_module):
|
||||
import inspect
|
||||
assert "base_url" not in inspect.signature(server_module.llm_endpoint).parameters
|
||||
|
||||
def test_md_endpoint_rejects_disallowed_provider(self, stock_client, server_module):
|
||||
from auth import create_access_token
|
||||
h = {"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'})}"}
|
||||
# LLM filter + a provider outside the allowed family -> 400 (not a 500,
|
||||
# and no LLM call to an attacker endpoint).
|
||||
r = stock_client.post(
|
||||
"/md",
|
||||
json={"url": "https://example.com", "f": "llm", "provider": "attacker/model",
|
||||
"base_url": "http://169.254.169.254"},
|
||||
headers=h,
|
||||
)
|
||||
assert r.status_code == 400, r.status_code
|
||||
@@ -0,0 +1,175 @@
|
||||
"""
|
||||
R5 resource-governance tests: request body-size limit (413) and the
|
||||
defense-in-depth deep-crawl clamp. (The client-driven unbounded-crawl critical
|
||||
is closed upstream by R2 forbidding deep_crawl_strategy on untrusted bodies.)
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.posture
|
||||
|
||||
|
||||
class TestBodySizeLimit:
|
||||
def test_oversize_content_length_413(self, stock_client, server_module):
|
||||
from auth import create_access_token
|
||||
h = {
|
||||
"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'})}",
|
||||
"Content-Length": str(50 * 1024 * 1024), # 50 MiB, over the 10 MiB cap
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
# TestClient won't actually send 50MiB; the declared Content-Length is
|
||||
# what the middleware checks.
|
||||
r = stock_client.post("/crawl", data=b"{}", headers=h)
|
||||
assert r.status_code == 413, r.status_code
|
||||
|
||||
def test_normal_body_not_blocked_by_size(self, stock_client, server_module):
|
||||
from auth import create_access_token
|
||||
h = {"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'})}"}
|
||||
# Small body: must pass the size gate (then rejected for missing urls, etc).
|
||||
r = stock_client.post("/crawl", json={}, headers=h)
|
||||
assert r.status_code != 413
|
||||
|
||||
|
||||
class TestDeepCrawlClamp:
|
||||
def test_infinite_max_pages_clamped(self):
|
||||
from governor import clamp_deep_crawl, DEFAULT_MAX_PAGES
|
||||
from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
|
||||
from crawl4ai import CrawlerRunConfig
|
||||
|
||||
strat = BFSDeepCrawlStrategy(max_depth=99) # max_pages defaults to infinity
|
||||
cfg = CrawlerRunConfig(deep_crawl_strategy=strat)
|
||||
clamp_deep_crawl(cfg)
|
||||
assert cfg.deep_crawl_strategy.max_pages <= DEFAULT_MAX_PAGES
|
||||
assert cfg.deep_crawl_strategy.max_depth <= 5
|
||||
|
||||
def test_no_deep_crawl_is_noop(self):
|
||||
from governor import clamp_deep_crawl
|
||||
from crawl4ai import CrawlerRunConfig
|
||||
cfg = CrawlerRunConfig()
|
||||
clamp_deep_crawl(cfg) # must not raise
|
||||
assert cfg.deep_crawl_strategy is None
|
||||
|
||||
|
||||
class TestUntrustedDeepCrawlStillForbidden:
|
||||
def test_request_cannot_set_deep_crawl(self):
|
||||
# The clamp is defense in depth; the primary control is R2 rejecting it.
|
||||
from crawl4ai.async_configs import CrawlerRunConfig, Provenance, UntrustedConfigError
|
||||
with pytest.raises(UntrustedConfigError):
|
||||
CrawlerRunConfig.load(
|
||||
{"deep_crawl_strategy": {"type": "BFSDeepCrawlStrategy", "params": {"max_depth": 9}}},
|
||||
provenance=Provenance.UNTRUSTED,
|
||||
)
|
||||
|
||||
|
||||
class TestConfigCaps:
|
||||
def test_job_queue_caps_defaults(self):
|
||||
from governor import job_queue_caps
|
||||
caps = job_queue_caps({})
|
||||
assert caps == {"maxsize": 1000, "workers": 4, "per_principal": 0}
|
||||
|
||||
def test_zero_means_unbounded(self):
|
||||
from governor import job_queue_caps, wall_clock_seconds
|
||||
caps = job_queue_caps({"limits": {"queue": {"maxsize": 0, "per_principal": 0}}})
|
||||
assert caps["maxsize"] == 0 and caps["per_principal"] == 0
|
||||
assert wall_clock_seconds({}) == 0 # no deadline by default
|
||||
|
||||
def test_wall_clock_configurable(self):
|
||||
from governor import wall_clock_seconds
|
||||
assert wall_clock_seconds({"limits": {"wall_clock_s": 30}}) == 30
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestWorkQueue:
|
||||
async def test_queue_full_without_drain(self):
|
||||
import asyncio
|
||||
from work_queue import WorkQueue, QueueFull
|
||||
q = WorkQueue(maxsize=1, workers=1)
|
||||
q._q = asyncio.Queue(maxsize=1) # bypass workers so nothing drains
|
||||
|
||||
async def noop():
|
||||
pass
|
||||
q.submit(noop)
|
||||
with pytest.raises(QueueFull):
|
||||
q.submit(noop)
|
||||
|
||||
async def test_unbounded_never_full(self):
|
||||
import asyncio
|
||||
from work_queue import WorkQueue
|
||||
q = WorkQueue(maxsize=0, workers=1)
|
||||
q._q = asyncio.Queue(maxsize=0)
|
||||
|
||||
async def noop():
|
||||
pass
|
||||
for _ in range(200):
|
||||
q.submit(noop) # maxsize 0 => unbounded, never raises
|
||||
|
||||
async def test_per_principal_quota(self):
|
||||
import asyncio
|
||||
from work_queue import WorkQueue, QuotaExceeded
|
||||
q = WorkQueue(maxsize=0, workers=1, per_principal=1)
|
||||
q._q = asyncio.Queue(maxsize=0)
|
||||
|
||||
async def noop():
|
||||
pass
|
||||
q.submit(noop, principal="alice")
|
||||
with pytest.raises(QuotaExceeded):
|
||||
q.submit(noop, principal="alice")
|
||||
q.submit(noop, principal="bob") # different caller is fine
|
||||
|
||||
async def test_runs_job_and_releases_counter(self):
|
||||
import asyncio
|
||||
from work_queue import WorkQueue
|
||||
q = WorkQueue(maxsize=10, workers=1, per_principal=5)
|
||||
await q.start()
|
||||
try:
|
||||
done = asyncio.Event()
|
||||
|
||||
async def job():
|
||||
done.set()
|
||||
q.submit(job, principal="p")
|
||||
await asyncio.wait_for(done.wait(), timeout=2)
|
||||
await asyncio.sleep(0.05) # allow the finally/release to run
|
||||
assert q._counts.get("p", 0) == 0
|
||||
finally:
|
||||
await q.stop()
|
||||
|
||||
|
||||
class TestEnqueueMapping:
|
||||
def test_enqueue_maps_queue_full_to_503(self):
|
||||
import asyncio
|
||||
import api
|
||||
import work_queue
|
||||
from fastapi import HTTPException
|
||||
q = work_queue.WorkQueue(maxsize=1, workers=1)
|
||||
q._q = asyncio.Queue(maxsize=1)
|
||||
work_queue.set_job_queue(q)
|
||||
|
||||
async def noop():
|
||||
pass
|
||||
try:
|
||||
api._enqueue_job(None, noop) # fills the queue
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
api._enqueue_job(None, noop)
|
||||
assert exc.value.status_code == 503
|
||||
assert exc.value.headers.get("Retry-After")
|
||||
finally:
|
||||
work_queue.set_job_queue(None)
|
||||
|
||||
def test_enqueue_maps_quota_to_429(self):
|
||||
import asyncio
|
||||
import api
|
||||
import work_queue
|
||||
from fastapi import HTTPException
|
||||
q = work_queue.WorkQueue(maxsize=0, workers=1, per_principal=1)
|
||||
q._q = asyncio.Queue(maxsize=0)
|
||||
work_queue.set_job_queue(q)
|
||||
|
||||
async def noop():
|
||||
pass
|
||||
try:
|
||||
api._enqueue_job(None, noop, principal="a")
|
||||
with pytest.raises(HTTPException) as exc:
|
||||
api._enqueue_job(None, noop, principal="a")
|
||||
assert exc.value.status_code == 429
|
||||
finally:
|
||||
work_queue.set_job_queue(None)
|
||||
@@ -0,0 +1,314 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Adversarial tests for SSRF protection on crawl/md/llm URL entry points.
|
||||
Reported by secsys_codex (2026-04-18).
|
||||
|
||||
Tests that validate_url_destination() blocks internal IPs on all crawl paths,
|
||||
and that CRAWL4AI_ALLOW_INTERNAL_URLS=true bypasses the check.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
import ipaddress
|
||||
import socket
|
||||
from urllib.parse import urlparse
|
||||
|
||||
DEPLOY_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..")
|
||||
|
||||
# Local copy of validation logic for self-contained testing
|
||||
_BLOCKED_NETWORKS = [
|
||||
ipaddress.ip_network("0.0.0.0/8"),
|
||||
ipaddress.ip_network("10.0.0.0/8"),
|
||||
ipaddress.ip_network("100.64.0.0/10"),
|
||||
ipaddress.ip_network("127.0.0.0/8"),
|
||||
ipaddress.ip_network("169.254.0.0/16"),
|
||||
ipaddress.ip_network("172.16.0.0/12"),
|
||||
ipaddress.ip_network("192.0.0.0/24"),
|
||||
ipaddress.ip_network("192.168.0.0/16"),
|
||||
ipaddress.ip_network("198.18.0.0/15"),
|
||||
ipaddress.ip_network("::1/128"),
|
||||
ipaddress.ip_network("fc00::/7"),
|
||||
ipaddress.ip_network("fe80::/10"),
|
||||
]
|
||||
_BLOCKED_HOSTNAMES = {
|
||||
"localhost", "metadata.google.internal", "metadata",
|
||||
"kubernetes.default", "kubernetes.default.svc",
|
||||
}
|
||||
|
||||
|
||||
def _expand_ip_candidates(ip):
|
||||
"""Mirror of utils.py: unwrap IPv4 from IPv4-mapped/compat IPv6."""
|
||||
candidates = [ip]
|
||||
if isinstance(ip, ipaddress.IPv6Address):
|
||||
if ip.ipv4_mapped is not None:
|
||||
candidates.append(ip.ipv4_mapped)
|
||||
else:
|
||||
as_int = int(ip)
|
||||
if 0 < as_int < 2**32:
|
||||
candidates.append(ipaddress.IPv4Address(as_int))
|
||||
return candidates
|
||||
|
||||
|
||||
def _validate_webhook_url(url):
|
||||
parsed = urlparse(str(url))
|
||||
hostname = parsed.hostname
|
||||
if not hostname:
|
||||
raise ValueError("URL must have a valid hostname")
|
||||
if hostname.lower() in _BLOCKED_HOSTNAMES:
|
||||
raise ValueError(f"Hostname '{hostname}' is blocked")
|
||||
if hostname.lower().startswith("host.docker.internal"):
|
||||
raise ValueError(f"Hostname '{hostname}' is blocked")
|
||||
try:
|
||||
resolved = socket.getaddrinfo(hostname, None)
|
||||
except socket.gaierror:
|
||||
raise ValueError(f"Cannot resolve hostname '{hostname}'")
|
||||
for _, _, _, _, sockaddr in resolved:
|
||||
ip = ipaddress.ip_address(sockaddr[0])
|
||||
for candidate in _expand_ip_candidates(ip):
|
||||
for network in _BLOCKED_NETWORKS:
|
||||
if candidate in network:
|
||||
raise ValueError(f"URL resolves to blocked address: {ip}")
|
||||
|
||||
|
||||
def validate_url_destination(url, allow_internal=False):
|
||||
"""Simulates the actual validate_url_destination from utils.py."""
|
||||
if allow_internal:
|
||||
return
|
||||
if str(url).startswith(("raw:", "raw://")):
|
||||
return
|
||||
_validate_webhook_url(url)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# SSRF attacks on crawl URLs
|
||||
# ============================================================================
|
||||
|
||||
class TestCrawlURLSSRF(unittest.TestCase):
|
||||
"""Test SSRF protection on URLs that go to crawler.arun()."""
|
||||
|
||||
def test_localhost_blocked(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_url_destination("http://127.0.0.1:8080/admin")
|
||||
|
||||
def test_localhost_name_blocked(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_url_destination("http://localhost:8080/secret")
|
||||
|
||||
def test_10_network_blocked(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_url_destination("http://10.0.0.1/internal")
|
||||
|
||||
def test_172_16_blocked(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_url_destination("http://172.16.0.1/dashboard")
|
||||
|
||||
def test_192_168_blocked(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_url_destination("http://192.168.1.1/router")
|
||||
|
||||
def test_aws_metadata(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_url_destination("http://169.254.169.254/latest/meta-data/")
|
||||
|
||||
def test_gcp_metadata(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_url_destination("http://metadata.google.internal/computeMetadata/v1/")
|
||||
|
||||
def test_docker_internal(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_url_destination("http://host.docker.internal:3000/api")
|
||||
|
||||
def test_kubernetes(self):
|
||||
with self.assertRaises(ValueError):
|
||||
validate_url_destination("http://kubernetes.default/api/v1/secrets")
|
||||
|
||||
# -- Must allow: external URLs --
|
||||
|
||||
def test_external_url_allowed(self):
|
||||
validate_url_destination("https://example.com")
|
||||
validate_url_destination("https://www.google.com")
|
||||
|
||||
# -- raw: URLs bypass (no network fetch) --
|
||||
|
||||
def test_raw_url_bypasses(self):
|
||||
validate_url_destination("raw:<html><body>hello</body></html>")
|
||||
validate_url_destination("raw://<html>test</html>")
|
||||
|
||||
# -- ALLOW_INTERNAL_URLS opt-out --
|
||||
|
||||
def test_allow_internal_bypasses(self):
|
||||
"""When opted in, internal URLs should pass."""
|
||||
validate_url_destination("http://127.0.0.1:8080", allow_internal=True)
|
||||
validate_url_destination("http://10.0.0.1/internal", allow_internal=True)
|
||||
validate_url_destination("http://169.254.169.254/meta", allow_internal=True)
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# IPv6-mapped IPv4 bypass (caught in internal security review)
|
||||
# Prior code only checked blocked networks directly; ::ffff:127.0.0.1
|
||||
# parses as ::ffff:7f00:1 which is not in 127.0.0.0/8, bypassing the guard.
|
||||
# ============================================================================
|
||||
|
||||
class TestIPv6MappedBypass(unittest.TestCase):
|
||||
"""Test the IPv4-mapped IPv6 SSRF bypass class."""
|
||||
|
||||
def test_mapped_loopback_blocked(self):
|
||||
"""http://[::ffff:127.0.0.1]/ must be blocked (maps to 127.0.0.1)."""
|
||||
with self.assertRaises(ValueError):
|
||||
validate_url_destination("http://[::ffff:127.0.0.1]/admin")
|
||||
|
||||
def test_mapped_private_10_blocked(self):
|
||||
"""::ffff:10.0.0.1 must be blocked (maps to RFC 1918)."""
|
||||
with self.assertRaises(ValueError):
|
||||
validate_url_destination("http://[::ffff:10.0.0.1]/")
|
||||
|
||||
def test_mapped_aws_metadata_blocked(self):
|
||||
"""::ffff:169.254.169.254 must be blocked (maps to cloud metadata)."""
|
||||
with self.assertRaises(ValueError):
|
||||
validate_url_destination("http://[::ffff:169.254.169.254]/latest/meta-data/")
|
||||
|
||||
def test_mapped_192_168_blocked(self):
|
||||
"""::ffff:192.168.1.1 must be blocked."""
|
||||
with self.assertRaises(ValueError):
|
||||
validate_url_destination("http://[::ffff:192.168.1.1]/")
|
||||
|
||||
def test_ipv4_compatible_loopback_blocked(self):
|
||||
"""IPv4-compatible IPv6 (deprecated but still exists): ::127.0.0.1."""
|
||||
with self.assertRaises(ValueError):
|
||||
validate_url_destination("http://[::127.0.0.1]/")
|
||||
|
||||
def test_regular_ipv6_loopback_still_blocked(self):
|
||||
"""::1 must remain blocked (IPv6 loopback)."""
|
||||
with self.assertRaises(ValueError):
|
||||
validate_url_destination("http://[::1]:8080/")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# Source-level verification
|
||||
# ============================================================================
|
||||
|
||||
class TestSSRFSourceCoverage(unittest.TestCase):
|
||||
"""Verify all URL entry points have SSRF validation."""
|
||||
|
||||
def test_server_validate_url_scheme_calls_destination(self):
|
||||
"""validate_url_scheme must also call validate_url_destination."""
|
||||
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
|
||||
source = f.read()
|
||||
# Find validate_url_scheme function body
|
||||
idx = source.index("def validate_url_scheme")
|
||||
func_end = source.index("\ndef ", idx + 1) if "\ndef " in source[idx+1:] else idx + 500
|
||||
func_body = source[idx:func_end]
|
||||
self.assertIn("validate_url_destination", func_body,
|
||||
"validate_url_scheme must call validate_url_destination")
|
||||
|
||||
def test_api_py_has_destination_validation(self):
|
||||
"""api.py must call validate_url_destination for all URL entry points."""
|
||||
with open(os.path.join(DEPLOY_DIR, "api.py")) as f:
|
||||
source = f.read()
|
||||
self.assertIn("validate_url_destination", source,
|
||||
"api.py must import and use validate_url_destination")
|
||||
# Count occurrences -- should have at least 4 (one per entry point)
|
||||
count = source.count("validate_url_destination")
|
||||
self.assertGreaterEqual(count, 5, # 1 import + 4 call sites
|
||||
f"api.py should call validate_url_destination at all URL entry points (found {count})")
|
||||
|
||||
def test_every_crawl_handler_validates_destination(self):
|
||||
"""Per-handler check: each crawl handler body must validate seed URL
|
||||
destinations (via the shared _normalize_and_validate_seeds helper). A
|
||||
bare occurrence count missed the streaming handler (it had zero calls
|
||||
while the count was still satisfied)."""
|
||||
with open(os.path.join(DEPLOY_DIR, "api.py")) as f:
|
||||
source = f.read()
|
||||
handlers = [
|
||||
"handle_crawl_request",
|
||||
"handle_stream_crawl_request",
|
||||
]
|
||||
for handler in handlers:
|
||||
idx = source.index(f"async def {handler}")
|
||||
nxt = source.find("\nasync def ", idx + 1)
|
||||
body = source[idx: nxt if nxt != -1 else len(source)]
|
||||
self.assertIn("_normalize_and_validate_seeds(", body,
|
||||
f"{handler} must validate its seed URLs via _normalize_and_validate_seeds")
|
||||
|
||||
def test_default_browser_config_pins_egress(self):
|
||||
"""get_default_browser_config must apply enforce_egress so /html,
|
||||
/screenshot, /pdf, /execute_js fetch through the pinning proxy
|
||||
(DNS-rebinding / redirect-to-internal protection), not just validate
|
||||
the seed and then let the browser re-resolve."""
|
||||
with open(os.path.join(DEPLOY_DIR, "server.py")) as f:
|
||||
source = f.read()
|
||||
idx = source.index("def get_default_browser_config")
|
||||
nxt = source.find("\ndef ", idx + 1)
|
||||
body = source[idx: nxt if nxt != -1 else idx + 800]
|
||||
self.assertIn("enforce_egress", body,
|
||||
"get_default_browser_config must call enforce_egress on the returned config")
|
||||
|
||||
def test_llm_and_markdown_handlers_pin_egress(self):
|
||||
"""The /md and /llm handlers and the LLM background worker must pin
|
||||
egress on their fetch config (same seed-only SSRF root cause)."""
|
||||
with open(os.path.join(DEPLOY_DIR, "api.py")) as f:
|
||||
source = f.read()
|
||||
for handler in ("handle_llm_qa", "handle_markdown_request", "process_llm_extraction"):
|
||||
idx = source.index(f"def {handler}")
|
||||
nxt = source.find("\nasync def ", idx + 1)
|
||||
nxt2 = source.find("\ndef ", idx + 1)
|
||||
ends = [e for e in (nxt, nxt2) if e != -1]
|
||||
body = source[idx: min(ends) if ends else len(source)]
|
||||
self.assertIn("enforce_egress", body,
|
||||
f"{handler} must call enforce_egress on its fetch config")
|
||||
|
||||
def test_utils_has_allow_internal_flag(self):
|
||||
"""utils.py must have CRAWL4AI_ALLOW_INTERNAL_URLS env var."""
|
||||
with open(os.path.join(DEPLOY_DIR, "utils.py")) as f:
|
||||
source = f.read()
|
||||
self.assertIn("CRAWL4AI_ALLOW_INTERNAL_URLS", source)
|
||||
self.assertIn("ALLOW_INTERNAL_URLS", source)
|
||||
|
||||
def test_validate_url_destination_skips_raw(self):
|
||||
"""validate_url_destination must skip raw: URLs."""
|
||||
with open(os.path.join(DEPLOY_DIR, "utils.py")) as f:
|
||||
source = f.read()
|
||||
idx = source.index("def validate_url_destination")
|
||||
func_body = source[idx:idx+500]
|
||||
self.assertIn("raw:", func_body,
|
||||
"validate_url_destination must skip raw: URLs")
|
||||
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestStreamCrawlSSRFBehavioral:
|
||||
"""End-to-end: an internal seed URL must be 400-blocked on the streaming
|
||||
path, not just asserted to exist in source. This is the behavioral guard
|
||||
for the KOH stream-SSRF fix (and proves the deliberate 400 is not masked
|
||||
to 500 by the handler's generic except)."""
|
||||
|
||||
def _auth(self):
|
||||
from auth import create_access_token
|
||||
return {"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'})}"}
|
||||
|
||||
INTERNAL = "http://169.254.169.254/latest/meta-data/"
|
||||
|
||||
def test_crawl_stream_blocks_internal(self, stock_client, server_module):
|
||||
r = stock_client.post("/crawl/stream", json={"urls": [self.INTERNAL]}, headers=self._auth())
|
||||
assert r.status_code == 400, f"expected 400 SSRF block, got {r.status_code}: {r.text[:200]}"
|
||||
assert "SSRF" in r.text or "blocked" in r.text.lower()
|
||||
|
||||
def test_crawl_with_stream_true_blocks_internal(self, stock_client, server_module):
|
||||
body = {"urls": [self.INTERNAL],
|
||||
"crawler_config": {"type": "CrawlerRunConfig", "params": {"stream": True}}}
|
||||
r = stock_client.post("/crawl", json=body, headers=self._auth())
|
||||
assert r.status_code == 400, f"expected 400 SSRF block, got {r.status_code}: {r.text[:200]}"
|
||||
|
||||
def test_non_stream_crawl_still_blocks_internal(self, stock_client, server_module):
|
||||
r = stock_client.post("/crawl", json={"urls": [self.INTERNAL]}, headers=self._auth())
|
||||
assert r.status_code == 400, f"expected 400 SSRF block, got {r.status_code}: {r.text[:200]}"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("=" * 70)
|
||||
print("Crawl4AI SSRF Tests - Crawl/MD/LLM Endpoints (secsys_codex)")
|
||||
print("=" * 70)
|
||||
print()
|
||||
unittest.main(verbosity=2)
|
||||
@@ -0,0 +1,142 @@
|
||||
"""
|
||||
R3 egress-broker behavioral tests (fully offline via the conftest DNS fixtures).
|
||||
|
||||
One rule: reject any resolved IP where not ip.is_global, on every transition
|
||||
form (v4-mapped / NAT64 / 6to4 / v4-compat). Resolve once and pin, so DNS
|
||||
rebinding never reaches the second (internal) answer. Errors are opaque.
|
||||
"""
|
||||
|
||||
import socket
|
||||
|
||||
import pytest
|
||||
|
||||
from egress_broker import (
|
||||
EgressBlocked,
|
||||
check_redirect,
|
||||
is_forbidden_ip,
|
||||
resolve_and_pin,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.posture
|
||||
|
||||
|
||||
class TestForbiddenIp:
|
||||
@pytest.mark.parametrize("ip", [
|
||||
"127.0.0.1", "169.254.169.254", "10.0.0.5", "192.168.1.1",
|
||||
"172.16.0.1", "100.64.0.1", "0.0.0.0",
|
||||
"::1", "::", "fc00::1", "fe80::1",
|
||||
"::ffff:127.0.0.1", # v4-mapped loopback
|
||||
"::ffff:169.254.169.254", # v4-mapped metadata
|
||||
"64:ff9b::a9fe:a9fe", # NAT64 -> 169.254.169.254
|
||||
"2002:a9fe:a9fe::1", # 6to4 embedding 169.254.169.254
|
||||
])
|
||||
def test_internal_forms_forbidden(self, ip):
|
||||
assert is_forbidden_ip(ip) is True
|
||||
|
||||
@pytest.mark.parametrize("ip", ["8.8.8.8", "1.1.1.1", "2606:4700:4700::1111"])
|
||||
def test_global_allowed(self, ip):
|
||||
assert is_forbidden_ip(ip) is False
|
||||
|
||||
|
||||
class TestResolveAndPin:
|
||||
def test_public_host_pins_ip(self, offline_dns):
|
||||
offline_dns.set("good.example", "93.184.216.34")
|
||||
t = resolve_and_pin("https://good.example/path")
|
||||
assert t.ip == "93.184.216.34" and t.host == "good.example" and t.port == 443
|
||||
|
||||
def test_metadata_blocked(self, offline_dns):
|
||||
offline_dns.set("meta.example", "169.254.169.254")
|
||||
with pytest.raises(EgressBlocked):
|
||||
resolve_and_pin("http://meta.example/latest/meta-data/")
|
||||
|
||||
def test_nat64_metadata_blocked(self, offline_dns):
|
||||
offline_dns.set("nat64.example", "64:ff9b::a9fe:a9fe")
|
||||
with pytest.raises(EgressBlocked):
|
||||
resolve_and_pin("http://nat64.example/")
|
||||
|
||||
def test_localhost_name_blocked(self, offline_dns):
|
||||
with pytest.raises(EgressBlocked):
|
||||
resolve_and_pin("http://localhost/")
|
||||
|
||||
def test_non_http_scheme_blocked(self):
|
||||
with pytest.raises(EgressBlocked):
|
||||
resolve_and_pin("file:///etc/passwd")
|
||||
with pytest.raises(EgressBlocked):
|
||||
resolve_and_pin("gopher://x/")
|
||||
|
||||
def test_error_is_opaque(self, offline_dns):
|
||||
offline_dns.set("meta.example", "169.254.169.254")
|
||||
try:
|
||||
resolve_and_pin("http://meta.example/")
|
||||
assert False
|
||||
except EgressBlocked as e:
|
||||
# must not leak the resolved IP or hostname
|
||||
assert "169.254" not in e.reason
|
||||
assert "meta.example" not in e.reason
|
||||
assert e.reason == "URL blocked"
|
||||
|
||||
def test_multi_record_one_internal_rejects_host(self, monkeypatch):
|
||||
def fake(host, port, *a, **k):
|
||||
return [
|
||||
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("93.184.216.34", port or 0)),
|
||||
(socket.AF_INET, socket.SOCK_STREAM, 6, "", ("127.0.0.1", port or 0)),
|
||||
]
|
||||
monkeypatch.setattr(socket, "getaddrinfo", fake)
|
||||
with pytest.raises(EgressBlocked):
|
||||
resolve_and_pin("http://mixed.example/")
|
||||
|
||||
|
||||
class TestDnsRebinding:
|
||||
def test_pins_first_answer_never_reresolves_internal(self, rebinding_dns):
|
||||
r = rebinding_dns("rebind.example", "93.184.216.34", "169.254.169.254")
|
||||
t = resolve_and_pin("http://rebind.example/")
|
||||
# Pinned to the public answer; the caller dials t.ip, not the host, so
|
||||
# the internal second answer is never used.
|
||||
assert t.ip == "93.184.216.34"
|
||||
assert r.calls == 1 # resolved exactly once
|
||||
|
||||
|
||||
class TestEnforceEgress:
|
||||
def test_tls_verification_forced_on(self, monkeypatch):
|
||||
import egress_broker
|
||||
monkeypatch.setattr(egress_broker, "ALLOW_INSECURE_TLS", False)
|
||||
from crawl4ai import BrowserConfig
|
||||
b = BrowserConfig(ignore_https_errors=True)
|
||||
egress_broker.enforce_egress(b)
|
||||
assert b.ignore_https_errors is False
|
||||
|
||||
def test_proxy_nulled(self):
|
||||
import egress_broker
|
||||
from crawl4ai import BrowserConfig
|
||||
b = BrowserConfig()
|
||||
b.proxy_config = object()
|
||||
egress_broker.enforce_egress(b)
|
||||
assert b.proxy_config is None
|
||||
|
||||
def test_dangerous_args_stripped(self):
|
||||
import egress_broker
|
||||
from crawl4ai import BrowserConfig
|
||||
b = BrowserConfig(extra_args=["--proxy-server=http://evil", "--ignore-certificate-errors", "--headless"])
|
||||
egress_broker.enforce_egress(b)
|
||||
assert b.extra_args == ["--headless"]
|
||||
|
||||
|
||||
class TestRedirectRevalidation:
|
||||
def test_redirect_to_internal_blocked(self, offline_dns):
|
||||
offline_dns.set("evil-redirect.example", "169.254.169.254")
|
||||
with pytest.raises(EgressBlocked):
|
||||
check_redirect("http://evil-redirect.example/")
|
||||
|
||||
|
||||
class TestWebhookValidatorUsesBroker:
|
||||
def test_webhook_internal_blocked_opaque(self, offline_dns):
|
||||
from utils import validate_webhook_url
|
||||
offline_dns.set("hook.example", "10.0.0.9")
|
||||
with pytest.raises(ValueError) as exc:
|
||||
validate_webhook_url("http://hook.example/cb")
|
||||
assert "10.0.0" not in str(exc.value) # no IP leak
|
||||
|
||||
def test_webhook_public_ok(self, offline_dns):
|
||||
from utils import validate_webhook_url
|
||||
offline_dns.set("hook.example", "93.184.216.34")
|
||||
validate_webhook_url("http://hook.example/cb") # no raise
|
||||
@@ -0,0 +1,236 @@
|
||||
"""
|
||||
R2 trust-boundary behavioral tests (library-level provenance machinery).
|
||||
|
||||
Untrusted request bodies may only construct a strict subset of types and may
|
||||
only set scalar, non-power fields, which are then clamped. Trusted (SDK /
|
||||
in-process) construction is unchanged.
|
||||
|
||||
Docker-layer wiring tests (handlers passing UNTRUSTED, /config/dump
|
||||
validate-only, declarative hooks) live alongside as the docker pieces land.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from crawl4ai.async_configs import (
|
||||
BrowserConfig,
|
||||
CrawlerRunConfig,
|
||||
Provenance,
|
||||
UntrustedConfigError,
|
||||
)
|
||||
|
||||
T = Provenance.TRUSTED
|
||||
U = Provenance.UNTRUSTED
|
||||
|
||||
pytestmark = pytest.mark.posture
|
||||
|
||||
|
||||
class TestTrustedUnchanged:
|
||||
"""SDK/in-process callers (default TRUSTED) keep full power."""
|
||||
|
||||
def test_trusted_keeps_power_fields(self):
|
||||
c = CrawlerRunConfig.load(
|
||||
{"js_code": "x=1", "page_timeout": 999999, "word_count_threshold": 5},
|
||||
provenance=T,
|
||||
)
|
||||
assert c.js_code == "x=1"
|
||||
assert c.page_timeout == 999999
|
||||
|
||||
def test_trusted_is_the_default(self):
|
||||
# No provenance argument => behaves exactly as before.
|
||||
c = CrawlerRunConfig.load({"js_code": "y=2"})
|
||||
assert c.js_code == "y=2"
|
||||
|
||||
def test_trusted_roundtrip(self):
|
||||
c = CrawlerRunConfig(js_code="z=3", screenshot=True, page_timeout=120000)
|
||||
c2 = CrawlerRunConfig.load(c.dump())
|
||||
assert c2.js_code == "z=3" and c2.page_timeout == 120000
|
||||
|
||||
|
||||
class TestUntrustedForbiddenFields:
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[
|
||||
("js_code", "alert(1)"),
|
||||
("js_code_before_wait", "x"),
|
||||
("deep_crawl_strategy", {"type": "BFSDeepCrawlStrategy", "params": {}}),
|
||||
("proxy_config", {"server": "http://evil"}),
|
||||
("base_url", "http://evil"),
|
||||
("simulate_user", True),
|
||||
("magic", True),
|
||||
("process_in_browser", True),
|
||||
("session_id", "shared"),
|
||||
],
|
||||
)
|
||||
def test_crawler_power_field_rejected(self, field, value):
|
||||
with pytest.raises(UntrustedConfigError):
|
||||
CrawlerRunConfig.load({field: value}, provenance=U)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field,value",
|
||||
[
|
||||
("extra_args", ["--proxy-server=http://evil"]),
|
||||
("proxy", "http://evil"),
|
||||
("user_data_dir", "/etc"),
|
||||
("cdp_url", "ws://evil"),
|
||||
("cookies", [{"name": "s", "value": "x"}]),
|
||||
("headers", {"X": "y"}),
|
||||
("init_scripts", ["evil()"]),
|
||||
],
|
||||
)
|
||||
def test_browser_power_field_rejected(self, field, value):
|
||||
with pytest.raises(UntrustedConfigError):
|
||||
BrowserConfig.load({field: value}, provenance=U)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"payload",
|
||||
[
|
||||
["--no-zygote", "--utility-cmd-prefix=sh -c id>/tmp/pwn"],
|
||||
["--renderer-cmd-prefix=/bin/sh"],
|
||||
["--gpu-launcher=touch /tmp/x"],
|
||||
["--browser-subprocess-path=/bin/sh"],
|
||||
],
|
||||
)
|
||||
def test_extra_args_chromium_cmd_injection_rejected(self, payload):
|
||||
"""Y4tacker RCE: Chromium launch-arg command injection via extra_args.
|
||||
extra_args is a forbidden field for untrusted bodies, so ANY value
|
||||
(including the --*-cmd-prefix / --no-zygote exec primitives) is rejected
|
||||
outright rather than scrubbed by an always-incomplete denylist."""
|
||||
with pytest.raises(UntrustedConfigError):
|
||||
BrowserConfig.load({"extra_args": payload}, provenance=U)
|
||||
|
||||
|
||||
class TestUntrustedTypeGate:
|
||||
def test_llm_strategy_rejected(self):
|
||||
body = {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"extraction_strategy": {
|
||||
"type": "LLMExtractionStrategy",
|
||||
"params": {},
|
||||
}
|
||||
},
|
||||
}
|
||||
with pytest.raises(UntrustedConfigError):
|
||||
CrawlerRunConfig.load(body, provenance=U)
|
||||
|
||||
def test_env_exfil_poc_rejected(self):
|
||||
"""The verified os.getenv-via-'env:' credential-exfil PoC must 400."""
|
||||
poc = {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"extraction_strategy": {
|
||||
"type": "LLMExtractionStrategy",
|
||||
"params": {
|
||||
"llm_config": {
|
||||
"type": "LLMConfig",
|
||||
"params": {"api_token": "env:SECRET_KEY"},
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
with pytest.raises(UntrustedConfigError):
|
||||
CrawlerRunConfig.load(poc, provenance=U)
|
||||
|
||||
def test_allowed_non_llm_strategy_accepted(self):
|
||||
body = {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"extraction_strategy": {
|
||||
"type": "JsonCssExtractionStrategy",
|
||||
"params": {"schema": {"name": "x", "baseSelector": "div", "fields": []}},
|
||||
}
|
||||
},
|
||||
}
|
||||
c = CrawlerRunConfig.load(body, provenance=U)
|
||||
assert c.extraction_strategy is not None
|
||||
|
||||
|
||||
class TestUntrustedClampAndDrop:
|
||||
def test_timeout_zero_becomes_cap(self):
|
||||
c = CrawlerRunConfig.load({"page_timeout": 0}, provenance=U)
|
||||
assert c.page_timeout == 60_000 # 0 ("no timeout") clamped, never unbounded
|
||||
|
||||
def test_timeout_clamped(self):
|
||||
c = CrawlerRunConfig.load({"wait_for_timeout": 500000}, provenance=U)
|
||||
assert c.wait_for_timeout == 60_000
|
||||
|
||||
def test_unknown_field_dropped_not_raised(self):
|
||||
c = CrawlerRunConfig.load(
|
||||
{"css_selector": ".x", "totally_unknown_field": 1}, provenance=U
|
||||
)
|
||||
assert c.css_selector == ".x"
|
||||
|
||||
def test_viewport_clamped(self):
|
||||
b = BrowserConfig.load({"viewport_width": 99999, "headless": True}, provenance=U)
|
||||
assert b.viewport_width == 4000
|
||||
assert b.headless is True
|
||||
|
||||
def test_safe_scalar_kept(self):
|
||||
c = CrawlerRunConfig.load(
|
||||
{"word_count_threshold": 7, "screenshot": True, "wait_until": "load"},
|
||||
provenance=U,
|
||||
)
|
||||
assert c.word_count_threshold == 7 and c.screenshot is True
|
||||
|
||||
|
||||
class TestLlmConfigGuard:
|
||||
def test_untrusted_env_token_rejected(self):
|
||||
from crawl4ai.async_configs import LLMConfig
|
||||
with pytest.raises(UntrustedConfigError):
|
||||
LLMConfig(api_token="env:SECRET_KEY", provenance=U)
|
||||
|
||||
def test_untrusted_never_reads_env(self, monkeypatch):
|
||||
from crawl4ai.async_configs import LLMConfig
|
||||
monkeypatch.setenv("OPENAI_API_KEY", "leak-me")
|
||||
cfg = LLMConfig(provider="openai/gpt-4o-mini", provenance=U)
|
||||
assert cfg.api_token != "leak-me"
|
||||
|
||||
|
||||
class TestDockerHandlersEnforceUntrusted:
|
||||
"""The Docker endpoints map UntrustedConfigError -> HTTP 400 (not 500)."""
|
||||
|
||||
def _auth(self, server_module):
|
||||
from auth import create_access_token
|
||||
return {"Authorization": f"Bearer {create_access_token({'sub': 'u@x.com'})}"}
|
||||
|
||||
def test_crawl_rejects_js_code(self, stock_client, server_module):
|
||||
r = stock_client.post(
|
||||
"/crawl",
|
||||
json={"urls": ["https://example.com"], "crawler_config": {"js_code": "fetch('http://169.254.169.254')"}},
|
||||
headers=self._auth(server_module),
|
||||
)
|
||||
assert r.status_code == 400, r.status_code
|
||||
|
||||
def test_crawl_rejects_proxy_config(self, stock_client, server_module):
|
||||
r = stock_client.post(
|
||||
"/crawl",
|
||||
json={"urls": ["https://example.com"],
|
||||
"browser_config": {"proxy_config": {"server": "http://evil"}}},
|
||||
headers=self._auth(server_module),
|
||||
)
|
||||
assert r.status_code == 400, r.status_code
|
||||
|
||||
def test_config_dump_rejects_power_field(self, stock_client, server_module):
|
||||
r = stock_client.post(
|
||||
"/config/dump",
|
||||
json={"type": "BrowserConfig", "params": {"extra_args": ["--proxy-server=x"]}},
|
||||
headers=self._auth(server_module),
|
||||
)
|
||||
assert r.status_code == 400, r.status_code
|
||||
|
||||
def test_config_dump_validates_safe_config(self, stock_client, server_module):
|
||||
r = stock_client.post(
|
||||
"/config/dump",
|
||||
json={"type": "CrawlerRunConfig", "params": {"word_count_threshold": 5, "screenshot": True}},
|
||||
headers=self._auth(server_module),
|
||||
)
|
||||
assert r.status_code == 200, r.status_code
|
||||
|
||||
def test_config_dump_drops_unknown_keeps_safe(self, stock_client, server_module):
|
||||
r = stock_client.post(
|
||||
"/config/dump",
|
||||
json={"type": "CrawlerRunConfig", "params": {"css_selector": ".x", "bogus_field": 1}},
|
||||
headers=self._auth(server_module),
|
||||
)
|
||||
assert r.status_code == 200, r.status_code
|
||||
@@ -0,0 +1,82 @@
|
||||
"""
|
||||
R3 webhook connection-pinning tests (real loopback server, offline).
|
||||
|
||||
The webhook sender pins the connection to the validated IP (aiohttp resolver),
|
||||
follows redirects manually, and re-validates every hop. We stub the egress
|
||||
broker so a "good" host pins to the loopback test server and "internal" hosts /
|
||||
redirects are refused.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
import egress_broker
|
||||
from egress_broker import EgressBlocked, PinnedTarget
|
||||
from webhook import WebhookDeliveryService
|
||||
|
||||
pytestmark = pytest.mark.posture
|
||||
|
||||
|
||||
async def _raw_server(response: bytes):
|
||||
async def handle(reader, writer):
|
||||
await reader.read(65536)
|
||||
writer.write(response)
|
||||
await writer.drain()
|
||||
writer.close()
|
||||
server = await asyncio.start_server(handle, "127.0.0.1", 0)
|
||||
return server, server.sockets[0].getsockname()[1]
|
||||
|
||||
|
||||
def _svc():
|
||||
return WebhookDeliveryService({"webhooks": {"retry": {"max_attempts": 2,
|
||||
"initial_delay_ms": 1, "timeout_ms": 5000}}})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestWebhookPinning:
|
||||
async def test_delivered_to_pinned_host(self, monkeypatch):
|
||||
server, port = await _raw_server(b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\nok")
|
||||
monkeypatch.setattr(
|
||||
egress_broker, "resolve_and_pin",
|
||||
lambda url: PinnedTarget("http", "good.example", port, "127.0.0.1"),
|
||||
)
|
||||
try:
|
||||
ok = await _svc().send_webhook(f"http://good.example:{port}/cb", {"x": 1})
|
||||
assert ok is True
|
||||
finally:
|
||||
server.close()
|
||||
|
||||
async def test_internal_target_blocked_no_retry(self, monkeypatch):
|
||||
def boom(url):
|
||||
raise EgressBlocked()
|
||||
monkeypatch.setattr(egress_broker, "resolve_and_pin", boom)
|
||||
ok = await _svc().send_webhook("http://169.254.169.254/cb", {"x": 1})
|
||||
assert ok is False
|
||||
|
||||
async def test_redirect_to_internal_blocked(self, monkeypatch):
|
||||
server, port = await _raw_server(
|
||||
b"HTTP/1.1 302 Found\r\nLocation: http://evil.internal/\r\nContent-Length: 0\r\n\r\n"
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
egress_broker, "resolve_and_pin",
|
||||
lambda url: PinnedTarget("http", "good.example", port, "127.0.0.1"),
|
||||
)
|
||||
|
||||
def check(loc):
|
||||
raise EgressBlocked() # the redirect target is internal
|
||||
monkeypatch.setattr(egress_broker, "check_redirect", check)
|
||||
try:
|
||||
ok = await _svc().send_webhook(f"http://good.example:{port}/cb", {"x": 1})
|
||||
assert ok is False # redirect re-validation refused it
|
||||
finally:
|
||||
server.close()
|
||||
|
||||
async def test_uses_pinned_resolver(self):
|
||||
# The resolver returns the pinned IP for any host (TLS still verifies the
|
||||
# hostname); confirm the wiring.
|
||||
from webhook import _PinnedResolver
|
||||
r = _PinnedResolver("good.example", "203.0.113.7")
|
||||
out = await r.resolve("good.example", 443)
|
||||
assert out[0]["host"] == "203.0.113.7"
|
||||
assert out[0]["hostname"] == "good.example"
|
||||
@@ -0,0 +1,434 @@
|
||||
import dns.resolver
|
||||
import logging
|
||||
import yaml
|
||||
import os
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from pathlib import Path
|
||||
from fastapi import Request
|
||||
from typing import Dict, Optional
|
||||
|
||||
class TaskStatus(str, Enum):
|
||||
PROCESSING = "processing"
|
||||
FAILED = "failed"
|
||||
COMPLETED = "completed"
|
||||
|
||||
class FilterType(str, Enum):
|
||||
RAW = "raw"
|
||||
FIT = "fit"
|
||||
BM25 = "bm25"
|
||||
LLM = "llm"
|
||||
|
||||
DEFAULT_CONFIG = {
|
||||
"app": {
|
||||
"title": "Crawl4AI API",
|
||||
"version": "1.0.0",
|
||||
"host": "0.0.0.0",
|
||||
"port": 11235,
|
||||
"reload": False,
|
||||
"workers": 1,
|
||||
"timeout_keep_alive": 300,
|
||||
},
|
||||
"llm": {
|
||||
"provider": "openai/gpt-4o-mini",
|
||||
},
|
||||
"redis": {
|
||||
"host": "localhost",
|
||||
"port": 6379,
|
||||
"db": 0,
|
||||
"password": "",
|
||||
"task_ttl_seconds": 3600,
|
||||
"ssl": False,
|
||||
},
|
||||
"rate_limiting": {
|
||||
"enabled": True,
|
||||
"default_limit": "1000/minute",
|
||||
"trusted_proxies": [],
|
||||
"storage_uri": "memory://",
|
||||
},
|
||||
"security": {
|
||||
"enabled": False,
|
||||
"jwt_enabled": False,
|
||||
"api_token": "",
|
||||
"https_redirect": False,
|
||||
"trusted_hosts": ["*"],
|
||||
"headers": {
|
||||
"x_content_type_options": "nosniff",
|
||||
"x_frame_options": "DENY",
|
||||
"content_security_policy": "default-src 'self'",
|
||||
"strict_transport_security": "max-age=63072000; includeSubDomains",
|
||||
},
|
||||
},
|
||||
"crawler": {
|
||||
"base_config": {"simulate_user": True},
|
||||
"memory_threshold_percent": 95.0,
|
||||
"rate_limiter": {"enabled": True, "base_delay": [1.0, 2.0]},
|
||||
"timeouts": {"stream_init": 30.0, "batch_process": 300.0},
|
||||
"pool": {"max_pages": 40, "idle_ttl_sec": 300},
|
||||
"browser": {
|
||||
"kwargs": {"headless": True, "text_mode": True},
|
||||
"extra_args": [
|
||||
"--no-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-gpu",
|
||||
"--disable-software-rasterizer",
|
||||
"--allow-insecure-localhost",
|
||||
"--ignore-certificate-errors",
|
||||
],
|
||||
},
|
||||
},
|
||||
"logging": {
|
||||
"level": "INFO",
|
||||
"format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
},
|
||||
"observability": {
|
||||
"prometheus": {"enabled": True, "endpoint": "/metrics"},
|
||||
"health_check": {"endpoint": "/health"},
|
||||
},
|
||||
"webhooks": {
|
||||
"enabled": True,
|
||||
"default_url": None,
|
||||
"data_in_payload": False,
|
||||
"retry": {
|
||||
"max_attempts": 5,
|
||||
"initial_delay_ms": 1000,
|
||||
"max_delay_ms": 32000,
|
||||
"timeout_ms": 30000,
|
||||
},
|
||||
"headers": {"User-Agent": "Crawl4AI-Webhook/1.0"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _deep_merge(base: dict, override: dict) -> dict:
|
||||
"""Recursively merge override into base. Override values take precedence."""
|
||||
merged = base.copy()
|
||||
for key, value in override.items():
|
||||
if key in merged and isinstance(merged[key], dict) and isinstance(value, dict):
|
||||
merged[key] = _deep_merge(merged[key], value)
|
||||
else:
|
||||
merged[key] = value
|
||||
return merged
|
||||
|
||||
|
||||
def load_config() -> Dict:
|
||||
"""Load and return application configuration with environment variable overrides."""
|
||||
config_path = Path(__file__).parent / "config.yml"
|
||||
with open(config_path, "r") as config_file:
|
||||
user_config = yaml.safe_load(config_file) or {}
|
||||
|
||||
# Deep-merge user config on top of defaults so missing keys get safe values
|
||||
config = _deep_merge(DEFAULT_CONFIG, user_config)
|
||||
|
||||
for section in DEFAULT_CONFIG:
|
||||
if section not in user_config:
|
||||
logging.warning(
|
||||
f"Config section '{section}' missing from config.yml, using defaults"
|
||||
)
|
||||
|
||||
# Override LLM provider from environment if set
|
||||
llm_provider = os.environ.get("LLM_PROVIDER")
|
||||
if llm_provider:
|
||||
config["llm"]["provider"] = llm_provider
|
||||
logging.info(f"LLM provider overridden from environment: {llm_provider}")
|
||||
|
||||
# Also support direct API key from environment if the provider-specific key isn't set
|
||||
llm_api_key = os.environ.get("LLM_API_KEY")
|
||||
if llm_api_key and "api_key" not in config["llm"]:
|
||||
config["llm"]["api_key"] = llm_api_key
|
||||
logging.info("LLM API key loaded from LLM_API_KEY environment variable")
|
||||
|
||||
# Override Redis task TTL from environment if set
|
||||
redis_task_ttl = os.environ.get("REDIS_TASK_TTL")
|
||||
if redis_task_ttl:
|
||||
try:
|
||||
config["redis"]["task_ttl_seconds"] = int(redis_task_ttl)
|
||||
logging.info(f"Redis task TTL overridden from REDIS_TASK_TTL: {redis_task_ttl}s")
|
||||
except ValueError:
|
||||
logging.warning(f"Invalid REDIS_TASK_TTL value: {redis_task_ttl}, using default")
|
||||
|
||||
return config
|
||||
|
||||
class CRLFSafeFilter(logging.Filter):
|
||||
"""Strip CR/LF/control chars from log records (log-injection / forging).
|
||||
|
||||
A crawl URL or error reflected into a log line could otherwise inject
|
||||
newlines and forge additional log entries.
|
||||
"""
|
||||
|
||||
_BAD = {ord(c): None for c in "\r\n"} | {i: None for i in range(0, 32) if i not in (9,)}
|
||||
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
try:
|
||||
msg = record.getMessage()
|
||||
cleaned = msg.translate(self._BAD)
|
||||
if cleaned != msg:
|
||||
record.msg = cleaned
|
||||
record.args = ()
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
|
||||
|
||||
def setup_logging(config: Dict) -> None:
|
||||
"""Configure application logging with CRLF-safe records."""
|
||||
logging.basicConfig(
|
||||
level=config["logging"]["level"],
|
||||
format=config["logging"]["format"]
|
||||
)
|
||||
crlf = CRLFSafeFilter()
|
||||
for handler in logging.getLogger().handlers:
|
||||
handler.addFilter(crlf)
|
||||
|
||||
def get_base_url(request: Request) -> str:
|
||||
"""Get base URL including scheme and host."""
|
||||
return f"{request.url.scheme}://{request.url.netloc}"
|
||||
|
||||
def is_task_id(value: str) -> bool:
|
||||
"""Check if the value matches task ID pattern."""
|
||||
return value.startswith("llm_") and "_" in value
|
||||
|
||||
def datetime_handler(obj: any) -> Optional[str]:
|
||||
"""Handle datetime serialization for JSON."""
|
||||
if hasattr(obj, 'isoformat'):
|
||||
return obj.isoformat()
|
||||
raise TypeError(f"Object of type {type(obj)} is not JSON serializable")
|
||||
|
||||
def should_cleanup_task(created_at: str, ttl_seconds: int = 3600) -> bool:
|
||||
"""Check if task should be cleaned up based on creation time."""
|
||||
created = datetime.fromisoformat(created_at)
|
||||
return (datetime.now() - created).total_seconds() > ttl_seconds
|
||||
|
||||
def decode_redis_hash(hash_data: Dict[bytes, bytes]) -> Dict[str, str]:
|
||||
"""Decode Redis hash data from bytes to strings."""
|
||||
return {k.decode('utf-8'): v.decode('utf-8') for k, v in hash_data.items()}
|
||||
|
||||
|
||||
def get_redis_task_ttl(config: Dict) -> int:
|
||||
"""Get Redis task TTL in seconds from config.
|
||||
|
||||
Args:
|
||||
config: The application configuration dictionary
|
||||
|
||||
Returns:
|
||||
TTL in seconds (default 3600). Returns 0 if TTL is disabled.
|
||||
"""
|
||||
return config.get("redis", {}).get("task_ttl_seconds", 3600)
|
||||
|
||||
|
||||
def get_llm_api_key(config: Dict, provider: Optional[str] = None) -> Optional[str]:
|
||||
"""Get the appropriate API key based on the LLM provider.
|
||||
|
||||
Args:
|
||||
config: The application configuration dictionary
|
||||
provider: Optional provider override (e.g., "openai/gpt-4")
|
||||
|
||||
Returns:
|
||||
The API key if directly configured, otherwise None to let litellm handle it
|
||||
"""
|
||||
# Check if direct API key is configured (for backward compatibility)
|
||||
if "api_key" in config["llm"]:
|
||||
return config["llm"]["api_key"]
|
||||
|
||||
# Return None - litellm will automatically find the right environment variable
|
||||
return None
|
||||
|
||||
|
||||
def validate_llm_provider(config: Dict, provider: Optional[str] = None) -> tuple[bool, str]:
|
||||
"""Validate that the LLM provider has an associated API key.
|
||||
|
||||
Args:
|
||||
config: The application configuration dictionary
|
||||
provider: Optional provider override (e.g., "openai/gpt-4")
|
||||
|
||||
Returns:
|
||||
Tuple of (is_valid, error_message)
|
||||
"""
|
||||
# If a direct API key is configured, validation passes
|
||||
if "api_key" in config["llm"]:
|
||||
return True, ""
|
||||
|
||||
# Otherwise, trust that litellm will find the appropriate environment variable
|
||||
# We can't easily validate this without reimplementing litellm's logic
|
||||
return True, ""
|
||||
|
||||
|
||||
def get_llm_temperature(config: Dict, provider: Optional[str] = None) -> Optional[float]:
|
||||
"""Get temperature setting based on the LLM provider.
|
||||
|
||||
Priority order:
|
||||
1. Provider-specific environment variable (e.g., OPENAI_TEMPERATURE)
|
||||
2. Global LLM_TEMPERATURE environment variable
|
||||
3. None (to use litellm/provider defaults)
|
||||
|
||||
Args:
|
||||
config: The application configuration dictionary
|
||||
provider: Optional provider override (e.g., "openai/gpt-4")
|
||||
|
||||
Returns:
|
||||
The temperature setting if configured, otherwise None
|
||||
"""
|
||||
# Check provider-specific temperature first
|
||||
if provider:
|
||||
provider_name = provider.split('/')[0].upper()
|
||||
provider_temp = os.environ.get(f"{provider_name}_TEMPERATURE")
|
||||
if provider_temp:
|
||||
try:
|
||||
return float(provider_temp)
|
||||
except ValueError:
|
||||
logging.warning(f"Invalid temperature value for {provider_name}: {provider_temp}")
|
||||
|
||||
# Check global LLM_TEMPERATURE
|
||||
global_temp = os.environ.get("LLM_TEMPERATURE")
|
||||
if global_temp:
|
||||
try:
|
||||
return float(global_temp)
|
||||
except ValueError:
|
||||
logging.warning(f"Invalid global temperature value: {global_temp}")
|
||||
|
||||
# Return None to use litellm/provider defaults
|
||||
return None
|
||||
|
||||
|
||||
def get_llm_base_url(config: Dict, provider: Optional[str] = None) -> Optional[str]:
|
||||
"""Get base URL setting based on the LLM provider.
|
||||
|
||||
Priority order:
|
||||
1. Provider-specific environment variable (e.g., OPENAI_BASE_URL)
|
||||
2. Global LLM_BASE_URL environment variable
|
||||
3. None (to use default endpoints)
|
||||
|
||||
Args:
|
||||
config: The application configuration dictionary
|
||||
provider: Optional provider override (e.g., "openai/gpt-4")
|
||||
|
||||
Returns:
|
||||
The base URL if configured, otherwise None
|
||||
"""
|
||||
# Check provider-specific base URL first
|
||||
if provider:
|
||||
provider_name = provider.split('/')[0].upper()
|
||||
provider_url = os.environ.get(f"{provider_name}_BASE_URL")
|
||||
if provider_url:
|
||||
return provider_url
|
||||
|
||||
# Check global LLM_BASE_URL
|
||||
return os.environ.get("LLM_BASE_URL")
|
||||
|
||||
|
||||
# ── Security utilities ──────────────────────────────────────
|
||||
# validate_output_path / ALLOWED_OUTPUT_DIR were removed: caller-supplied
|
||||
# output paths are no longer accepted (string-only validation was bypassable via
|
||||
# symlink/TOCTOU and sibling-prefix names -> arbitrary write -> RCE). The server
|
||||
# owns all artifact paths now (see artifacts.py).
|
||||
|
||||
|
||||
import ipaddress
|
||||
import socket
|
||||
from urllib.parse import urlparse
|
||||
|
||||
_BLOCKED_NETWORKS = [
|
||||
ipaddress.ip_network("0.0.0.0/8"),
|
||||
ipaddress.ip_network("10.0.0.0/8"),
|
||||
ipaddress.ip_network("100.64.0.0/10"),
|
||||
ipaddress.ip_network("127.0.0.0/8"),
|
||||
ipaddress.ip_network("169.254.0.0/16"),
|
||||
ipaddress.ip_network("172.16.0.0/12"),
|
||||
ipaddress.ip_network("192.0.0.0/24"),
|
||||
ipaddress.ip_network("192.168.0.0/16"),
|
||||
ipaddress.ip_network("198.18.0.0/15"),
|
||||
ipaddress.ip_network("::1/128"),
|
||||
ipaddress.ip_network("fc00::/7"),
|
||||
ipaddress.ip_network("fe80::/10"),
|
||||
]
|
||||
|
||||
_BLOCKED_HOSTNAMES = {
|
||||
"localhost", "metadata.google.internal", "metadata",
|
||||
"kubernetes.default", "kubernetes.default.svc",
|
||||
}
|
||||
|
||||
|
||||
ALLOW_INTERNAL_URLS = os.environ.get("CRAWL4AI_ALLOW_INTERNAL_URLS", "false").lower() == "true"
|
||||
|
||||
|
||||
def validate_url_destination(url: str) -> None:
|
||||
"""Block crawl URLs targeting internal/private networks (SSRF protection).
|
||||
Skipped when CRAWL4AI_ALLOW_INTERNAL_URLS=true.
|
||||
Skipped for raw: URLs (inline HTML, no network fetch)."""
|
||||
if ALLOW_INTERNAL_URLS:
|
||||
return
|
||||
if str(url).startswith(("raw:", "raw://")):
|
||||
return
|
||||
try:
|
||||
validate_webhook_url(url)
|
||||
except ValueError as e:
|
||||
from fastapi import HTTPException
|
||||
raise HTTPException(status_code=400, detail=f"URL blocked (SSRF protection): {e}")
|
||||
|
||||
|
||||
def _expand_ip_candidates(ip):
|
||||
"""Return [ip] plus any IPv4 form wrapped inside the IPv6 address.
|
||||
SSRF guards must check the unwrapped form because ::ffff:127.0.0.1 and
|
||||
::127.0.0.1 route to 127.0.0.1 but would not match IPv4 blocklists directly."""
|
||||
candidates = [ip]
|
||||
if isinstance(ip, ipaddress.IPv6Address):
|
||||
if ip.ipv4_mapped is not None:
|
||||
candidates.append(ip.ipv4_mapped)
|
||||
else:
|
||||
as_int = int(ip)
|
||||
if 0 < as_int < 2**32:
|
||||
candidates.append(ipaddress.IPv4Address(as_int))
|
||||
return candidates
|
||||
|
||||
|
||||
def validate_webhook_url(url: str) -> None:
|
||||
"""Reject webhook/crawl URLs targeting non-global networks (SSRF protection).
|
||||
|
||||
Delegates to the single egress rule (egress_broker: reject any resolved IP
|
||||
where not ip.is_global, including v4-mapped/NAT64/6to4/v4-compat embedded
|
||||
forms). The raised message is intentionally opaque - it never echoes the
|
||||
resolved IP or hostname, so this is not a DNS/oracle leak.
|
||||
"""
|
||||
from egress_broker import resolve_and_pin, EgressBlocked
|
||||
parsed = urlparse(str(url))
|
||||
if not parsed.hostname:
|
||||
raise ValueError("URL must have a valid hostname")
|
||||
try:
|
||||
resolve_and_pin(url)
|
||||
except EgressBlocked:
|
||||
raise ValueError("URL blocked")
|
||||
|
||||
|
||||
def verify_email_domain(email: str) -> bool:
|
||||
try:
|
||||
domain = email.split('@')[1]
|
||||
# Try to resolve MX records for the domain.
|
||||
records = dns.resolver.resolve(domain, 'MX')
|
||||
return True if records else False
|
||||
except Exception as e:
|
||||
return False
|
||||
|
||||
def get_container_memory_percent() -> float:
|
||||
"""Get actual container memory usage vs limit (cgroup v1/v2 aware)."""
|
||||
try:
|
||||
# Try cgroup v2 first
|
||||
usage_path = Path("/sys/fs/cgroup/memory.current")
|
||||
limit_path = Path("/sys/fs/cgroup/memory.max")
|
||||
if not usage_path.exists():
|
||||
# Fall back to cgroup v1
|
||||
usage_path = Path("/sys/fs/cgroup/memory/memory.usage_in_bytes")
|
||||
limit_path = Path("/sys/fs/cgroup/memory/memory.limit_in_bytes")
|
||||
|
||||
usage = int(usage_path.read_text())
|
||||
limit = int(limit_path.read_text())
|
||||
|
||||
# Handle unlimited (v2: "max", v1: > 1e18)
|
||||
if limit > 1e18:
|
||||
import psutil
|
||||
limit = psutil.virtual_memory().total
|
||||
|
||||
return (usage / limit) * 100
|
||||
except:
|
||||
# Non-container or unsupported: fallback to host
|
||||
import psutil
|
||||
return psutil.virtual_memory().percent
|
||||
@@ -0,0 +1,248 @@
|
||||
"""
|
||||
Webhook delivery service for Crawl4AI.
|
||||
|
||||
This module provides webhook notification functionality with exponential backoff retry logic.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
import socket
|
||||
from typing import Dict, List, Optional
|
||||
from datetime import datetime, timezone
|
||||
|
||||
import aiohttp
|
||||
from aiohttp.abc import AbstractResolver
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MAX_WEBHOOK_REDIRECTS = 5
|
||||
|
||||
|
||||
class _WebhookBlocked(Exception):
|
||||
"""Webhook target (or a redirect hop) resolved to a non-global address."""
|
||||
|
||||
|
||||
class _PinnedResolver(AbstractResolver):
|
||||
"""aiohttp resolver that returns a single pre-pinned IP for the target host.
|
||||
|
||||
aiohttp connects to this IP but still performs TLS SNI / certificate
|
||||
verification against the original hostname, so this pins the connection
|
||||
(closing DNS rebinding) without weakening TLS or doing a MITM.
|
||||
"""
|
||||
|
||||
def __init__(self, host: str, ip: str):
|
||||
self._host = host
|
||||
self._ip = ip
|
||||
|
||||
async def resolve(self, host, port=0, family=socket.AF_INET):
|
||||
return [{
|
||||
"hostname": host,
|
||||
"host": self._ip,
|
||||
"port": port,
|
||||
"family": family,
|
||||
"proto": 0,
|
||||
"flags": 0,
|
||||
}]
|
||||
|
||||
async def close(self):
|
||||
pass
|
||||
|
||||
# Webhook request-header policy: user-controlled outbound headers could inject
|
||||
# hop-by-hop / smuggling headers or CRLF. Allow only well-formed names, reject
|
||||
# control chars in values, and deny sensitive/hop-by-hop names.
|
||||
_WEBHOOK_HEADER_NAME = re.compile(r"^[A-Za-z0-9-]{1,64}$")
|
||||
_WEBHOOK_DENY_HEADERS = {
|
||||
"host", "content-length", "transfer-encoding", "connection",
|
||||
"content-type", "proxy-authorization", "authorization", "cookie",
|
||||
"expect", "upgrade", "te", "trailer",
|
||||
}
|
||||
_MAX_WEBHOOK_HEADERS = 20
|
||||
_MAX_WEBHOOK_HEADER_VALUE = 2048
|
||||
|
||||
|
||||
def sanitize_webhook_headers(headers: Optional[Dict[str, str]]) -> Dict[str, str]:
|
||||
"""Validate user-supplied webhook headers; raise ValueError on any bad one."""
|
||||
if not headers:
|
||||
return {}
|
||||
if len(headers) > _MAX_WEBHOOK_HEADERS:
|
||||
raise ValueError("too many webhook headers")
|
||||
clean: Dict[str, str] = {}
|
||||
for name, value in headers.items():
|
||||
if not isinstance(name, str) or not _WEBHOOK_HEADER_NAME.match(name):
|
||||
raise ValueError(f"invalid webhook header name: {name!r}")
|
||||
if name.lower() in _WEBHOOK_DENY_HEADERS:
|
||||
raise ValueError(f"webhook header not allowed: {name}")
|
||||
sval = str(value)
|
||||
if len(sval) > _MAX_WEBHOOK_HEADER_VALUE or any(c in sval for c in "\r\n\x00"):
|
||||
raise ValueError(f"invalid value for webhook header {name}")
|
||||
clean[name] = sval
|
||||
return clean
|
||||
|
||||
|
||||
class WebhookDeliveryService:
|
||||
"""Handles webhook delivery with exponential backoff retry logic."""
|
||||
|
||||
def __init__(self, config: Dict):
|
||||
"""
|
||||
Initialize the webhook delivery service.
|
||||
|
||||
Args:
|
||||
config: Application configuration dictionary containing webhook settings
|
||||
"""
|
||||
self.config = config.get("webhooks", {})
|
||||
self.max_attempts = self.config.get("retry", {}).get("max_attempts", 5)
|
||||
self.initial_delay = self.config.get("retry", {}).get("initial_delay_ms", 1000) / 1000
|
||||
self.max_delay = self.config.get("retry", {}).get("max_delay_ms", 32000) / 1000
|
||||
self.timeout = self.config.get("retry", {}).get("timeout_ms", 30000) / 1000
|
||||
|
||||
async def send_webhook(
|
||||
self,
|
||||
webhook_url: str,
|
||||
payload: Dict,
|
||||
headers: Optional[Dict[str, str]] = None
|
||||
) -> bool:
|
||||
"""
|
||||
Send webhook with exponential backoff retry logic.
|
||||
|
||||
Args:
|
||||
webhook_url: The URL to send the webhook to
|
||||
payload: The JSON payload to send
|
||||
headers: Optional custom headers
|
||||
|
||||
Returns:
|
||||
bool: True if delivered successfully, False otherwise
|
||||
"""
|
||||
default_headers = self.config.get("headers", {})
|
||||
try:
|
||||
safe_custom = sanitize_webhook_headers(headers)
|
||||
except ValueError as e:
|
||||
# Defense in depth (the schema validator rejects these at request
|
||||
# time); never send a forged/unsafe header.
|
||||
logger.warning(f"Dropping unsafe webhook headers: {e}")
|
||||
safe_custom = {}
|
||||
merged_headers = {**default_headers, **safe_custom}
|
||||
merged_headers["Content-Type"] = "application/json"
|
||||
|
||||
for attempt in range(self.max_attempts):
|
||||
try:
|
||||
status = await self._deliver(webhook_url, payload, merged_headers)
|
||||
if 200 <= status < 300:
|
||||
logger.info("Webhook delivered successfully")
|
||||
return True
|
||||
if status < 500:
|
||||
logger.warning(f"Webhook rejected with status {status}")
|
||||
return False # client error - don't retry
|
||||
logger.warning(f"Webhook failed with status {status}, will retry")
|
||||
except _WebhookBlocked as exc:
|
||||
# SSRF: target (or a redirect hop) resolved non-global. Do not
|
||||
# retry - it will not become safe.
|
||||
logger.warning(f"Webhook blocked (SSRF protection): {exc}")
|
||||
return False
|
||||
except Exception as exc:
|
||||
logger.error(f"Webhook delivery error (attempt {attempt + 1}): {exc}")
|
||||
|
||||
if attempt < self.max_attempts - 1:
|
||||
delay = min(self.initial_delay * (2 ** attempt), self.max_delay)
|
||||
await asyncio.sleep(delay)
|
||||
return False
|
||||
|
||||
async def _deliver(self, url: str, payload: Dict, headers: Dict[str, str]) -> int:
|
||||
"""POST with the connection pinned to the validated IP, following (and
|
||||
re-validating) redirects manually. Returns the final status code."""
|
||||
from egress_broker import resolve_and_pin, check_redirect, EgressBlocked, ALLOW_INSECURE_TLS
|
||||
|
||||
current = url
|
||||
for _hop in range(_MAX_WEBHOOK_REDIRECTS + 1):
|
||||
try:
|
||||
pin = resolve_and_pin(current)
|
||||
except EgressBlocked as e:
|
||||
raise _WebhookBlocked(str(e))
|
||||
|
||||
connector = aiohttp.TCPConnector(resolver=_PinnedResolver(pin.host, pin.ip))
|
||||
ssl = None if not ALLOW_INSECURE_TLS else False
|
||||
timeout = aiohttp.ClientTimeout(total=self.timeout)
|
||||
async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
|
||||
async with session.post(
|
||||
current, json=payload, headers=headers,
|
||||
allow_redirects=False, ssl=ssl,
|
||||
) as resp:
|
||||
if resp.status in (301, 302, 303, 307, 308):
|
||||
loc = resp.headers.get("Location")
|
||||
if not loc:
|
||||
return resp.status
|
||||
# Re-validate every redirect hop before following it.
|
||||
try:
|
||||
check_redirect(loc)
|
||||
except EgressBlocked as e:
|
||||
raise _WebhookBlocked(f"redirect to blocked target: {e}")
|
||||
current = loc
|
||||
continue
|
||||
return resp.status
|
||||
raise _WebhookBlocked("too many webhook redirects")
|
||||
|
||||
logger.error(
|
||||
f"Webhook delivery failed after {self.max_attempts} attempts to {webhook_url}"
|
||||
)
|
||||
return False
|
||||
|
||||
async def notify_job_completion(
|
||||
self,
|
||||
task_id: str,
|
||||
task_type: str,
|
||||
status: str,
|
||||
urls: list,
|
||||
webhook_config: Optional[Dict],
|
||||
result: Optional[Dict] = None,
|
||||
error: Optional[str] = None
|
||||
):
|
||||
"""
|
||||
Notify webhook of job completion.
|
||||
|
||||
Args:
|
||||
task_id: The task identifier
|
||||
task_type: Type of task (e.g., "crawl", "llm_extraction")
|
||||
status: Task status ("completed" or "failed")
|
||||
urls: List of URLs that were crawled
|
||||
webhook_config: Webhook configuration from the job request
|
||||
result: Optional crawl result data
|
||||
error: Optional error message if failed
|
||||
"""
|
||||
# Determine webhook URL
|
||||
webhook_url = None
|
||||
data_in_payload = self.config.get("data_in_payload", False)
|
||||
custom_headers = None
|
||||
|
||||
if webhook_config:
|
||||
webhook_url = webhook_config.get("webhook_url")
|
||||
data_in_payload = webhook_config.get("webhook_data_in_payload", data_in_payload)
|
||||
custom_headers = webhook_config.get("webhook_headers")
|
||||
|
||||
if not webhook_url:
|
||||
webhook_url = self.config.get("default_url")
|
||||
|
||||
if not webhook_url:
|
||||
logger.debug("No webhook URL configured, skipping notification")
|
||||
return
|
||||
|
||||
# Check if webhooks are enabled
|
||||
if not self.config.get("enabled", True):
|
||||
logger.debug("Webhooks are disabled, skipping notification")
|
||||
return
|
||||
|
||||
# Build payload
|
||||
payload = {
|
||||
"task_id": task_id,
|
||||
"task_type": task_type,
|
||||
"status": status,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"urls": urls
|
||||
}
|
||||
|
||||
if error:
|
||||
payload["error"] = error
|
||||
|
||||
if data_in_payload and result:
|
||||
payload["data"] = result
|
||||
|
||||
# Send webhook (fire and forget - don't block on completion)
|
||||
await self.send_webhook(webhook_url, payload, custom_headers)
|
||||
@@ -0,0 +1,110 @@
|
||||
"""
|
||||
work_queue.py - bounded background-job execution with per-principal quotas.
|
||||
|
||||
/crawl/job and /llm/job used FastAPI BackgroundTasks with no bound: a client
|
||||
could enqueue unlimited background jobs and exhaust memory / browser slots, and
|
||||
one caller could starve others.
|
||||
|
||||
This replaces that with a fixed worker pool draining an asyncio.Queue, plus an
|
||||
optional per-principal concurrency cap. Everything is configurable, and any
|
||||
limit set to 0 (or null) means "unbounded" - i.e. the previous behavior is fully
|
||||
recoverable:
|
||||
|
||||
limits.queue.maxsize 0 => unbounded queue (never 503)
|
||||
limits.queue.workers worker pool size (>=1)
|
||||
limits.queue.per_principal 0 => no per-caller cap (never 429)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Awaitable, Callable, Dict, Optional
|
||||
|
||||
logger = logging.getLogger("crawl4ai.workqueue")
|
||||
|
||||
JobFactory = Callable[[], Awaitable[None]]
|
||||
|
||||
|
||||
class QueueFull(Exception):
|
||||
"""The bounded job queue is full -> 503 Retry-After."""
|
||||
|
||||
|
||||
class QuotaExceeded(Exception):
|
||||
"""The principal has too many concurrent jobs -> 429."""
|
||||
|
||||
|
||||
class WorkQueue:
|
||||
def __init__(self, maxsize: int = 0, workers: int = 4, per_principal: int = 0):
|
||||
self.maxsize = max(0, int(maxsize)) # 0 = unbounded
|
||||
self.workers = max(1, int(workers))
|
||||
self.per_principal = max(0, int(per_principal)) # 0 = unlimited
|
||||
self._q: Optional[asyncio.Queue] = None
|
||||
self._tasks: list = []
|
||||
self._counts: Dict[str, int] = {}
|
||||
|
||||
@property
|
||||
def started(self) -> bool:
|
||||
return self._q is not None
|
||||
|
||||
async def start(self) -> None:
|
||||
self._q = asyncio.Queue(maxsize=self.maxsize)
|
||||
self._tasks = [asyncio.create_task(self._worker()) for _ in range(self.workers)]
|
||||
logger.info(
|
||||
"work queue started (maxsize=%s, workers=%s, per_principal=%s)",
|
||||
self.maxsize or "unbounded", self.workers, self.per_principal or "unlimited",
|
||||
)
|
||||
|
||||
async def stop(self) -> None:
|
||||
for t in self._tasks:
|
||||
t.cancel()
|
||||
self._tasks = []
|
||||
self._q = None
|
||||
|
||||
async def _worker(self) -> None:
|
||||
assert self._q is not None
|
||||
while True:
|
||||
principal, factory = await self._q.get()
|
||||
try:
|
||||
await factory()
|
||||
except Exception:
|
||||
logger.exception("background job failed")
|
||||
finally:
|
||||
self._release(principal)
|
||||
self._q.task_done()
|
||||
|
||||
def _release(self, principal: Optional[str]) -> None:
|
||||
if not principal:
|
||||
return
|
||||
n = self._counts.get(principal, 0) - 1
|
||||
if n <= 0:
|
||||
self._counts.pop(principal, None)
|
||||
else:
|
||||
self._counts[principal] = n
|
||||
|
||||
def submit(self, factory: JobFactory, principal: Optional[str] = None) -> None:
|
||||
"""Enqueue a job. Raises QuotaExceeded / QueueFull (mapped to 429 / 503)."""
|
||||
if self._q is None:
|
||||
raise RuntimeError("work queue not started")
|
||||
if self.per_principal and principal:
|
||||
if self._counts.get(principal, 0) >= self.per_principal:
|
||||
raise QuotaExceeded()
|
||||
try:
|
||||
self._q.put_nowait((principal, factory))
|
||||
except asyncio.QueueFull:
|
||||
raise QueueFull()
|
||||
if principal:
|
||||
self._counts[principal] = self._counts.get(principal, 0) + 1
|
||||
|
||||
|
||||
# Process-wide singleton, set at server boot.
|
||||
_JOB_QUEUE: Optional[WorkQueue] = None
|
||||
|
||||
|
||||
def set_job_queue(q: Optional[WorkQueue]) -> None:
|
||||
global _JOB_QUEUE
|
||||
_JOB_QUEUE = q
|
||||
|
||||
|
||||
def get_job_queue() -> Optional[WorkQueue]:
|
||||
return _JOB_QUEUE
|
||||
Reference in New Issue
Block a user