chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,334 @@
|
||||
"""
|
||||
Unit tests for antibot_detector.is_blocked().
|
||||
|
||||
Tests are organized into:
|
||||
- TRUE POSITIVES: Real block pages that MUST be detected
|
||||
- TRUE NEGATIVES: Legitimate pages that MUST NOT be flagged
|
||||
- EDGE CASES: Boundary conditions
|
||||
"""
|
||||
|
||||
import sys, os
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../.."))
|
||||
|
||||
from crawl4ai.antibot_detector import is_blocked
|
||||
|
||||
PASS = 0
|
||||
FAIL = 0
|
||||
|
||||
def check(name, result, expected_blocked, expected_substr=None):
|
||||
global PASS, FAIL
|
||||
blocked, reason = result
|
||||
ok = blocked == expected_blocked
|
||||
if expected_substr and blocked:
|
||||
ok = ok and expected_substr.lower() in reason.lower()
|
||||
status = "PASS" if ok else "FAIL"
|
||||
if not ok:
|
||||
FAIL += 1
|
||||
print(f" {status}: {name}")
|
||||
print(f" got blocked={blocked}, reason={reason!r}")
|
||||
print(f" expected blocked={expected_blocked}" +
|
||||
(f", substr={expected_substr!r}" if expected_substr else ""))
|
||||
else:
|
||||
PASS += 1
|
||||
if blocked:
|
||||
print(f" {status}: {name} -> {reason}")
|
||||
else:
|
||||
print(f" {status}: {name} -> not blocked")
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# TRUE POSITIVES — real block pages that MUST be detected
|
||||
# =========================================================================
|
||||
print("\n=== TRUE POSITIVES (must detect as blocked) ===\n")
|
||||
|
||||
# --- Akamai ---
|
||||
check("Akamai Reference #",
|
||||
is_blocked(403, '<html><body>Access Denied\nYour request was blocked.\nReference #18.2d351ab8.1557333295.a4e16ab</body></html>'),
|
||||
True, "Akamai")
|
||||
|
||||
check("Akamai Pardon Our Interruption",
|
||||
is_blocked(403, '<html><head><title>Pardon Our Interruption</title></head><body><p>Please verify you are human</p></body></html>'),
|
||||
True, "Pardon")
|
||||
|
||||
check("Akamai 403 short Access Denied",
|
||||
is_blocked(403, '<html><body><h1>Access Denied</h1></body></html>'),
|
||||
True) # Detected via near-empty 403 or Access Denied pattern
|
||||
|
||||
# --- Cloudflare ---
|
||||
check("Cloudflare challenge form",
|
||||
is_blocked(403, '''<html><body>
|
||||
<form id="challenge-form" action="/cdn-cgi/l/chk_jschl?__cf_chl_f_tk=abc123">
|
||||
<input type="hidden" name="jschl_vc" value="test"/>
|
||||
</form></body></html>'''),
|
||||
True, "Cloudflare challenge")
|
||||
|
||||
check("Cloudflare error 1020",
|
||||
is_blocked(403, '''<html><body>
|
||||
<div class="cf-wrapper"><span class="cf-error-code">1020</span></div>
|
||||
<p>Access denied</p></body></html>'''),
|
||||
True, "Cloudflare firewall")
|
||||
|
||||
check("Cloudflare IUAM script",
|
||||
is_blocked(403, '<html><script src="/cdn-cgi/challenge-platform/h/g/orchestrate/jsch/v1"></script></html>'),
|
||||
True, "Cloudflare JS challenge")
|
||||
|
||||
check("Cloudflare Just a moment",
|
||||
is_blocked(403, '<html><head><title>Just a moment...</title></head><body>Checking your browser</body></html>'),
|
||||
True) # Detected via near-empty 403 or Cloudflare pattern
|
||||
|
||||
check("Cloudflare Checking your browser (short 503)",
|
||||
is_blocked(503, '<html><body>Checking your browser before accessing the site.</body></html>'),
|
||||
True, "503")
|
||||
|
||||
# --- PerimeterX ---
|
||||
check("PerimeterX block page",
|
||||
is_blocked(403, '''<html><head><title>Access to This Page Has Been Blocked</title></head>
|
||||
<body><div id="px-captcha"></div>
|
||||
<script>window._pxAppId = 'PX12345';</script></body></html>'''),
|
||||
True, "PerimeterX")
|
||||
|
||||
check("PerimeterX captcha CDN",
|
||||
is_blocked(403, '<html><body><script src="https://captcha.px-cdn.net/PX12345/captcha.js"></script></body></html>'),
|
||||
True, "PerimeterX captcha")
|
||||
|
||||
# --- DataDome ---
|
||||
check("DataDome captcha delivery",
|
||||
is_blocked(403, '''<html><body><script>
|
||||
var dd = {'rt':'i','cid':'AHrlq...','host':'geo.captcha-delivery.com'};
|
||||
</script></body></html>'''),
|
||||
True, "DataDome")
|
||||
|
||||
# --- Imperva/Incapsula ---
|
||||
check("Imperva Incapsula Resource",
|
||||
is_blocked(403, '<html><body><iframe src="/_Incapsula_Resource?incident_id=123&sess_id=abc"></iframe></body></html>'),
|
||||
True, "Imperva")
|
||||
|
||||
check("Imperva incident ID",
|
||||
is_blocked(200, '<html><body>Request unsuccessful. Incapsula incident ID: 12345-67890</body></html>'),
|
||||
True, "Incapsula incident")
|
||||
|
||||
# --- Sucuri ---
|
||||
check("Sucuri firewall",
|
||||
is_blocked(403, '<html><body><h1>Sucuri WebSite Firewall - Access Denied</h1></body></html>'),
|
||||
True, "Sucuri")
|
||||
|
||||
# --- Kasada ---
|
||||
check("Kasada challenge",
|
||||
is_blocked(403, '<html><script>KPSDK.scriptStart = KPSDK.now();</script></html>'),
|
||||
True, "Kasada")
|
||||
|
||||
# --- Reddit / Network Security ---
|
||||
check("Reddit blocked by network security (small)",
|
||||
is_blocked(403, '<html><body>You\'ve been blocked by network security.</body></html>'),
|
||||
True, "Network security block")
|
||||
|
||||
check("Reddit blocked by network security (190KB SPA shell)",
|
||||
is_blocked(403, '<html><body><style>' + 'x' * 180000 + '</style>' +
|
||||
'You\'ve been blocked by network security. Log in to continue.</body></html>'),
|
||||
True, "Network security block")
|
||||
|
||||
check("Network security block on HTTP 200 (buried in large page)",
|
||||
is_blocked(200, '<html><body><style>' + 'a:b;' * 30000 + '</style>' +
|
||||
'<p>blocked by network security</p></body></html>'),
|
||||
True, "Network security block")
|
||||
|
||||
# --- HTTP 429 ---
|
||||
check("HTTP 429 rate limit",
|
||||
is_blocked(429, '<html><body>Rate limit exceeded</body></html>'),
|
||||
True, "429")
|
||||
|
||||
check("HTTP 429 empty body",
|
||||
is_blocked(429, ''),
|
||||
True, "429")
|
||||
|
||||
# --- Empty 200 ---
|
||||
check("HTTP 200 empty page",
|
||||
is_blocked(200, ''),
|
||||
True, "empty")
|
||||
|
||||
check("HTTP 200 whitespace only",
|
||||
is_blocked(200, ' \n\n '),
|
||||
True, "empty")
|
||||
|
||||
# --- 403 near-empty ---
|
||||
check("HTTP 403 near-empty (10 bytes)",
|
||||
is_blocked(403, '<html></html>'),
|
||||
True, "403")
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# TRUE NEGATIVES — legitimate pages that MUST NOT be flagged
|
||||
# =========================================================================
|
||||
print("\n=== TRUE NEGATIVES (must NOT detect as blocked) ===\n")
|
||||
|
||||
# --- Normal pages ---
|
||||
check("Normal 200 page (example.com size)",
|
||||
is_blocked(200, '<html><head><title>Example</title></head><body><p>' + 'x' * 500 + '</p></body></html>'),
|
||||
False)
|
||||
|
||||
check("Normal 200 large page",
|
||||
is_blocked(200, '<html><body>' + '<p>Some content here.</p>\n' * 5000 + '</body></html>'),
|
||||
False)
|
||||
|
||||
# --- Security articles (false positive trap!) ---
|
||||
check("Article about bot detection (large page)",
|
||||
is_blocked(200, '<html><head><title>How to Detect Bots</title></head><body>' +
|
||||
'<h1>How to Detect Bots on Your Website</h1>' +
|
||||
'<p>Anti-bot solutions like DataDome, PerimeterX, and Cloudflare ' +
|
||||
'help detect and block bot traffic. When a bot is detected, ' +
|
||||
'services show a CAPTCHA or Access Denied page. ' +
|
||||
'Common signals include blocked by security warnings.</p>' +
|
||||
'<p>The g-recaptcha and h-captcha widgets are used for challenges.</p>' +
|
||||
'<p>' + 'More article content. ' * 500 + '</p>' +
|
||||
'</body></html>'),
|
||||
False)
|
||||
|
||||
check("DataDome marketing page (large)",
|
||||
is_blocked(200, '<html><body><h1>DataDome Bot Protection</h1>' +
|
||||
'<p>DataDome protects websites from bot attacks. ' +
|
||||
'Our solution detects automated traffic using advanced fingerprinting. ' +
|
||||
'Competitors like PerimeterX use window._pxAppId for tracking.</p>' +
|
||||
'<p>' + 'Marketing content. ' * 1000 + '</p>' +
|
||||
'</body></html>'),
|
||||
False)
|
||||
|
||||
|
||||
# --- Login pages with CAPTCHA (not a block!) ---
|
||||
check("Login page with reCAPTCHA (large page)",
|
||||
is_blocked(200, '<html><head><title>Sign In</title></head><body>' +
|
||||
'<nav>Home | Products | Contact</nav>' +
|
||||
'<form action="/login" method="POST">' +
|
||||
'<input name="email" type="email"/>' +
|
||||
'<input name="password" type="password"/>' +
|
||||
'<div class="g-recaptcha" data-sitekey="abc123"></div>' +
|
||||
'<button type="submit">Sign In</button>' +
|
||||
'</form>' +
|
||||
'<footer>Copyright 2024</footer>' +
|
||||
'<p>' + 'Page content. ' * 500 + '</p>' +
|
||||
'</body></html>'),
|
||||
False)
|
||||
|
||||
check("Signup page with hCaptcha (large page)",
|
||||
is_blocked(200, '<html><body>' +
|
||||
'<h1>Create Account</h1>' +
|
||||
'<form><div class="h-captcha" data-sitekey="xyz"></div></form>' +
|
||||
'<p>' + 'Registration info. ' * 500 + '</p>' +
|
||||
'</body></html>'),
|
||||
False)
|
||||
|
||||
# --- 403 pages — ALL non-data 403 HTML is now treated as blocked ---
|
||||
# Rationale: 403 is never the content the user wants. Even for legitimate
|
||||
# auth errors (Apache/Nginx), the fallback will also get 403 and we report
|
||||
# failure correctly. False positives are cheap; false negatives are catastrophic.
|
||||
check("Apache directory listing denied (403, large-ish)",
|
||||
is_blocked(403, '<html><head><title>403 Forbidden</title></head><body>' +
|
||||
'<h1>Forbidden</h1>' +
|
||||
'<p>You don\'t have permission to access this resource on this server.</p>' +
|
||||
'<hr><address>Apache/2.4.41 (Ubuntu) Server at example.com Port 80</address>' +
|
||||
'<p>' + 'Server info. ' * 500 + '</p>' +
|
||||
'</body></html>'),
|
||||
True, "403")
|
||||
|
||||
check("Nginx 403 (large page)",
|
||||
is_blocked(403, '<html><head><title>403 Forbidden</title></head><body>' +
|
||||
'<center><h1>403 Forbidden</h1></center>' +
|
||||
'<hr><center>nginx/1.18.0</center>' +
|
||||
'<p>' + 'Content. ' * 500 + '</p>' +
|
||||
'</body></html>'),
|
||||
True, "403")
|
||||
|
||||
check("API 403 auth required (JSON)",
|
||||
is_blocked(403, '{"error": "Forbidden", "message": "Invalid API key", "code": 403}'),
|
||||
False)
|
||||
|
||||
# --- Cloudflare-served normal pages (not blocked!) ---
|
||||
check("Cloudflare-served normal page with footer",
|
||||
is_blocked(200, '<html><body>' +
|
||||
'<h1>Welcome to Our Site</h1>' +
|
||||
'<p>This is a normal page served through Cloudflare CDN.</p>' +
|
||||
'<footer>Performance & security by Cloudflare</footer>' +
|
||||
'<p>' + 'Normal content. ' * 500 + '</p>' +
|
||||
'</body></html>'),
|
||||
False)
|
||||
|
||||
# --- Small but legitimate pages ---
|
||||
check("Small valid 200 page (with content element)",
|
||||
is_blocked(200, '<html><head><title>OK</title></head><body><p>Your request was processed successfully. Everything is fine.</p></body></html>'),
|
||||
False)
|
||||
|
||||
check("Small JSON 200 response",
|
||||
is_blocked(200, '{"status": "ok", "data": {"id": 123, "name": "test"}, "timestamp": "2024-01-01T00:00:00Z"}'),
|
||||
False)
|
||||
|
||||
check("Redirect page 200",
|
||||
is_blocked(200, '<html><head><meta http-equiv="refresh" content="0;url=/dashboard"></head><body><p>Redirecting to your dashboard. Please wait while we prepare your personalized experience.</p></body></html>'),
|
||||
False)
|
||||
|
||||
# --- 503 pages — ALL non-data 503 HTML is now treated as blocked ---
|
||||
# Same rationale as 403: 503 is never desired content. Fallback rescues false positives.
|
||||
check("503 maintenance page (treated as blocked)",
|
||||
is_blocked(503, '<html><body><h1>Service Temporarily Unavailable</h1>' +
|
||||
'<p>We are performing scheduled maintenance. Please try again later.</p>' +
|
||||
'<p>' + 'Maintenance info. ' * 500 + '</p>' +
|
||||
'</body></html>'),
|
||||
True, "503")
|
||||
|
||||
# --- 200 with short but real content ---
|
||||
check("Short thank you page (200, 120 bytes)",
|
||||
is_blocked(200, '<html><body><h1>Thank You!</h1><p>Your order has been placed. Confirmation email sent.</p></body></html>'),
|
||||
False)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# EDGE CASES
|
||||
# =========================================================================
|
||||
print("\n=== EDGE CASES ===\n")
|
||||
|
||||
check("None status code + empty html",
|
||||
is_blocked(None, ''),
|
||||
True, "no <body>")
|
||||
|
||||
check("None status code + block content",
|
||||
is_blocked(None, '<html><body>Reference #18.2d351ab8.1557333295.a4e16ab</body></html>'),
|
||||
True, "Akamai")
|
||||
|
||||
check("200 + tier1 pattern (Imperva deceptive 200)",
|
||||
is_blocked(200, '<html><body>Request unsuccessful. Incapsula incident ID: 555-999</body></html>'),
|
||||
True, "Incapsula")
|
||||
|
||||
check("403 + 4999 bytes (just under threshold)",
|
||||
is_blocked(403, '<html><body>Access Denied' + 'x' * 4950 + '</body></html>'),
|
||||
True, "Access Denied")
|
||||
|
||||
check("403 + 5001 bytes (over old threshold, now blocked)",
|
||||
is_blocked(403, '<html><body>Some error page' + 'x' * 4960 + '</body></html>'),
|
||||
True, "403")
|
||||
|
||||
check("403 + 9999 bytes with generic block text",
|
||||
is_blocked(403, '<html><body>blocked by security' + 'x' * 9950 + '</body></html>'),
|
||||
True, "Blocked by security")
|
||||
|
||||
check("403 + 10001 bytes with generic block text (now detected regardless of size)",
|
||||
is_blocked(403, '<html><body>blocked by security' + 'x' * 9970 + '</body></html>'),
|
||||
True, "Blocked by security")
|
||||
|
||||
check("200 + whitespace-padded but 89 bytes content (above threshold for meaningful)",
|
||||
is_blocked(200, ' ' * 10 + 'x' * 89 + ' ' * 10),
|
||||
True, "empty")
|
||||
|
||||
check("200 + exactly 100 bytes stripped (at threshold, no body = structural fail)",
|
||||
is_blocked(200, 'x' * 100),
|
||||
True, "no <body>")
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# SUMMARY
|
||||
# =========================================================================
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"RESULTS: {PASS} passed, {FAIL} failed out of {PASS + FAIL} tests")
|
||||
print(f"{'=' * 60}")
|
||||
if FAIL > 0:
|
||||
print("SOME TESTS FAILED!")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("ALL TESTS PASSED!")
|
||||
@@ -0,0 +1,112 @@
|
||||
"""
|
||||
Test: Chanel.com anti-bot bypass via crawl4ai
|
||||
|
||||
Requires env vars:
|
||||
MASSIVE_USERNAME — Massive residential proxy username
|
||||
MASSIVE_PASSWORD — Massive residential proxy password
|
||||
|
||||
Optional:
|
||||
--cdp URL Connect to external browser via CDP (e.g. http://localhost:9223)
|
||||
--attempts N Number of attempts per test (default 3)
|
||||
|
||||
Usage:
|
||||
export MASSIVE_USERNAME="your_user"
|
||||
export MASSIVE_PASSWORD="your_pass"
|
||||
.venv/bin/python tests/proxy/test_chanel_cdp_proxy.py
|
||||
.venv/bin/python tests/proxy/test_chanel_cdp_proxy.py --cdp http://localhost:9223
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
import re
|
||||
import tempfile
|
||||
import shutil
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.async_configs import ProxyConfig
|
||||
|
||||
URL = "https://www.chanel.com/us/fashion/handbags/c/1x1x1/"
|
||||
|
||||
MASSIVE_USERNAME = os.environ.get("MASSIVE_USERNAME", "")
|
||||
MASSIVE_PASSWORD = os.environ.get("MASSIVE_PASSWORD", "")
|
||||
MASSIVE_SERVER = "https://network.joinmassive.com:65535"
|
||||
|
||||
|
||||
def get_proxy_config():
|
||||
if not MASSIVE_USERNAME or not MASSIVE_PASSWORD:
|
||||
print("ERROR: Set MASSIVE_USERNAME and MASSIVE_PASSWORD env vars")
|
||||
sys.exit(1)
|
||||
return ProxyConfig(
|
||||
server=MASSIVE_SERVER,
|
||||
username=MASSIVE_USERNAME,
|
||||
password=MASSIVE_PASSWORD,
|
||||
)
|
||||
|
||||
|
||||
async def test_isolated_context(cdp_url: str = None, attempts: int = 3):
|
||||
"""Test with isolated context (works with both Playwright and CDP)."""
|
||||
mode = f"CDP ({cdp_url})" if cdp_url else "Playwright Chromium"
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Mode: Isolated context — {mode}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
kwargs = dict(
|
||||
enable_stealth=True,
|
||||
create_isolated_context=True,
|
||||
viewport_width=1920,
|
||||
viewport_height=1080,
|
||||
)
|
||||
if cdp_url:
|
||||
kwargs["cdp_url"] = cdp_url
|
||||
else:
|
||||
kwargs["headless"] = True
|
||||
|
||||
config = BrowserConfig(**kwargs)
|
||||
run_config = CrawlerRunConfig(
|
||||
magic=True,
|
||||
simulate_user=True,
|
||||
override_navigator=True,
|
||||
proxy_config=get_proxy_config(),
|
||||
page_timeout=120000,
|
||||
wait_until="load",
|
||||
delay_before_return_html=15.0,
|
||||
)
|
||||
|
||||
passed = 0
|
||||
async with AsyncWebCrawler(config=config) as crawler:
|
||||
for i in range(attempts):
|
||||
result = await crawler.arun(URL, config=run_config)
|
||||
ok = result.status_code == 200 and len(result.html) > 10000
|
||||
title = ""
|
||||
if ok:
|
||||
passed += 1
|
||||
m = re.search(r"<title>(.*?)</title>", result.html)
|
||||
title = f" title={m.group(1)}" if m else ""
|
||||
print(f" Attempt {i+1}: status={result.status_code} html={len(result.html):>10,} bytes {'PASS' if ok else 'FAIL'}{title}")
|
||||
|
||||
print(f"\nResult: {passed}/{attempts} passed")
|
||||
return passed > 0
|
||||
|
||||
|
||||
async def main():
|
||||
cdp_url = None
|
||||
attempts = 3
|
||||
|
||||
args = sys.argv[1:]
|
||||
for j, arg in enumerate(args):
|
||||
if arg == "--cdp" and j + 1 < len(args):
|
||||
cdp_url = args[j + 1]
|
||||
if arg == "--attempts" and j + 1 < len(args):
|
||||
attempts = int(args[j + 1])
|
||||
|
||||
ok = await test_isolated_context(cdp_url=cdp_url, attempts=attempts)
|
||||
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Result: {'PASS' if ok else 'FAIL'}")
|
||||
print(f"{'='*60}")
|
||||
return ok
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
ok = asyncio.run(main())
|
||||
sys.exit(0 if ok else 1)
|
||||
@@ -0,0 +1,68 @@
|
||||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.async_configs import ProxyConfig
|
||||
|
||||
|
||||
async def crawl_chanel(url: str):
|
||||
profile_dir = os.path.expanduser(f"~/.crawl4ai/chanel_{uuid.uuid4().hex[:8]}")
|
||||
os.makedirs(profile_dir, exist_ok=True)
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
enable_stealth=True,
|
||||
use_persistent_context=True,
|
||||
user_data_dir=profile_dir,
|
||||
viewport_width=1920,
|
||||
viewport_height=1080,
|
||||
user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
||||
headers={
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
},
|
||||
proxy_config=ProxyConfig(
|
||||
server="https://network.joinmassive.com:65535",
|
||||
username="mpuQHs4sWZ-country-US",
|
||||
password="D0yWxVQo8wQ05RWqz1Bn",
|
||||
),
|
||||
)
|
||||
|
||||
run_config = CrawlerRunConfig(
|
||||
magic=True,
|
||||
simulate_user=True,
|
||||
override_navigator=True,
|
||||
page_timeout=120000,
|
||||
wait_until="load",
|
||||
delay_before_return_html=10.0,
|
||||
)
|
||||
|
||||
try:
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(url, config=run_config)
|
||||
return result
|
||||
finally:
|
||||
shutil.rmtree(profile_dir, ignore_errors=True)
|
||||
|
||||
|
||||
async def main():
|
||||
url = "https://www.chanel.com/us/fashion/handbags/c/1x1x1/"
|
||||
result = await crawl_chanel(url)
|
||||
print(f"Status: {result.status_code}")
|
||||
print(f"Success: {result.success}")
|
||||
print(f"HTML: {len(result.html):,} bytes")
|
||||
if result.markdown:
|
||||
md_len = len(result.markdown.raw_markdown)
|
||||
print(f"Markdown: {md_len:,} chars")
|
||||
if md_len > 500:
|
||||
print(f"\nFirst 500 chars of markdown:\n{result.markdown.raw_markdown[:500]}")
|
||||
if result.error_message:
|
||||
print(f"Error: {result.error_message}")
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,582 @@
|
||||
"""
|
||||
Comprehensive test suite for ProxyConfig in different forms:
|
||||
1. String form (ip:port:username:password)
|
||||
2. Dict form (dictionary with keys)
|
||||
3. Object form (ProxyConfig instance)
|
||||
4. Environment variable form (from env vars)
|
||||
|
||||
Tests cover all possible scenarios and edge cases using pytest.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import pytest
|
||||
import tempfile
|
||||
from unittest.mock import patch
|
||||
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig
|
||||
from crawl4ai.async_configs import CrawlerRunConfig, ProxyConfig
|
||||
from crawl4ai.cache_context import CacheMode
|
||||
|
||||
|
||||
class TestProxyConfig:
|
||||
"""Comprehensive test suite for ProxyConfig functionality."""
|
||||
|
||||
# Test data for different scenarios
|
||||
# get free proxy server from from webshare.io https://www.webshare.io/?referral_code=3sqog0y1fvsl
|
||||
TEST_PROXY_DATA = {
|
||||
"server": "",
|
||||
"username": "",
|
||||
"password": "",
|
||||
"ip": ""
|
||||
}
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.test_url = "https://httpbin.org/ip" # Use httpbin for testing
|
||||
|
||||
# ==================== OBJECT FORM TESTS ====================
|
||||
|
||||
def test_proxy_config_object_creation_basic(self):
|
||||
"""Test basic ProxyConfig object creation."""
|
||||
proxy = ProxyConfig(server="127.0.0.1:8080")
|
||||
assert proxy.server == "127.0.0.1:8080"
|
||||
assert proxy.username is None
|
||||
assert proxy.password is None
|
||||
assert proxy.ip == "127.0.0.1" # Should auto-extract IP
|
||||
|
||||
def test_proxy_config_object_creation_full(self):
|
||||
"""Test ProxyConfig object creation with all parameters."""
|
||||
proxy = ProxyConfig(
|
||||
server=f"http://{self.TEST_PROXY_DATA['server']}",
|
||||
username=self.TEST_PROXY_DATA['username'],
|
||||
password=self.TEST_PROXY_DATA['password'],
|
||||
ip=self.TEST_PROXY_DATA['ip']
|
||||
)
|
||||
assert proxy.server == f"http://{self.TEST_PROXY_DATA['server']}"
|
||||
assert proxy.username == self.TEST_PROXY_DATA['username']
|
||||
assert proxy.password == self.TEST_PROXY_DATA['password']
|
||||
assert proxy.ip == self.TEST_PROXY_DATA['ip']
|
||||
|
||||
def test_proxy_config_object_ip_extraction(self):
|
||||
"""Test automatic IP extraction from server URL."""
|
||||
test_cases = [
|
||||
("http://192.168.1.1:8080", "192.168.1.1"),
|
||||
("https://10.0.0.1:3128", "10.0.0.1"),
|
||||
("192.168.1.100:8080", "192.168.1.100"),
|
||||
("proxy.example.com:8080", "proxy.example.com"),
|
||||
]
|
||||
|
||||
for server, expected_ip in test_cases:
|
||||
proxy = ProxyConfig(server=server)
|
||||
assert proxy.ip == expected_ip, f"Failed for server: {server}"
|
||||
|
||||
def test_proxy_config_object_invalid_server(self):
|
||||
"""Test ProxyConfig with invalid server formats."""
|
||||
# Should not raise exception but may not extract IP properly
|
||||
proxy = ProxyConfig(server="invalid-format")
|
||||
assert proxy.server == "invalid-format"
|
||||
# IP extraction might fail but object should still be created
|
||||
|
||||
# ==================== DICT FORM TESTS ====================
|
||||
|
||||
def test_proxy_config_from_dict_basic(self):
|
||||
"""Test creating ProxyConfig from basic dictionary."""
|
||||
proxy_dict = {"server": "127.0.0.1:8080"}
|
||||
proxy = ProxyConfig.from_dict(proxy_dict)
|
||||
assert proxy.server == "127.0.0.1:8080"
|
||||
assert proxy.username is None
|
||||
assert proxy.password is None
|
||||
|
||||
def test_proxy_config_from_dict_full(self):
|
||||
"""Test creating ProxyConfig from complete dictionary."""
|
||||
proxy_dict = {
|
||||
"server": f"http://{self.TEST_PROXY_DATA['server']}",
|
||||
"username": self.TEST_PROXY_DATA['username'],
|
||||
"password": self.TEST_PROXY_DATA['password'],
|
||||
"ip": self.TEST_PROXY_DATA['ip']
|
||||
}
|
||||
proxy = ProxyConfig.from_dict(proxy_dict)
|
||||
assert proxy.server == proxy_dict["server"]
|
||||
assert proxy.username == proxy_dict["username"]
|
||||
assert proxy.password == proxy_dict["password"]
|
||||
assert proxy.ip == proxy_dict["ip"]
|
||||
|
||||
def test_proxy_config_from_dict_missing_keys(self):
|
||||
"""Test creating ProxyConfig from dictionary with missing keys."""
|
||||
proxy_dict = {"server": "127.0.0.1:8080", "username": "user"}
|
||||
proxy = ProxyConfig.from_dict(proxy_dict)
|
||||
assert proxy.server == "127.0.0.1:8080"
|
||||
assert proxy.username == "user"
|
||||
assert proxy.password is None
|
||||
assert proxy.ip == "127.0.0.1" # Should auto-extract
|
||||
|
||||
def test_proxy_config_from_dict_empty(self):
|
||||
"""Test creating ProxyConfig from empty dictionary."""
|
||||
proxy_dict = {}
|
||||
proxy = ProxyConfig.from_dict(proxy_dict)
|
||||
assert proxy.server is None
|
||||
assert proxy.username is None
|
||||
assert proxy.password is None
|
||||
assert proxy.ip is None
|
||||
|
||||
def test_proxy_config_from_dict_none_values(self):
|
||||
"""Test creating ProxyConfig from dictionary with None values."""
|
||||
proxy_dict = {
|
||||
"server": "127.0.0.1:8080",
|
||||
"username": None,
|
||||
"password": None,
|
||||
"ip": None
|
||||
}
|
||||
proxy = ProxyConfig.from_dict(proxy_dict)
|
||||
assert proxy.server == "127.0.0.1:8080"
|
||||
assert proxy.username is None
|
||||
assert proxy.password is None
|
||||
assert proxy.ip == "127.0.0.1" # Should auto-extract despite None
|
||||
|
||||
# ==================== STRING FORM TESTS ====================
|
||||
|
||||
def test_proxy_config_from_string_full_format(self):
|
||||
"""Test creating ProxyConfig from full string format (ip:port:username:password)."""
|
||||
proxy_str = f"{self.TEST_PROXY_DATA['ip']}:6114:{self.TEST_PROXY_DATA['username']}:{self.TEST_PROXY_DATA['password']}"
|
||||
proxy = ProxyConfig.from_string(proxy_str)
|
||||
assert proxy.server == f"http://{self.TEST_PROXY_DATA['ip']}:6114"
|
||||
assert proxy.username == self.TEST_PROXY_DATA['username']
|
||||
assert proxy.password == self.TEST_PROXY_DATA['password']
|
||||
assert proxy.ip == self.TEST_PROXY_DATA['ip']
|
||||
|
||||
def test_proxy_config_from_string_ip_port_only(self):
|
||||
"""Test creating ProxyConfig from string with only ip:port."""
|
||||
proxy_str = "192.168.1.1:8080"
|
||||
proxy = ProxyConfig.from_string(proxy_str)
|
||||
assert proxy.server == "http://192.168.1.1:8080"
|
||||
assert proxy.username is None
|
||||
assert proxy.password is None
|
||||
assert proxy.ip == "192.168.1.1"
|
||||
|
||||
def test_proxy_config_from_string_invalid_format(self):
|
||||
"""Test creating ProxyConfig from invalid string formats."""
|
||||
invalid_formats = [
|
||||
"invalid",
|
||||
"ip:port:user", # Missing password (3 parts)
|
||||
"ip:port:user:pass:extra", # Too many parts (5 parts)
|
||||
"",
|
||||
"::", # Empty parts but 3 total (invalid)
|
||||
"::::", # Empty parts but 5 total (invalid)
|
||||
]
|
||||
|
||||
for proxy_str in invalid_formats:
|
||||
with pytest.raises(ValueError, match="Invalid proxy string format"):
|
||||
ProxyConfig.from_string(proxy_str)
|
||||
|
||||
def test_proxy_config_from_string_edge_cases_that_work(self):
|
||||
"""Test string formats that should work but might be edge cases."""
|
||||
# These cases actually work as valid formats
|
||||
edge_cases = [
|
||||
(":", "http://:", ""), # ip:port format with empty values
|
||||
(":::", "http://:", ""), # ip:port:user:pass format with empty values
|
||||
]
|
||||
|
||||
for proxy_str, expected_server, expected_ip in edge_cases:
|
||||
proxy = ProxyConfig.from_string(proxy_str)
|
||||
assert proxy.server == expected_server
|
||||
assert proxy.ip == expected_ip
|
||||
|
||||
def test_proxy_config_from_string_edge_cases(self):
|
||||
"""Test string parsing edge cases."""
|
||||
# Test with different port numbers
|
||||
proxy_str = "10.0.0.1:3128:user:pass"
|
||||
proxy = ProxyConfig.from_string(proxy_str)
|
||||
assert proxy.server == "http://10.0.0.1:3128"
|
||||
|
||||
# Test with special characters in credentials
|
||||
proxy_str = "10.0.0.1:8080:user@domain:pass:word"
|
||||
with pytest.raises(ValueError): # Should fail due to extra colon in password
|
||||
ProxyConfig.from_string(proxy_str)
|
||||
|
||||
# ==================== ENVIRONMENT VARIABLE TESTS ====================
|
||||
|
||||
def test_proxy_config_from_env_single_proxy(self):
|
||||
"""Test loading single proxy from environment variable."""
|
||||
proxy_str = f"{self.TEST_PROXY_DATA['ip']}:6114:{self.TEST_PROXY_DATA['username']}:{self.TEST_PROXY_DATA['password']}"
|
||||
|
||||
with patch.dict(os.environ, {'TEST_PROXIES': proxy_str}):
|
||||
proxies = ProxyConfig.from_env('TEST_PROXIES')
|
||||
assert len(proxies) == 1
|
||||
proxy = proxies[0]
|
||||
assert proxy.ip == self.TEST_PROXY_DATA['ip']
|
||||
assert proxy.username == self.TEST_PROXY_DATA['username']
|
||||
assert proxy.password == self.TEST_PROXY_DATA['password']
|
||||
|
||||
def test_proxy_config_from_env_multiple_proxies(self):
|
||||
"""Test loading multiple proxies from environment variable."""
|
||||
proxy_list = [
|
||||
"192.168.1.1:8080:user1:pass1",
|
||||
"192.168.1.2:8080:user2:pass2",
|
||||
"10.0.0.1:3128" # No auth
|
||||
]
|
||||
proxy_str = ",".join(proxy_list)
|
||||
|
||||
with patch.dict(os.environ, {'TEST_PROXIES': proxy_str}):
|
||||
proxies = ProxyConfig.from_env('TEST_PROXIES')
|
||||
assert len(proxies) == 3
|
||||
|
||||
# Check first proxy
|
||||
assert proxies[0].ip == "192.168.1.1"
|
||||
assert proxies[0].username == "user1"
|
||||
assert proxies[0].password == "pass1"
|
||||
|
||||
# Check second proxy
|
||||
assert proxies[1].ip == "192.168.1.2"
|
||||
assert proxies[1].username == "user2"
|
||||
assert proxies[1].password == "pass2"
|
||||
|
||||
# Check third proxy (no auth)
|
||||
assert proxies[2].ip == "10.0.0.1"
|
||||
assert proxies[2].username is None
|
||||
assert proxies[2].password is None
|
||||
|
||||
def test_proxy_config_from_env_empty_var(self):
|
||||
"""Test loading from empty environment variable."""
|
||||
with patch.dict(os.environ, {'TEST_PROXIES': ''}):
|
||||
proxies = ProxyConfig.from_env('TEST_PROXIES')
|
||||
assert len(proxies) == 0
|
||||
|
||||
def test_proxy_config_from_env_missing_var(self):
|
||||
"""Test loading from missing environment variable."""
|
||||
# Ensure the env var doesn't exist
|
||||
with patch.dict(os.environ, {}, clear=True):
|
||||
proxies = ProxyConfig.from_env('NON_EXISTENT_VAR')
|
||||
assert len(proxies) == 0
|
||||
|
||||
def test_proxy_config_from_env_with_empty_entries(self):
|
||||
"""Test loading proxies with empty entries in the list."""
|
||||
proxy_str = "192.168.1.1:8080:user:pass,,10.0.0.1:3128,"
|
||||
|
||||
with patch.dict(os.environ, {'TEST_PROXIES': proxy_str}):
|
||||
proxies = ProxyConfig.from_env('TEST_PROXIES')
|
||||
assert len(proxies) == 2 # Empty entries should be skipped
|
||||
assert proxies[0].ip == "192.168.1.1"
|
||||
assert proxies[1].ip == "10.0.0.1"
|
||||
|
||||
def test_proxy_config_from_env_with_invalid_entries(self):
|
||||
"""Test loading proxies with some invalid entries."""
|
||||
proxy_str = "192.168.1.1:8080:user:pass,invalid_proxy,10.0.0.1:3128"
|
||||
|
||||
with patch.dict(os.environ, {'TEST_PROXIES': proxy_str}):
|
||||
# Should handle errors gracefully and return valid proxies
|
||||
proxies = ProxyConfig.from_env('TEST_PROXIES')
|
||||
# Depending on implementation, might return partial list or empty
|
||||
# This tests error handling
|
||||
assert isinstance(proxies, list)
|
||||
|
||||
# ==================== SERIALIZATION TESTS ====================
|
||||
|
||||
def test_proxy_config_to_dict(self):
|
||||
"""Test converting ProxyConfig to dictionary."""
|
||||
proxy = ProxyConfig(
|
||||
server=f"http://{self.TEST_PROXY_DATA['server']}",
|
||||
username=self.TEST_PROXY_DATA['username'],
|
||||
password=self.TEST_PROXY_DATA['password'],
|
||||
ip=self.TEST_PROXY_DATA['ip']
|
||||
)
|
||||
|
||||
result_dict = proxy.to_dict()
|
||||
expected = {
|
||||
"server": f"http://{self.TEST_PROXY_DATA['server']}",
|
||||
"username": self.TEST_PROXY_DATA['username'],
|
||||
"password": self.TEST_PROXY_DATA['password'],
|
||||
"ip": self.TEST_PROXY_DATA['ip']
|
||||
}
|
||||
assert result_dict == expected
|
||||
|
||||
def test_proxy_config_clone(self):
|
||||
"""Test cloning ProxyConfig with modifications."""
|
||||
original = ProxyConfig(
|
||||
server="http://127.0.0.1:8080",
|
||||
username="user",
|
||||
password="pass"
|
||||
)
|
||||
|
||||
# Clone with modifications
|
||||
cloned = original.clone(username="new_user", password="new_pass")
|
||||
|
||||
# Original should be unchanged
|
||||
assert original.username == "user"
|
||||
assert original.password == "pass"
|
||||
|
||||
# Clone should have new values
|
||||
assert cloned.username == "new_user"
|
||||
assert cloned.password == "new_pass"
|
||||
assert cloned.server == original.server # Unchanged value
|
||||
|
||||
def test_proxy_config_roundtrip_serialization(self):
|
||||
"""Test that ProxyConfig can be serialized and deserialized without loss."""
|
||||
original = ProxyConfig(
|
||||
server=f"http://{self.TEST_PROXY_DATA['server']}",
|
||||
username=self.TEST_PROXY_DATA['username'],
|
||||
password=self.TEST_PROXY_DATA['password'],
|
||||
ip=self.TEST_PROXY_DATA['ip']
|
||||
)
|
||||
|
||||
# Serialize to dict and back
|
||||
serialized = original.to_dict()
|
||||
deserialized = ProxyConfig.from_dict(serialized)
|
||||
|
||||
assert deserialized.server == original.server
|
||||
assert deserialized.username == original.username
|
||||
assert deserialized.password == original.password
|
||||
assert deserialized.ip == original.ip
|
||||
|
||||
# ==================== INTEGRATION TESTS ====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawler_with_proxy_config_object(self):
|
||||
"""Test AsyncWebCrawler with ProxyConfig object."""
|
||||
proxy_config = ProxyConfig(
|
||||
server=f"http://{self.TEST_PROXY_DATA['server']}",
|
||||
username=self.TEST_PROXY_DATA['username'],
|
||||
password=self.TEST_PROXY_DATA['password']
|
||||
)
|
||||
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
|
||||
# Test that the crawler accepts the ProxyConfig object without errors
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
try:
|
||||
# Note: This might fail due to actual proxy connection, but should not fail due to config issues
|
||||
result = await crawler.arun(
|
||||
url=self.test_url,
|
||||
config=CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
proxy_config=proxy_config,
|
||||
page_timeout=10000 # Short timeout for testing
|
||||
)
|
||||
)
|
||||
# If we get here, proxy config was accepted
|
||||
assert result is not None
|
||||
except Exception as e:
|
||||
# We expect connection errors with test proxies, but not config errors
|
||||
error_msg = str(e).lower()
|
||||
assert "attribute" not in error_msg, f"Config error: {e}"
|
||||
assert "proxy_config" not in error_msg, f"Proxy config error: {e}"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawler_with_proxy_config_dict(self):
|
||||
"""Test AsyncWebCrawler with ProxyConfig from dictionary."""
|
||||
proxy_dict = {
|
||||
"server": f"http://{self.TEST_PROXY_DATA['server']}",
|
||||
"username": self.TEST_PROXY_DATA['username'],
|
||||
"password": self.TEST_PROXY_DATA['password']
|
||||
}
|
||||
proxy_config = ProxyConfig.from_dict(proxy_dict)
|
||||
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
try:
|
||||
result = await crawler.arun(
|
||||
url=self.test_url,
|
||||
config=CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
proxy_config=proxy_config,
|
||||
page_timeout=10000
|
||||
)
|
||||
)
|
||||
assert result is not None
|
||||
except Exception as e:
|
||||
error_msg = str(e).lower()
|
||||
assert "attribute" not in error_msg, f"Config error: {e}"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawler_with_proxy_config_from_string(self):
|
||||
"""Test AsyncWebCrawler with ProxyConfig from string."""
|
||||
proxy_str = f"{self.TEST_PROXY_DATA['ip']}:6114:{self.TEST_PROXY_DATA['username']}:{self.TEST_PROXY_DATA['password']}"
|
||||
proxy_config = ProxyConfig.from_string(proxy_str)
|
||||
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
try:
|
||||
result = await crawler.arun(
|
||||
url=self.test_url,
|
||||
config=CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
proxy_config=proxy_config,
|
||||
page_timeout=10000
|
||||
)
|
||||
)
|
||||
assert result is not None
|
||||
except Exception as e:
|
||||
error_msg = str(e).lower()
|
||||
assert "attribute" not in error_msg, f"Config error: {e}"
|
||||
|
||||
# ==================== EDGE CASES AND ERROR HANDLING ====================
|
||||
|
||||
def test_proxy_config_with_none_server(self):
|
||||
"""Test ProxyConfig behavior with None server."""
|
||||
proxy = ProxyConfig(server=None)
|
||||
assert proxy.server is None
|
||||
assert proxy.ip is None # Should not crash
|
||||
|
||||
def test_proxy_config_with_empty_string_server(self):
|
||||
"""Test ProxyConfig behavior with empty string server."""
|
||||
proxy = ProxyConfig(server="")
|
||||
assert proxy.server == ""
|
||||
assert proxy.ip is None or proxy.ip == ""
|
||||
|
||||
def test_proxy_config_special_characters_in_credentials(self):
|
||||
"""Test ProxyConfig with special characters in username/password."""
|
||||
special_chars_tests = [
|
||||
("user@domain.com", "pass!@#$%"),
|
||||
("user_123", "p@ssw0rd"),
|
||||
("user-test", "pass-word"),
|
||||
]
|
||||
|
||||
for username, password in special_chars_tests:
|
||||
proxy = ProxyConfig(
|
||||
server="http://127.0.0.1:8080",
|
||||
username=username,
|
||||
password=password
|
||||
)
|
||||
assert proxy.username == username
|
||||
assert proxy.password == password
|
||||
|
||||
def test_proxy_config_unicode_handling(self):
|
||||
"""Test ProxyConfig with unicode characters."""
|
||||
proxy = ProxyConfig(
|
||||
server="http://127.0.0.1:8080",
|
||||
username="ユーザー", # Japanese characters
|
||||
password="пароль" # Cyrillic characters
|
||||
)
|
||||
assert proxy.username == "ユーザー"
|
||||
assert proxy.password == "пароль"
|
||||
|
||||
# ==================== PERFORMANCE TESTS ====================
|
||||
|
||||
def test_proxy_config_creation_performance(self):
|
||||
"""Test that ProxyConfig creation is reasonably fast."""
|
||||
import time
|
||||
|
||||
start_time = time.time()
|
||||
for i in range(1000):
|
||||
proxy = ProxyConfig(
|
||||
server=f"http://192.168.1.{i % 255}:8080",
|
||||
username=f"user{i}",
|
||||
password=f"pass{i}"
|
||||
)
|
||||
end_time = time.time()
|
||||
|
||||
# Should be able to create 1000 configs in less than 1 second
|
||||
assert (end_time - start_time) < 1.0
|
||||
|
||||
def test_proxy_config_from_env_performance(self):
|
||||
"""Test that loading many proxies from env is reasonably fast."""
|
||||
import time
|
||||
|
||||
# Create a large list of proxy strings
|
||||
proxy_list = [f"192.168.1.{i}:8080:user{i}:pass{i}" for i in range(100)]
|
||||
proxy_str = ",".join(proxy_list)
|
||||
|
||||
with patch.dict(os.environ, {'PERF_TEST_PROXIES': proxy_str}):
|
||||
start_time = time.time()
|
||||
proxies = ProxyConfig.from_env('PERF_TEST_PROXIES')
|
||||
end_time = time.time()
|
||||
|
||||
assert len(proxies) == 100
|
||||
# Should be able to parse 100 proxies in less than 1 second
|
||||
assert (end_time - start_time) < 1.0
|
||||
|
||||
|
||||
# ==================== STANDALONE TEST FUNCTIONS ====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dict_proxy():
|
||||
"""Original test function for dict proxy - kept for backward compatibility."""
|
||||
proxy_config = {
|
||||
"server": "23.95.150.145:6114",
|
||||
"username": "cfyswbwn",
|
||||
"password": "1gs266hoqysi"
|
||||
}
|
||||
proxy_config_obj = ProxyConfig.from_dict(proxy_config)
|
||||
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
try:
|
||||
result = await crawler.arun(url="https://httpbin.org/ip", config=CrawlerRunConfig(
|
||||
stream=False,
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
proxy_config=proxy_config_obj,
|
||||
page_timeout=10000
|
||||
))
|
||||
print("Dict proxy test passed!")
|
||||
print(result.markdown[:200] if result and result.markdown else "No result")
|
||||
except Exception as e:
|
||||
print(f"Dict proxy test error (expected): {e}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_string_proxy():
|
||||
"""Test function for string proxy format."""
|
||||
proxy_str = "23.95.150.145:6114:cfyswbwn:1gs266hoqysi"
|
||||
proxy_config_obj = ProxyConfig.from_string(proxy_str)
|
||||
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
try:
|
||||
result = await crawler.arun(url="https://httpbin.org/ip", config=CrawlerRunConfig(
|
||||
stream=False,
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
proxy_config=proxy_config_obj,
|
||||
page_timeout=10000
|
||||
))
|
||||
print("String proxy test passed!")
|
||||
print(result.markdown[:200] if result and result.markdown else "No result")
|
||||
except Exception as e:
|
||||
print(f"String proxy test error (expected): {e}")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_env_proxy():
|
||||
"""Test function for environment variable proxy."""
|
||||
# Set environment variable
|
||||
os.environ['TEST_PROXIES'] = "23.95.150.145:6114:cfyswbwn:1gs266hoqysi"
|
||||
|
||||
proxies = ProxyConfig.from_env('TEST_PROXIES')
|
||||
if proxies:
|
||||
proxy_config_obj = proxies[0] # Use first proxy
|
||||
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
try:
|
||||
result = await crawler.arun(url="https://httpbin.org/ip", config=CrawlerRunConfig(
|
||||
stream=False,
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
proxy_config=proxy_config_obj,
|
||||
page_timeout=10000
|
||||
))
|
||||
print("Environment proxy test passed!")
|
||||
print(result.markdown[:200] if result and result.markdown else "No result")
|
||||
except Exception as e:
|
||||
print(f"Environment proxy test error (expected): {e}")
|
||||
else:
|
||||
print("No proxies loaded from environment")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("Running comprehensive ProxyConfig tests...")
|
||||
print("=" * 50)
|
||||
|
||||
# Run the standalone test functions
|
||||
print("\n1. Testing dict proxy format...")
|
||||
asyncio.run(test_dict_proxy())
|
||||
|
||||
print("\n2. Testing string proxy format...")
|
||||
asyncio.run(test_string_proxy())
|
||||
|
||||
print("\n3. Testing environment variable proxy format...")
|
||||
asyncio.run(test_env_proxy())
|
||||
|
||||
print("\n" + "=" * 50)
|
||||
print("To run the full pytest suite, use: pytest " + __file__)
|
||||
print("=" * 50)
|
||||
@@ -0,0 +1,42 @@
|
||||
import warnings
|
||||
|
||||
import pytest
|
||||
|
||||
from crawl4ai.async_configs import BrowserConfig, ProxyConfig
|
||||
|
||||
|
||||
def test_browser_config_proxy_string_emits_deprecation_and_autoconverts():
|
||||
warnings.simplefilter("always", DeprecationWarning)
|
||||
|
||||
proxy_str = "23.95.150.145:6114:username:password"
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
cfg = BrowserConfig(proxy=proxy_str, headless=True)
|
||||
|
||||
dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)]
|
||||
assert dep_warnings, "Expected DeprecationWarning when using BrowserConfig(proxy=...)"
|
||||
|
||||
assert cfg.proxy is None, "cfg.proxy should be None after auto-conversion"
|
||||
assert isinstance(cfg.proxy_config, ProxyConfig), "cfg.proxy_config should be ProxyConfig instance"
|
||||
assert cfg.proxy_config.username == "username"
|
||||
assert cfg.proxy_config.password == "password"
|
||||
assert cfg.proxy_config.server.startswith("http://")
|
||||
assert cfg.proxy_config.server.endswith(":6114")
|
||||
|
||||
|
||||
def test_browser_config_with_proxy_config_emits_no_deprecation():
|
||||
warnings.simplefilter("always", DeprecationWarning)
|
||||
|
||||
with warnings.catch_warnings(record=True) as caught:
|
||||
cfg = BrowserConfig(
|
||||
headless=True,
|
||||
proxy_config={
|
||||
"server": "http://127.0.0.1:8080",
|
||||
"username": "u",
|
||||
"password": "p",
|
||||
},
|
||||
)
|
||||
|
||||
dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)]
|
||||
assert not dep_warnings, "Did not expect DeprecationWarning when using proxy_config"
|
||||
assert cfg.proxy is None
|
||||
assert isinstance(cfg.proxy_config, ProxyConfig)
|
||||
@@ -0,0 +1,96 @@
|
||||
"""Regression tests for proxy fix:
|
||||
1. Persistent context + proxy (new path via launch_persistent_context)
|
||||
2. Persistent context WITHOUT proxy (should still use launch_persistent_context)
|
||||
3. Non-persistent + proxy on CrawlerRunConfig (existing path, must not break)
|
||||
4. Non-persistent, no proxy (basic crawl, must not break)
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
import shutil
|
||||
import uuid
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.async_configs import ProxyConfig
|
||||
|
||||
TEST_URL = "https://httpbin.org/ip" # Simple endpoint, returns IP
|
||||
|
||||
|
||||
async def test(label, browser_config, run_config=None):
|
||||
print(f"\n{'='*60}")
|
||||
print(f"Test: {label}")
|
||||
print(f"{'='*60}")
|
||||
run_config = run_config or CrawlerRunConfig()
|
||||
try:
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(TEST_URL, config=run_config)
|
||||
print(f" Status: {result.status_code}")
|
||||
print(f" HTML bytes: {len(result.html)}")
|
||||
if result.markdown:
|
||||
# httpbin.org/ip returns JSON with "origin" key
|
||||
md = result.markdown.raw_markdown.strip()
|
||||
print(f" Content: {md[:200]}")
|
||||
if result.error_message:
|
||||
print(f" ERROR: {result.error_message}")
|
||||
return result
|
||||
except Exception as e:
|
||||
print(f" EXCEPTION: {e}")
|
||||
return None
|
||||
|
||||
|
||||
async def main():
|
||||
proxy = ProxyConfig(
|
||||
server="https://network.joinmassive.com:65535",
|
||||
username="mpuQHs4sWZ-country-US",
|
||||
password="D0yWxVQo8wQ05RWqz1Bn",
|
||||
)
|
||||
|
||||
# 1. Persistent context + proxy (the fixed path)
|
||||
pd = os.path.expanduser(f"~/.crawl4ai/test_{uuid.uuid4().hex[:8]}")
|
||||
os.makedirs(pd, exist_ok=True)
|
||||
try:
|
||||
await test(
|
||||
"Persistent + proxy (launch_persistent_context)",
|
||||
BrowserConfig(
|
||||
headless=True,
|
||||
use_persistent_context=True,
|
||||
user_data_dir=pd,
|
||||
proxy_config=proxy,
|
||||
),
|
||||
)
|
||||
finally:
|
||||
shutil.rmtree(pd, ignore_errors=True)
|
||||
|
||||
# 2. Persistent context WITHOUT proxy
|
||||
pd2 = os.path.expanduser(f"~/.crawl4ai/test_{uuid.uuid4().hex[:8]}")
|
||||
os.makedirs(pd2, exist_ok=True)
|
||||
try:
|
||||
await test(
|
||||
"Persistent, no proxy (launch_persistent_context)",
|
||||
BrowserConfig(
|
||||
headless=True,
|
||||
use_persistent_context=True,
|
||||
user_data_dir=pd2,
|
||||
),
|
||||
)
|
||||
finally:
|
||||
shutil.rmtree(pd2, ignore_errors=True)
|
||||
|
||||
# 3. Non-persistent + proxy on CrawlerRunConfig
|
||||
await test(
|
||||
"Non-persistent + proxy on RunConfig",
|
||||
BrowserConfig(headless=True),
|
||||
CrawlerRunConfig(
|
||||
proxy_config=proxy,
|
||||
),
|
||||
)
|
||||
|
||||
# 4. Basic crawl - no proxy, no persistent
|
||||
await test(
|
||||
"Basic crawl (no proxy, no persistent)",
|
||||
BrowserConfig(headless=True),
|
||||
)
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("All regression tests complete.")
|
||||
|
||||
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,109 @@
|
||||
"""
|
||||
Verify proxies are working and check what IPs they resolve to.
|
||||
Then test Chanel through NST proxy (different provider).
|
||||
"""
|
||||
import requests
|
||||
|
||||
# Check our real IP
|
||||
def check_ip(label, proxy=None):
|
||||
print(f"\n--- {label} ---")
|
||||
try:
|
||||
kwargs = {"url": "https://httpbin.org/ip", "timeout": 15}
|
||||
if proxy:
|
||||
kwargs["proxies"] = {"https": proxy, "http": proxy}
|
||||
resp = requests.get(**kwargs)
|
||||
print(f" IP: {resp.json()}")
|
||||
except Exception as e:
|
||||
print(f" ERROR: {e}")
|
||||
|
||||
# Get NST proxy credentials
|
||||
def get_nst_proxy(channel_id):
|
||||
token = "NSTPROXY-DA9C7A614946EA8FCEFDA9FD3B3F4A9D"
|
||||
api_url = f"https://api.nstproxy.com/api/v1/generate/apiproxies?count=1&country=US&protocol=http&sessionDuration=0&channelId={channel_id}&token={token}"
|
||||
print(f"\nFetching NST proxy ({channel_id[:8]}...):")
|
||||
print(f" URL: {api_url}")
|
||||
try:
|
||||
resp = requests.get(api_url, timeout=15)
|
||||
print(f" HTTP {resp.status_code}")
|
||||
print(f" Body: {resp.text[:500]}")
|
||||
data = resp.json()
|
||||
if data.get("code") == 200 and data.get("data"):
|
||||
proxy_str = data["data"][0]
|
||||
parts = proxy_str.split(":")
|
||||
if len(parts) == 4:
|
||||
ip, port, user, pwd = parts
|
||||
proxy_url = f"http://{user}:{pwd}@{ip}:{port}"
|
||||
print(f" Proxy URL: http://{user[:10]}...@{ip}:{port}")
|
||||
return proxy_url
|
||||
except Exception as e:
|
||||
print(f" ERROR: {e}")
|
||||
return None
|
||||
|
||||
# Test Chanel
|
||||
def test_chanel(label, proxy=None, use_cffi=False):
|
||||
url = "https://www.chanel.com/us/fashion/handbags/c/1x1x1/"
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
}
|
||||
print(f"\n{'='*60}")
|
||||
print(f"TEST: {label}")
|
||||
try:
|
||||
if use_cffi:
|
||||
from curl_cffi import requests as cffi_requests
|
||||
kwargs = {"url": url, "headers": headers, "impersonate": "chrome", "timeout": 30, "allow_redirects": True}
|
||||
if proxy:
|
||||
kwargs["proxies"] = {"https": proxy, "http": proxy}
|
||||
resp = cffi_requests.get(**kwargs)
|
||||
else:
|
||||
kwargs = {"url": url, "headers": headers, "timeout": 30, "allow_redirects": True}
|
||||
if proxy:
|
||||
kwargs["proxies"] = {"https": proxy, "http": proxy}
|
||||
resp = requests.get(**kwargs)
|
||||
|
||||
blocked = "Access Denied" in resp.text
|
||||
print(f" Status: {resp.status_code}")
|
||||
print(f" Size: {len(resp.text):,} bytes")
|
||||
print(f" Result: {'BLOCKED' if blocked else 'SUCCESS' if resp.status_code == 200 and len(resp.text) > 10000 else 'UNCLEAR'}")
|
||||
if not blocked and resp.status_code == 200:
|
||||
print(f" First 300 chars: {resp.text[:300]}")
|
||||
except Exception as e:
|
||||
print(f" ERROR: {e}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
MASSIVE_RES = "https://mpuQHs4sWZ-country-US:D0yWxVQo8wQ05RWqz1Bn@network.joinmassive.com:65535"
|
||||
MASSIVE_DC = "http://mpuQHs4sWZ-country-US:D0yWxVQo8wQ05RWqz1Bn@isp.joinmassive.com:8000"
|
||||
|
||||
# Step 1: Verify IPs
|
||||
print("="*60)
|
||||
print("STEP 1: Verify proxy IPs")
|
||||
check_ip("Direct (Hetzner)")
|
||||
check_ip("Massive Residential", MASSIVE_RES)
|
||||
check_ip("Massive Datacenter/ISP", MASSIVE_DC)
|
||||
|
||||
# Step 2: Get NST proxies
|
||||
print("\n" + "="*60)
|
||||
print("STEP 2: Get NST proxy credentials")
|
||||
nst_res = get_nst_proxy("7864DDA266D5899C") # residential
|
||||
nst_dc = get_nst_proxy("AE0C3B5547F8A021") # datacenter
|
||||
|
||||
if nst_res:
|
||||
check_ip("NST Residential", nst_res)
|
||||
if nst_dc:
|
||||
check_ip("NST Datacenter", nst_dc)
|
||||
|
||||
# Step 3: Test Chanel with all available proxies
|
||||
print("\n" + "="*60)
|
||||
print("STEP 3: Test Chanel.com")
|
||||
|
||||
if nst_res:
|
||||
test_chanel("curl_cffi + NST residential", proxy=nst_res, use_cffi=True)
|
||||
test_chanel("plain requests + NST residential", proxy=nst_res, use_cffi=False)
|
||||
|
||||
if nst_dc:
|
||||
test_chanel("curl_cffi + NST datacenter", proxy=nst_dc, use_cffi=True)
|
||||
|
||||
# Also try Massive ISP/datacenter (different from residential)
|
||||
test_chanel("curl_cffi + Massive ISP", proxy=MASSIVE_DC, use_cffi=True)
|
||||
@@ -0,0 +1,569 @@
|
||||
"""
|
||||
Comprehensive test suite for Sticky Proxy Sessions functionality.
|
||||
|
||||
Tests cover:
|
||||
1. Basic sticky session - same proxy for same session_id
|
||||
2. Different sessions get different proxies
|
||||
3. Session release
|
||||
4. TTL expiration
|
||||
5. Thread safety / concurrent access
|
||||
6. Integration tests with AsyncWebCrawler
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import time
|
||||
import pytest
|
||||
from unittest.mock import patch
|
||||
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig
|
||||
from crawl4ai.async_configs import CrawlerRunConfig, ProxyConfig
|
||||
from crawl4ai.proxy_strategy import RoundRobinProxyStrategy
|
||||
from crawl4ai.cache_context import CacheMode
|
||||
|
||||
|
||||
class TestRoundRobinProxyStrategySession:
|
||||
"""Test suite for RoundRobinProxyStrategy session methods."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.proxies = [
|
||||
ProxyConfig(server=f"http://proxy{i}.test:8080")
|
||||
for i in range(5)
|
||||
]
|
||||
|
||||
# ==================== BASIC STICKY SESSION TESTS ====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sticky_session_same_proxy(self):
|
||||
"""Verify same proxy is returned for same session_id."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
# First call - acquires proxy
|
||||
proxy1 = await strategy.get_proxy_for_session("session-1")
|
||||
|
||||
# Second call - should return same proxy
|
||||
proxy2 = await strategy.get_proxy_for_session("session-1")
|
||||
|
||||
# Third call - should return same proxy
|
||||
proxy3 = await strategy.get_proxy_for_session("session-1")
|
||||
|
||||
assert proxy1 is not None
|
||||
assert proxy1.server == proxy2.server == proxy3.server
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_different_sessions_different_proxies(self):
|
||||
"""Verify different session_ids can get different proxies."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
proxy_a = await strategy.get_proxy_for_session("session-a")
|
||||
proxy_b = await strategy.get_proxy_for_session("session-b")
|
||||
proxy_c = await strategy.get_proxy_for_session("session-c")
|
||||
|
||||
# All should be different (round-robin)
|
||||
servers = {proxy_a.server, proxy_b.server, proxy_c.server}
|
||||
assert len(servers) == 3
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sticky_session_with_regular_rotation(self):
|
||||
"""Verify sticky sessions don't interfere with regular rotation."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
# Acquire a sticky session
|
||||
session_proxy = await strategy.get_proxy_for_session("sticky-session")
|
||||
|
||||
# Regular rotation should continue independently
|
||||
regular_proxy1 = await strategy.get_next_proxy()
|
||||
regular_proxy2 = await strategy.get_next_proxy()
|
||||
|
||||
# Sticky session should still return same proxy
|
||||
session_proxy_again = await strategy.get_proxy_for_session("sticky-session")
|
||||
|
||||
assert session_proxy.server == session_proxy_again.server
|
||||
# Regular proxies should rotate
|
||||
assert regular_proxy1.server != regular_proxy2.server
|
||||
|
||||
# ==================== SESSION RELEASE TESTS ====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_release(self):
|
||||
"""Verify session can be released and reacquired."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
# Acquire session
|
||||
proxy1 = await strategy.get_proxy_for_session("session-1")
|
||||
assert strategy.get_session_proxy("session-1") is not None
|
||||
|
||||
# Release session
|
||||
await strategy.release_session("session-1")
|
||||
assert strategy.get_session_proxy("session-1") is None
|
||||
|
||||
# Reacquire - should get a new proxy (next in round-robin)
|
||||
proxy2 = await strategy.get_proxy_for_session("session-1")
|
||||
assert proxy2 is not None
|
||||
# After release, next call gets the next proxy in rotation
|
||||
# (not necessarily the same as before)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_nonexistent_session(self):
|
||||
"""Verify releasing non-existent session doesn't raise error."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
# Should not raise
|
||||
await strategy.release_session("nonexistent-session")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_twice(self):
|
||||
"""Verify releasing session twice doesn't raise error."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
await strategy.get_proxy_for_session("session-1")
|
||||
await strategy.release_session("session-1")
|
||||
await strategy.release_session("session-1") # Should not raise
|
||||
|
||||
# ==================== GET SESSION PROXY TESTS ====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_session_proxy_existing(self):
|
||||
"""Verify get_session_proxy returns proxy for existing session."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
acquired = await strategy.get_proxy_for_session("session-1")
|
||||
retrieved = strategy.get_session_proxy("session-1")
|
||||
|
||||
assert retrieved is not None
|
||||
assert acquired.server == retrieved.server
|
||||
|
||||
def test_get_session_proxy_nonexistent(self):
|
||||
"""Verify get_session_proxy returns None for non-existent session."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
result = strategy.get_session_proxy("nonexistent-session")
|
||||
assert result is None
|
||||
|
||||
# ==================== TTL EXPIRATION TESTS ====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_ttl_not_expired(self):
|
||||
"""Verify session returns same proxy when TTL not expired."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
# Acquire with 10 second TTL
|
||||
proxy1 = await strategy.get_proxy_for_session("session-1", ttl=10)
|
||||
|
||||
# Immediately request again - should return same proxy
|
||||
proxy2 = await strategy.get_proxy_for_session("session-1", ttl=10)
|
||||
|
||||
assert proxy1.server == proxy2.server
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_ttl_expired(self):
|
||||
"""Verify new proxy acquired after TTL expires."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
# Acquire with 1 second TTL
|
||||
proxy1 = await strategy.get_proxy_for_session("session-1", ttl=1)
|
||||
|
||||
# Wait for TTL to expire
|
||||
await asyncio.sleep(1.1)
|
||||
|
||||
# Request again - should get new proxy due to expiration
|
||||
proxy2 = await strategy.get_proxy_for_session("session-1", ttl=1)
|
||||
|
||||
# May or may not be same server depending on round-robin state,
|
||||
# but session should have been recreated
|
||||
assert proxy2 is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_session_proxy_ttl_expired(self):
|
||||
"""Verify get_session_proxy returns None after TTL expires."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
await strategy.get_proxy_for_session("session-1", ttl=1)
|
||||
|
||||
# Wait for expiration
|
||||
await asyncio.sleep(1.1)
|
||||
|
||||
# Should return None for expired session
|
||||
result = strategy.get_session_proxy("session-1")
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cleanup_expired_sessions(self):
|
||||
"""Verify cleanup_expired_sessions removes expired sessions."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
# Create sessions with short TTL
|
||||
await strategy.get_proxy_for_session("short-ttl-1", ttl=1)
|
||||
await strategy.get_proxy_for_session("short-ttl-2", ttl=1)
|
||||
# Create session without TTL (should not be cleaned up)
|
||||
await strategy.get_proxy_for_session("no-ttl")
|
||||
|
||||
# Wait for TTL to expire
|
||||
await asyncio.sleep(1.1)
|
||||
|
||||
# Cleanup
|
||||
removed = await strategy.cleanup_expired_sessions()
|
||||
|
||||
assert removed == 2
|
||||
assert strategy.get_session_proxy("short-ttl-1") is None
|
||||
assert strategy.get_session_proxy("short-ttl-2") is None
|
||||
assert strategy.get_session_proxy("no-ttl") is not None
|
||||
|
||||
# ==================== GET ACTIVE SESSIONS TESTS ====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_active_sessions(self):
|
||||
"""Verify get_active_sessions returns all active sessions."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
await strategy.get_proxy_for_session("session-a")
|
||||
await strategy.get_proxy_for_session("session-b")
|
||||
await strategy.get_proxy_for_session("session-c")
|
||||
|
||||
active = strategy.get_active_sessions()
|
||||
|
||||
assert len(active) == 3
|
||||
assert "session-a" in active
|
||||
assert "session-b" in active
|
||||
assert "session-c" in active
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_active_sessions_excludes_expired(self):
|
||||
"""Verify get_active_sessions excludes expired sessions."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
await strategy.get_proxy_for_session("short-ttl", ttl=1)
|
||||
await strategy.get_proxy_for_session("no-ttl")
|
||||
|
||||
# Before expiration
|
||||
active = strategy.get_active_sessions()
|
||||
assert len(active) == 2
|
||||
|
||||
# Wait for TTL to expire
|
||||
await asyncio.sleep(1.1)
|
||||
|
||||
# After expiration
|
||||
active = strategy.get_active_sessions()
|
||||
assert len(active) == 1
|
||||
assert "no-ttl" in active
|
||||
assert "short-ttl" not in active
|
||||
|
||||
# ==================== THREAD SAFETY TESTS ====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_session_access(self):
|
||||
"""Verify thread-safe access to sessions."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
async def acquire_session(session_id: str):
|
||||
proxy = await strategy.get_proxy_for_session(session_id)
|
||||
await asyncio.sleep(0.01) # Simulate work
|
||||
return proxy.server
|
||||
|
||||
# Acquire same session from multiple coroutines
|
||||
results = await asyncio.gather(*[
|
||||
acquire_session("shared-session") for _ in range(10)
|
||||
])
|
||||
|
||||
# All should get same proxy
|
||||
assert len(set(results)) == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_different_sessions(self):
|
||||
"""Verify concurrent acquisition of different sessions works correctly."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
async def acquire_session(session_id: str):
|
||||
proxy = await strategy.get_proxy_for_session(session_id)
|
||||
await asyncio.sleep(0.01)
|
||||
return (session_id, proxy.server)
|
||||
|
||||
# Acquire different sessions concurrently
|
||||
results = await asyncio.gather(*[
|
||||
acquire_session(f"session-{i}") for i in range(5)
|
||||
])
|
||||
|
||||
# Each session should have a consistent proxy
|
||||
session_proxies = dict(results)
|
||||
assert len(session_proxies) == 5
|
||||
|
||||
# Verify each session still returns same proxy
|
||||
for session_id, expected_server in session_proxies.items():
|
||||
actual = await strategy.get_proxy_for_session(session_id)
|
||||
assert actual.server == expected_server
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_session_acquire_and_release(self):
|
||||
"""Verify concurrent acquire and release operations work correctly."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
async def acquire_and_release(session_id: str):
|
||||
proxy = await strategy.get_proxy_for_session(session_id)
|
||||
await asyncio.sleep(0.01)
|
||||
await strategy.release_session(session_id)
|
||||
return proxy.server
|
||||
|
||||
# Run multiple acquire/release cycles concurrently
|
||||
await asyncio.gather(*[
|
||||
acquire_and_release(f"session-{i}") for i in range(10)
|
||||
])
|
||||
|
||||
# All sessions should be released
|
||||
active = strategy.get_active_sessions()
|
||||
assert len(active) == 0
|
||||
|
||||
# ==================== EMPTY PROXY POOL TESTS ====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_proxy_pool_session(self):
|
||||
"""Verify behavior with empty proxy pool."""
|
||||
strategy = RoundRobinProxyStrategy() # No proxies
|
||||
|
||||
result = await strategy.get_proxy_for_session("session-1")
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_proxies_after_session(self):
|
||||
"""Verify adding proxies after session creation works."""
|
||||
strategy = RoundRobinProxyStrategy()
|
||||
|
||||
# No proxies initially
|
||||
result1 = await strategy.get_proxy_for_session("session-1")
|
||||
assert result1 is None
|
||||
|
||||
# Add proxies
|
||||
strategy.add_proxies(self.proxies)
|
||||
|
||||
# Now should work
|
||||
result2 = await strategy.get_proxy_for_session("session-2")
|
||||
assert result2 is not None
|
||||
|
||||
|
||||
class TestCrawlerRunConfigSession:
|
||||
"""Test CrawlerRunConfig with sticky session parameters."""
|
||||
|
||||
def test_config_has_session_fields(self):
|
||||
"""Verify CrawlerRunConfig has sticky session fields."""
|
||||
config = CrawlerRunConfig(
|
||||
proxy_session_id="test-session",
|
||||
proxy_session_ttl=300,
|
||||
proxy_session_auto_release=True
|
||||
)
|
||||
|
||||
assert config.proxy_session_id == "test-session"
|
||||
assert config.proxy_session_ttl == 300
|
||||
assert config.proxy_session_auto_release is True
|
||||
|
||||
def test_config_session_defaults(self):
|
||||
"""Verify default values for session fields."""
|
||||
config = CrawlerRunConfig()
|
||||
|
||||
assert config.proxy_session_id is None
|
||||
assert config.proxy_session_ttl is None
|
||||
assert config.proxy_session_auto_release is False
|
||||
|
||||
|
||||
class TestCrawlerStickySessionIntegration:
|
||||
"""Integration tests for AsyncWebCrawler with sticky sessions."""
|
||||
|
||||
def setup_method(self):
|
||||
"""Setup for each test method."""
|
||||
self.proxies = [
|
||||
ProxyConfig(server=f"http://proxy{i}.test:8080")
|
||||
for i in range(3)
|
||||
]
|
||||
self.test_url = "https://httpbin.org/ip"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawler_sticky_session_without_proxy(self):
|
||||
"""Test that crawler works when proxy_session_id set but no strategy."""
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
proxy_session_id="test-session",
|
||||
page_timeout=15000
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(url=self.test_url, config=config)
|
||||
# Should work without errors (no proxy strategy means no proxy)
|
||||
assert result is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawler_sticky_session_basic(self):
|
||||
"""Test basic sticky session with crawler."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
proxy_rotation_strategy=strategy,
|
||||
proxy_session_id="integration-test",
|
||||
page_timeout=10000
|
||||
)
|
||||
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
# First request
|
||||
try:
|
||||
result1 = await crawler.arun(url=self.test_url, config=config)
|
||||
except Exception:
|
||||
pass # Proxy connection may fail, but session should be tracked
|
||||
|
||||
# Verify session was created
|
||||
session_proxy = strategy.get_session_proxy("integration-test")
|
||||
assert session_proxy is not None
|
||||
|
||||
# Cleanup
|
||||
await strategy.release_session("integration-test")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_crawler_rotating_vs_sticky(self):
|
||||
"""Compare rotating behavior vs sticky session behavior."""
|
||||
strategy = RoundRobinProxyStrategy(self.proxies)
|
||||
|
||||
# Config WITHOUT sticky session - should rotate
|
||||
rotating_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
proxy_rotation_strategy=strategy,
|
||||
page_timeout=5000
|
||||
)
|
||||
|
||||
# Config WITH sticky session - should use same proxy
|
||||
sticky_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
proxy_rotation_strategy=strategy,
|
||||
proxy_session_id="sticky-test",
|
||||
page_timeout=5000
|
||||
)
|
||||
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
# Track proxy configs used
|
||||
rotating_proxies = []
|
||||
sticky_proxies = []
|
||||
|
||||
# Try rotating requests (may fail due to test proxies, but config should be set)
|
||||
for _ in range(3):
|
||||
try:
|
||||
await crawler.arun(url=self.test_url, config=rotating_config)
|
||||
except Exception:
|
||||
pass
|
||||
rotating_proxies.append(rotating_config.proxy_config.server if rotating_config.proxy_config else None)
|
||||
|
||||
# Try sticky requests
|
||||
for _ in range(3):
|
||||
try:
|
||||
await crawler.arun(url=self.test_url, config=sticky_config)
|
||||
except Exception:
|
||||
pass
|
||||
sticky_proxies.append(sticky_config.proxy_config.server if sticky_config.proxy_config else None)
|
||||
|
||||
# Rotating should have different proxies (or cycle through them)
|
||||
# Sticky should have same proxy for all requests
|
||||
if all(sticky_proxies):
|
||||
assert len(set(sticky_proxies)) == 1, "Sticky session should use same proxy"
|
||||
|
||||
await strategy.release_session("sticky-test")
|
||||
|
||||
|
||||
class TestStickySessionRealWorld:
|
||||
"""Real-world scenario tests for sticky sessions.
|
||||
|
||||
Note: These tests require actual proxy servers to verify IP consistency.
|
||||
They are marked to be skipped if no proxy is configured.
|
||||
"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
not os.environ.get('TEST_PROXY_1'),
|
||||
reason="Requires TEST_PROXY_1 environment variable"
|
||||
)
|
||||
async def test_verify_ip_consistency(self):
|
||||
"""Verify that sticky session actually uses same IP.
|
||||
|
||||
This test requires real proxies set in environment variables:
|
||||
TEST_PROXY_1=ip:port:user:pass
|
||||
TEST_PROXY_2=ip:port:user:pass
|
||||
"""
|
||||
import re
|
||||
|
||||
# Load proxies from environment
|
||||
proxy_strs = [
|
||||
os.environ.get('TEST_PROXY_1', ''),
|
||||
os.environ.get('TEST_PROXY_2', '')
|
||||
]
|
||||
proxies = [ProxyConfig.from_string(p) for p in proxy_strs if p]
|
||||
|
||||
if len(proxies) < 2:
|
||||
pytest.skip("Need at least 2 proxies for this test")
|
||||
|
||||
strategy = RoundRobinProxyStrategy(proxies)
|
||||
|
||||
# Config WITH sticky session
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
proxy_rotation_strategy=strategy,
|
||||
proxy_session_id="ip-verify-session",
|
||||
page_timeout=30000
|
||||
)
|
||||
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
ips = []
|
||||
|
||||
for i in range(3):
|
||||
result = await crawler.arun(
|
||||
url="https://httpbin.org/ip",
|
||||
config=config
|
||||
)
|
||||
|
||||
if result and result.success and result.html:
|
||||
# Extract IP from response
|
||||
ip_match = re.search(r'"origin":\s*"([^"]+)"', result.html)
|
||||
if ip_match:
|
||||
ips.append(ip_match.group(1))
|
||||
|
||||
await strategy.release_session("ip-verify-session")
|
||||
|
||||
# All IPs should be same for sticky session
|
||||
if len(ips) >= 2:
|
||||
assert len(set(ips)) == 1, f"Expected same IP, got: {ips}"
|
||||
|
||||
|
||||
# ==================== STANDALONE TEST FUNCTIONS ====================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sticky_session_simple():
|
||||
"""Simple test for sticky session functionality."""
|
||||
proxies = [
|
||||
ProxyConfig(server=f"http://proxy{i}.test:8080")
|
||||
for i in range(3)
|
||||
]
|
||||
strategy = RoundRobinProxyStrategy(proxies)
|
||||
|
||||
# Same session should return same proxy
|
||||
p1 = await strategy.get_proxy_for_session("test")
|
||||
p2 = await strategy.get_proxy_for_session("test")
|
||||
p3 = await strategy.get_proxy_for_session("test")
|
||||
|
||||
assert p1.server == p2.server == p3.server
|
||||
print(f"Sticky session works! All requests use: {p1.server}")
|
||||
|
||||
# Cleanup
|
||||
await strategy.release_session("test")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("Running Sticky Session tests...")
|
||||
print("=" * 50)
|
||||
|
||||
asyncio.run(test_sticky_session_simple())
|
||||
|
||||
print("\n" + "=" * 50)
|
||||
print("To run the full pytest suite, use: pytest " + __file__)
|
||||
print("=" * 50)
|
||||
Reference in New Issue
Block a user