chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,365 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Simple API Test for Crawl4AI Docker Server v0.7.0
|
||||
Uses only built-in Python modules to test all endpoints.
|
||||
"""
|
||||
|
||||
import urllib.request
|
||||
import urllib.parse
|
||||
import json
|
||||
import time
|
||||
import sys
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
# Configuration
|
||||
BASE_URL = "http://localhost:11234" # Change to your server URL
|
||||
TEST_TIMEOUT = 30
|
||||
|
||||
class SimpleApiTester:
|
||||
def __init__(self, base_url: str = BASE_URL):
|
||||
self.base_url = base_url
|
||||
self.token = None
|
||||
self.results = []
|
||||
|
||||
def log(self, message: str):
|
||||
print(f"[INFO] {message}")
|
||||
|
||||
def test_get_endpoint(self, endpoint: str) -> Dict:
|
||||
"""Test a GET endpoint"""
|
||||
url = f"{self.base_url}{endpoint}"
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
req = urllib.request.Request(url)
|
||||
if self.token:
|
||||
req.add_header('Authorization', f'Bearer {self.token}')
|
||||
|
||||
with urllib.request.urlopen(req, timeout=TEST_TIMEOUT) as response:
|
||||
response_time = time.time() - start_time
|
||||
status_code = response.getcode()
|
||||
content = response.read().decode('utf-8')
|
||||
|
||||
# Try to parse JSON
|
||||
try:
|
||||
data = json.loads(content)
|
||||
except:
|
||||
data = {"raw_response": content[:200]}
|
||||
|
||||
return {
|
||||
"endpoint": endpoint,
|
||||
"method": "GET",
|
||||
"status": "PASS" if status_code < 400 else "FAIL",
|
||||
"status_code": status_code,
|
||||
"response_time": response_time,
|
||||
"data": data
|
||||
}
|
||||
except Exception as e:
|
||||
response_time = time.time() - start_time
|
||||
return {
|
||||
"endpoint": endpoint,
|
||||
"method": "GET",
|
||||
"status": "FAIL",
|
||||
"status_code": None,
|
||||
"response_time": response_time,
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
def test_post_endpoint(self, endpoint: str, payload: Dict) -> Dict:
|
||||
"""Test a POST endpoint"""
|
||||
url = f"{self.base_url}{endpoint}"
|
||||
start_time = time.time()
|
||||
|
||||
try:
|
||||
data = json.dumps(payload).encode('utf-8')
|
||||
req = urllib.request.Request(url, data=data, method='POST')
|
||||
req.add_header('Content-Type', 'application/json')
|
||||
|
||||
if self.token:
|
||||
req.add_header('Authorization', f'Bearer {self.token}')
|
||||
|
||||
with urllib.request.urlopen(req, timeout=TEST_TIMEOUT) as response:
|
||||
response_time = time.time() - start_time
|
||||
status_code = response.getcode()
|
||||
content = response.read().decode('utf-8')
|
||||
|
||||
# Try to parse JSON
|
||||
try:
|
||||
data = json.loads(content)
|
||||
except:
|
||||
data = {"raw_response": content[:200]}
|
||||
|
||||
return {
|
||||
"endpoint": endpoint,
|
||||
"method": "POST",
|
||||
"status": "PASS" if status_code < 400 else "FAIL",
|
||||
"status_code": status_code,
|
||||
"response_time": response_time,
|
||||
"data": data
|
||||
}
|
||||
except Exception as e:
|
||||
response_time = time.time() - start_time
|
||||
return {
|
||||
"endpoint": endpoint,
|
||||
"method": "POST",
|
||||
"status": "FAIL",
|
||||
"status_code": None,
|
||||
"response_time": response_time,
|
||||
"error": str(e)
|
||||
}
|
||||
|
||||
def print_result(self, result: Dict):
|
||||
"""Print a formatted test result"""
|
||||
status_color = {
|
||||
"PASS": "✅",
|
||||
"FAIL": "❌",
|
||||
"SKIP": "⏭️"
|
||||
}
|
||||
|
||||
print(f"{status_color[result['status']]} {result['method']} {result['endpoint']} "
|
||||
f"| {result['response_time']:.3f}s | Status: {result['status_code'] or 'N/A'}")
|
||||
|
||||
if result['status'] == 'FAIL' and 'error' in result:
|
||||
print(f" Error: {result['error']}")
|
||||
|
||||
self.results.append(result)
|
||||
|
||||
def run_all_tests(self):
|
||||
"""Run all API tests"""
|
||||
print("🚀 Starting Crawl4AI v0.7.0 API Test Suite")
|
||||
print(f"📡 Testing server at: {self.base_url}")
|
||||
print("=" * 60)
|
||||
|
||||
# # Test basic endpoints
|
||||
# print("\n=== BASIC ENDPOINTS ===")
|
||||
|
||||
# # Health check
|
||||
# result = self.test_get_endpoint("/health")
|
||||
# self.print_result(result)
|
||||
|
||||
|
||||
# # Schema endpoint
|
||||
# result = self.test_get_endpoint("/schema")
|
||||
# self.print_result(result)
|
||||
|
||||
# # Metrics endpoint
|
||||
# result = self.test_get_endpoint("/metrics")
|
||||
# self.print_result(result)
|
||||
|
||||
# # Root redirect
|
||||
# result = self.test_get_endpoint("/")
|
||||
# self.print_result(result)
|
||||
|
||||
# # Test authentication
|
||||
# print("\n=== AUTHENTICATION ===")
|
||||
|
||||
# # Get token
|
||||
# token_payload = {"email": "test@example.com"}
|
||||
# result = self.test_post_endpoint("/token", token_payload)
|
||||
# self.print_result(result)
|
||||
|
||||
# # Extract token if successful
|
||||
# if result['status'] == 'PASS' and 'data' in result:
|
||||
# token = result['data'].get('access_token')
|
||||
# if token:
|
||||
# self.token = token
|
||||
# self.log(f"Successfully obtained auth token: {token[:20]}...")
|
||||
|
||||
# Test core APIs
|
||||
print("\n=== CORE APIs ===")
|
||||
|
||||
test_url = "https://example.com"
|
||||
test_raw_html_url = "raw://<html><body><h1>Hello, World!</h1></body></html>"
|
||||
# Test markdown endpoint
|
||||
md_payload = {
|
||||
"url": test_url,
|
||||
"f": "fit",
|
||||
"q": "test query",
|
||||
"c": "0"
|
||||
}
|
||||
result = self.test_post_endpoint("/md", md_payload)
|
||||
# print(result['data'].get('markdown', ''))
|
||||
self.print_result(result)
|
||||
|
||||
# Test markdown endpoint with raw HTML
|
||||
raw_md_payload = {
|
||||
"url": test_raw_html_url,
|
||||
"f": "fit",
|
||||
"q": "test query",
|
||||
"c": "0"
|
||||
}
|
||||
result = self.test_post_endpoint("/md", raw_md_payload)
|
||||
self.print_result(result)
|
||||
|
||||
|
||||
# Test HTML endpoint
|
||||
html_payload = {"url": test_url}
|
||||
result = self.test_post_endpoint("/html", html_payload)
|
||||
self.print_result(result)
|
||||
|
||||
# Test screenshot endpoint
|
||||
screenshot_payload = {
|
||||
"url": test_url,
|
||||
"screenshot_wait_for": 2
|
||||
}
|
||||
result = self.test_post_endpoint("/screenshot", screenshot_payload)
|
||||
self.print_result(result)
|
||||
|
||||
# Test PDF endpoint
|
||||
pdf_payload = {"url": test_url}
|
||||
result = self.test_post_endpoint("/pdf", pdf_payload)
|
||||
self.print_result(result)
|
||||
|
||||
# Test JavaScript execution
|
||||
js_payload = {
|
||||
"url": test_url,
|
||||
"scripts": ["(() => document.title)()"]
|
||||
}
|
||||
result = self.test_post_endpoint("/execute_js", js_payload)
|
||||
self.print_result(result)
|
||||
|
||||
# Test crawl endpoint
|
||||
crawl_payload = {
|
||||
"urls": [test_url],
|
||||
"browser_config": {},
|
||||
"crawler_config": {}
|
||||
}
|
||||
result = self.test_post_endpoint("/crawl", crawl_payload)
|
||||
self.print_result(result)
|
||||
|
||||
# Test crawl endpoint with raw HTML
|
||||
crawl_payload = {
|
||||
"urls": [test_raw_html_url],
|
||||
"browser_config": {},
|
||||
"crawler_config": {}
|
||||
}
|
||||
result = self.test_post_endpoint("/crawl", crawl_payload)
|
||||
self.print_result(result)
|
||||
|
||||
# Test config dump
|
||||
config_payload = {"code": "CrawlerRunConfig()"}
|
||||
result = self.test_post_endpoint("/config/dump", config_payload)
|
||||
self.print_result(result)
|
||||
|
||||
# Test LLM endpoint
|
||||
llm_endpoint = f"/llm/{test_url}?q=Extract%20main%20content"
|
||||
result = self.test_get_endpoint(llm_endpoint)
|
||||
self.print_result(result)
|
||||
|
||||
# Test ask endpoint
|
||||
ask_endpoint = "/ask?context_type=all&query=crawl4ai&max_results=5"
|
||||
result = self.test_get_endpoint(ask_endpoint)
|
||||
print(result)
|
||||
self.print_result(result)
|
||||
|
||||
# Test job APIs
|
||||
print("\n=== JOB APIs ===")
|
||||
|
||||
# Test LLM job
|
||||
llm_job_payload = {
|
||||
"url": test_url,
|
||||
"q": "Extract main content",
|
||||
"cache": False
|
||||
}
|
||||
result = self.test_post_endpoint("/llm/job", llm_job_payload)
|
||||
self.print_result(result)
|
||||
|
||||
# Test crawl job
|
||||
crawl_job_payload = {
|
||||
"urls": [test_url],
|
||||
"browser_config": {},
|
||||
"crawler_config": {}
|
||||
}
|
||||
result = self.test_post_endpoint("/crawl/job", crawl_job_payload)
|
||||
self.print_result(result)
|
||||
|
||||
# Test MCP
|
||||
print("\n=== MCP APIs ===")
|
||||
|
||||
# Test MCP schema
|
||||
result = self.test_get_endpoint("/mcp/schema")
|
||||
self.print_result(result)
|
||||
|
||||
# Test error handling
|
||||
print("\n=== ERROR HANDLING ===")
|
||||
|
||||
# Test invalid URL
|
||||
invalid_payload = {"url": "invalid-url", "f": "fit"}
|
||||
result = self.test_post_endpoint("/md", invalid_payload)
|
||||
self.print_result(result)
|
||||
|
||||
# Test invalid endpoint
|
||||
result = self.test_get_endpoint("/nonexistent")
|
||||
self.print_result(result)
|
||||
|
||||
# Print summary
|
||||
self.print_summary()
|
||||
|
||||
def print_summary(self):
|
||||
"""Print test results summary"""
|
||||
print("\n" + "=" * 60)
|
||||
print("📊 TEST RESULTS SUMMARY")
|
||||
print("=" * 60)
|
||||
|
||||
total = len(self.results)
|
||||
passed = sum(1 for r in self.results if r['status'] == 'PASS')
|
||||
failed = sum(1 for r in self.results if r['status'] == 'FAIL')
|
||||
|
||||
print(f"Total Tests: {total}")
|
||||
print(f"✅ Passed: {passed}")
|
||||
print(f"❌ Failed: {failed}")
|
||||
print(f"📈 Success Rate: {(passed/total)*100:.1f}%")
|
||||
|
||||
if failed > 0:
|
||||
print("\n❌ FAILED TESTS:")
|
||||
for result in self.results:
|
||||
if result['status'] == 'FAIL':
|
||||
print(f" • {result['method']} {result['endpoint']}")
|
||||
if 'error' in result:
|
||||
print(f" Error: {result['error']}")
|
||||
|
||||
# Performance statistics
|
||||
response_times = [r['response_time'] for r in self.results if r['response_time'] > 0]
|
||||
if response_times:
|
||||
avg_time = sum(response_times) / len(response_times)
|
||||
max_time = max(response_times)
|
||||
print(f"\n⏱️ Average Response Time: {avg_time:.3f}s")
|
||||
print(f"⏱️ Max Response Time: {max_time:.3f}s")
|
||||
|
||||
# Save detailed report
|
||||
report_file = f"crawl4ai_test_report_{int(time.time())}.json"
|
||||
with open(report_file, 'w') as f:
|
||||
json.dump({
|
||||
"timestamp": time.time(),
|
||||
"server_url": self.base_url,
|
||||
"version": "0.7.0",
|
||||
"summary": {
|
||||
"total": total,
|
||||
"passed": passed,
|
||||
"failed": failed
|
||||
},
|
||||
"results": self.results
|
||||
}, f, indent=2)
|
||||
|
||||
print(f"\n📄 Detailed report saved to: {report_file}")
|
||||
|
||||
def main():
|
||||
"""Main test runner"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description='Crawl4AI v0.7.0 API Test Suite')
|
||||
parser.add_argument('--url', default=BASE_URL, help='Base URL of the server')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
tester = SimpleApiTester(args.url)
|
||||
|
||||
try:
|
||||
tester.run_all_tests()
|
||||
except KeyboardInterrupt:
|
||||
print("\n🛑 Test suite interrupted by user")
|
||||
except Exception as e:
|
||||
print(f"\n💥 Test suite failed with error: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,113 @@
|
||||
import json
|
||||
from crawl4ai import (
|
||||
CrawlerRunConfig,
|
||||
DefaultMarkdownGenerator,
|
||||
RegexChunking,
|
||||
JsonCssExtractionStrategy,
|
||||
BM25ContentFilter,
|
||||
CacheMode
|
||||
)
|
||||
from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
|
||||
from crawl4ai.deep_crawling.filters import FastFilterChain
|
||||
from crawl4ai.deep_crawling.filters import FastContentTypeFilter, FastDomainFilter
|
||||
from crawl4ai.deep_crawling.scorers import FastKeywordRelevanceScorer
|
||||
|
||||
def create_test_config() -> CrawlerRunConfig:
|
||||
# Set up content filtering and markdown generation
|
||||
content_filter = BM25ContentFilter(
|
||||
user_query="technology articles",
|
||||
)
|
||||
|
||||
markdown_generator = DefaultMarkdownGenerator(
|
||||
content_filter=content_filter,
|
||||
options={"ignore_links": False, "body_width": 0}
|
||||
)
|
||||
|
||||
# Set up extraction strategy
|
||||
extraction_schema = {
|
||||
"name": "ArticleExtractor",
|
||||
"baseSelector": "article.content",
|
||||
"fields": [
|
||||
{"name": "title", "selector": "h1", "type": "text"},
|
||||
{"name": "content", "selector": ".article-body", "type": "html"}
|
||||
]
|
||||
}
|
||||
extraction_strategy = JsonCssExtractionStrategy(schema=extraction_schema)
|
||||
|
||||
# Set up deep crawling
|
||||
filter_chain = FastFilterChain([
|
||||
FastContentTypeFilter(["text/html"]),
|
||||
FastDomainFilter(blocked_domains=["ads.*"])
|
||||
])
|
||||
|
||||
url_scorer = FastKeywordRelevanceScorer(
|
||||
keywords=["article", "blog"],
|
||||
weight=1.0
|
||||
)
|
||||
|
||||
deep_crawl_strategy = BFSDeepCrawlStrategy(
|
||||
max_depth=3,
|
||||
filter_chain=filter_chain,
|
||||
url_scorer=url_scorer
|
||||
)
|
||||
|
||||
# Create the config
|
||||
config = CrawlerRunConfig(
|
||||
word_count_threshold=200,
|
||||
extraction_strategy=extraction_strategy,
|
||||
chunking_strategy=RegexChunking(patterns=[r"\n\n"]),
|
||||
markdown_generator=markdown_generator,
|
||||
css_selector="main.content",
|
||||
excluded_tags=["nav", "footer"],
|
||||
keep_attrs=["href", "src"],
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
wait_until="networkidle",
|
||||
page_timeout=30000,
|
||||
scan_full_page=True,
|
||||
deep_crawl_strategy=deep_crawl_strategy,
|
||||
verbose=True,
|
||||
stream=True
|
||||
)
|
||||
|
||||
return config
|
||||
|
||||
def test_config_serialization_cycle():
|
||||
# Create original config
|
||||
original_config = create_test_config()
|
||||
|
||||
# Dump to serializable dictionary
|
||||
serialized = original_config.dump()
|
||||
|
||||
print(json.dumps(serialized, indent=2))
|
||||
|
||||
# Load back into config object
|
||||
deserialized_config = CrawlerRunConfig.load(serialized)
|
||||
|
||||
# Verify core attributes
|
||||
assert deserialized_config.word_count_threshold == original_config.word_count_threshold
|
||||
assert deserialized_config.css_selector == original_config.css_selector
|
||||
assert deserialized_config.excluded_tags == original_config.excluded_tags
|
||||
assert deserialized_config.keep_attrs == original_config.keep_attrs
|
||||
assert deserialized_config.cache_mode == original_config.cache_mode
|
||||
assert deserialized_config.wait_until == original_config.wait_until
|
||||
assert deserialized_config.page_timeout == original_config.page_timeout
|
||||
assert deserialized_config.scan_full_page == original_config.scan_full_page
|
||||
assert deserialized_config.verbose == original_config.verbose
|
||||
assert deserialized_config.stream == original_config.stream
|
||||
|
||||
# Verify complex objects
|
||||
assert isinstance(deserialized_config.extraction_strategy, JsonCssExtractionStrategy)
|
||||
assert isinstance(deserialized_config.chunking_strategy, RegexChunking)
|
||||
assert isinstance(deserialized_config.markdown_generator, DefaultMarkdownGenerator)
|
||||
assert isinstance(deserialized_config.markdown_generator.content_filter, BM25ContentFilter)
|
||||
assert isinstance(deserialized_config.deep_crawl_strategy, BFSDeepCrawlStrategy)
|
||||
|
||||
# Verify deep crawl strategy configuration
|
||||
assert deserialized_config.deep_crawl_strategy.max_depth == 3
|
||||
assert isinstance(deserialized_config.deep_crawl_strategy.filter_chain, FastFilterChain)
|
||||
assert isinstance(deserialized_config.deep_crawl_strategy.url_scorer, FastKeywordRelevanceScorer)
|
||||
|
||||
print("Serialization cycle test passed successfully!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_config_serialization_cycle()
|
||||
@@ -0,0 +1,186 @@
|
||||
import requests
|
||||
import time
|
||||
import httpx
|
||||
import asyncio
|
||||
from typing import Dict, Any
|
||||
from crawl4ai import (
|
||||
BrowserConfig, CrawlerRunConfig, DefaultMarkdownGenerator,
|
||||
PruningContentFilter, JsonCssExtractionStrategy, LLMContentFilter, CacheMode
|
||||
)
|
||||
from crawl4ai import LLMConfig
|
||||
from crawl4ai.docker_client import Crawl4aiDockerClient
|
||||
|
||||
class Crawl4AiTester:
|
||||
def __init__(self, base_url: str = "http://localhost:11235"):
|
||||
self.base_url = base_url
|
||||
|
||||
def submit_and_wait(
|
||||
self, request_data: Dict[str, Any], timeout: int = 300
|
||||
) -> Dict[str, Any]:
|
||||
# Submit crawl job
|
||||
response = requests.post(f"{self.base_url}/crawl", json=request_data)
|
||||
task_id = response.json()["task_id"]
|
||||
print(f"Task ID: {task_id}")
|
||||
|
||||
# Poll for result
|
||||
start_time = time.time()
|
||||
while True:
|
||||
if time.time() - start_time > timeout:
|
||||
raise TimeoutError(
|
||||
f"Task {task_id} did not complete within {timeout} seconds"
|
||||
)
|
||||
|
||||
result = requests.get(f"{self.base_url}/task/{task_id}")
|
||||
status = result.json()
|
||||
|
||||
if status["status"] == "failed":
|
||||
print("Task failed:", status.get("error"))
|
||||
raise Exception(f"Task failed: {status.get('error')}")
|
||||
|
||||
if status["status"] == "completed":
|
||||
return status
|
||||
|
||||
time.sleep(2)
|
||||
|
||||
async def test_direct_api():
|
||||
"""Test direct API endpoints without using the client SDK"""
|
||||
print("\n=== Testing Direct API Calls ===")
|
||||
|
||||
# Test 1: Basic crawl with content filtering
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
viewport_width=1200,
|
||||
viewport_height=800
|
||||
)
|
||||
|
||||
crawler_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
markdown_generator=DefaultMarkdownGenerator(
|
||||
content_filter=PruningContentFilter(
|
||||
threshold=0.48,
|
||||
threshold_type="fixed",
|
||||
min_word_threshold=0
|
||||
),
|
||||
options={"ignore_links": True}
|
||||
)
|
||||
)
|
||||
|
||||
request_data = {
|
||||
"urls": ["https://example.com"],
|
||||
"browser_config": browser_config.dump(),
|
||||
"crawler_config": crawler_config.dump()
|
||||
}
|
||||
|
||||
# Make direct API call
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(
|
||||
"http://localhost:11235/crawl",
|
||||
json=request_data,
|
||||
timeout=300
|
||||
)
|
||||
assert response.status_code == 200
|
||||
result = response.json()
|
||||
print("Basic crawl result:", result["success"])
|
||||
|
||||
# Test 2: Structured extraction with JSON CSS
|
||||
schema = {
|
||||
"baseSelector": "article.post",
|
||||
"fields": [
|
||||
{"name": "title", "selector": "h1", "type": "text"},
|
||||
{"name": "content", "selector": ".content", "type": "html"}
|
||||
]
|
||||
}
|
||||
|
||||
crawler_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
extraction_strategy=JsonCssExtractionStrategy(schema=schema)
|
||||
)
|
||||
|
||||
request_data["crawler_config"] = crawler_config.dump()
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(
|
||||
"http://localhost:11235/crawl",
|
||||
json=request_data
|
||||
)
|
||||
assert response.status_code == 200
|
||||
result = response.json()
|
||||
print("Structured extraction result:", result["success"])
|
||||
|
||||
# Test 3: Raw HTML
|
||||
request_data["urls"] = ["raw://<html><body><h1>Hello, World!</h1><a href='https://example.com'>Example</a></body></html>"]
|
||||
async with httpx.AsyncClient() as client:
|
||||
response = await client.post(
|
||||
"http://localhost:11235/crawl",
|
||||
json=request_data
|
||||
)
|
||||
assert response.status_code == 200
|
||||
result = response.json()
|
||||
print("Raw HTML result:", result["success"])
|
||||
|
||||
# Test 3: Get schema
|
||||
# async with httpx.AsyncClient() as client:
|
||||
# response = await client.get("http://localhost:8000/schema")
|
||||
# assert response.status_code == 200
|
||||
# schemas = response.json()
|
||||
# print("Retrieved schemas for:", list(schemas.keys()))
|
||||
|
||||
async def test_with_client():
|
||||
"""Test using the Crawl4AI Docker client SDK"""
|
||||
print("\n=== Testing Client SDK ===")
|
||||
|
||||
async with Crawl4aiDockerClient(base_url="http://localhost:11235", verbose=True) as client:
|
||||
# Test 1: Basic crawl
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
crawler_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
markdown_generator=DefaultMarkdownGenerator(
|
||||
content_filter=PruningContentFilter(
|
||||
threshold=0.48,
|
||||
threshold_type="fixed"
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
result = await client.crawl(
|
||||
urls=["https://example.com"],
|
||||
browser_config=browser_config,
|
||||
crawler_config=crawler_config
|
||||
)
|
||||
print("Client SDK basic crawl:", result.success)
|
||||
|
||||
# Test 2: LLM extraction with streaming
|
||||
crawler_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
markdown_generator=DefaultMarkdownGenerator(
|
||||
content_filter=LLMContentFilter(
|
||||
llm_config=LLMConfig(provider="openai/gpt-40"),
|
||||
instruction="Extract key technical concepts"
|
||||
)
|
||||
),
|
||||
stream=True
|
||||
)
|
||||
|
||||
async for result in await client.crawl(
|
||||
urls=["https://example.com"],
|
||||
browser_config=browser_config,
|
||||
crawler_config=crawler_config
|
||||
):
|
||||
print(f"Streaming result for: {result.url}")
|
||||
|
||||
# # Test 3: Get schema
|
||||
# schemas = await client.get_schema()
|
||||
# print("Retrieved client schemas for:", list(schemas.keys()))
|
||||
|
||||
async def main():
|
||||
"""Run all tests"""
|
||||
# Test direct API
|
||||
print("Testing direct API calls...")
|
||||
await test_direct_api()
|
||||
|
||||
# Test client SDK
|
||||
print("\nTesting client SDK...")
|
||||
await test_with_client()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,34 @@
|
||||
import asyncio
|
||||
from crawl4ai.docker_client import Crawl4aiDockerClient
|
||||
from crawl4ai import (
|
||||
BrowserConfig,
|
||||
CrawlerRunConfig
|
||||
)
|
||||
|
||||
async def main():
|
||||
async with Crawl4aiDockerClient(base_url="http://localhost:8000", verbose=True) as client:
|
||||
await client.authenticate("test@example.com")
|
||||
|
||||
# Non-streaming crawl
|
||||
results = await client.crawl(
|
||||
["https://example.com", "https://python.org"],
|
||||
browser_config=BrowserConfig(headless=True),
|
||||
crawler_config=CrawlerRunConfig()
|
||||
)
|
||||
print(f"Non-streaming results: {results}")
|
||||
|
||||
# Streaming crawl
|
||||
crawler_config = CrawlerRunConfig(stream=True)
|
||||
async for result in await client.crawl(
|
||||
["https://example.com", "https://python.org"],
|
||||
browser_config=BrowserConfig(headless=True),
|
||||
crawler_config=crawler_config
|
||||
):
|
||||
print(f"Streamed result: {result}")
|
||||
|
||||
# Get schema
|
||||
schema = await client.get_schema()
|
||||
print(f"Schema: {schema}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,245 @@
|
||||
"""
|
||||
Test the complete fix for both the filter serialization and JSON serialization issues.
|
||||
"""
|
||||
import os
|
||||
import traceback
|
||||
from typing import Any
|
||||
|
||||
import asyncio
|
||||
import httpx
|
||||
|
||||
from crawl4ai import BrowserConfig, CacheMode, CrawlerRunConfig
|
||||
from crawl4ai.deep_crawling import (
|
||||
BFSDeepCrawlStrategy,
|
||||
ContentRelevanceFilter,
|
||||
FilterChain,
|
||||
URLFilter,
|
||||
URLPatternFilter,
|
||||
)
|
||||
|
||||
CRAWL4AI_DOCKER_PORT = os.environ.get("CRAWL4AI_DOCKER_PORT", "11234")
|
||||
try:
|
||||
BASE_PORT = int(CRAWL4AI_DOCKER_PORT)
|
||||
except TypeError:
|
||||
BASE_PORT = 11234
|
||||
BASE_URL = f"http://localhost:{BASE_PORT}/" # Adjust port as needed
|
||||
|
||||
|
||||
async def test_with_docker_client(filter_chain: list[URLFilter], max_pages: int = 20, timeout: int = 30) -> bool:
|
||||
"""Test using the Docker client (same as 1419.py)."""
|
||||
from crawl4ai.docker_client import Crawl4aiDockerClient
|
||||
|
||||
print("=" * 60)
|
||||
print("Testing with Docker Client")
|
||||
print("=" * 60)
|
||||
|
||||
try:
|
||||
async with Crawl4aiDockerClient(
|
||||
base_url=BASE_URL,
|
||||
verbose=True,
|
||||
) as client:
|
||||
|
||||
crawler_config = CrawlerRunConfig(
|
||||
deep_crawl_strategy=BFSDeepCrawlStrategy(
|
||||
max_depth=2, # Keep it shallow for testing
|
||||
max_pages=max_pages, # Limit pages for testing
|
||||
filter_chain=FilterChain(filter_chain)
|
||||
),
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
)
|
||||
|
||||
print("\n1. Testing crawl with filters...")
|
||||
results = await client.crawl(
|
||||
["https://docs.crawl4ai.com"], # Simple test page
|
||||
browser_config=BrowserConfig(headless=True),
|
||||
crawler_config=crawler_config,
|
||||
hooks_timeout=timeout,
|
||||
)
|
||||
|
||||
if results:
|
||||
print(f"✅ Crawl succeeded! Type: {type(results)}")
|
||||
if hasattr(results, 'success'):
|
||||
print(f"✅ Results success: {results.success}")
|
||||
# Test that we can iterate results without JSON errors
|
||||
if hasattr(results, '__iter__'):
|
||||
for i, result in enumerate(results):
|
||||
if hasattr(result, 'url'):
|
||||
print(f" Result {i}: {result.url[:50]}...")
|
||||
else:
|
||||
print(f" Result {i}: {str(result)[:50]}...")
|
||||
else:
|
||||
# Handle list of results
|
||||
print(f"✅ Got {len(results)} results")
|
||||
for i, result in enumerate(results[:3]): # Show first 3
|
||||
print(f" Result {i}: {result.url[:50]}...")
|
||||
else:
|
||||
print("❌ Crawl failed - no results returned")
|
||||
return False
|
||||
|
||||
print("\n✅ Docker client test completed successfully!")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Docker client test failed: {e}")
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
|
||||
async def test_with_rest_api(filters: list[dict[str, Any]], max_pages: int = 20, timeout: int = 30) -> bool:
|
||||
"""Test using REST API directly."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Testing with REST API")
|
||||
print("=" * 60)
|
||||
|
||||
# Create filter configuration
|
||||
deep_crawl_strategy_payload = {
|
||||
"type": "BFSDeepCrawlStrategy",
|
||||
"params": {
|
||||
"max_depth": 2,
|
||||
"max_pages": max_pages,
|
||||
"filter_chain": {
|
||||
"type": "FilterChain",
|
||||
"params": {
|
||||
"filters": filters
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
crawl_payload = {
|
||||
"urls": ["https://docs.crawl4ai.com"],
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"deep_crawl_strategy": deep_crawl_strategy_payload,
|
||||
"cache_mode": "bypass"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
print("\n1. Sending crawl request to REST API...")
|
||||
response = await client.post(
|
||||
f"{BASE_URL}crawl",
|
||||
json=crawl_payload,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
print(f"✅ REST API returned 200 OK")
|
||||
data = response.json()
|
||||
if data.get("success"):
|
||||
results = data.get("results", [])
|
||||
print(f"✅ Got {len(results)} results")
|
||||
for i, result in enumerate(results[:3]):
|
||||
print(f" Result {i}: {result.get('url', 'unknown')[:50]}...")
|
||||
else:
|
||||
print(f"❌ Crawl not successful: {data}")
|
||||
return False
|
||||
else:
|
||||
print(f"❌ REST API returned {response.status_code}")
|
||||
print(f" Response: {response.text[:500]}")
|
||||
return False
|
||||
|
||||
print("\n✅ REST API test completed successfully!")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ REST API test failed: {e}")
|
||||
traceback.print_exc()
|
||||
return False
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all tests."""
|
||||
print("\n🧪 TESTING COMPLETE FIX FOR DOCKER FILTER AND JSON ISSUES")
|
||||
print("=" * 60)
|
||||
print("Make sure the server is running with the updated code!")
|
||||
print("=" * 60)
|
||||
|
||||
results = []
|
||||
|
||||
# Test 1: Docker client
|
||||
max_pages_ = [20, 5]
|
||||
timeouts = [30, 60]
|
||||
filter_chain_test_cases = [
|
||||
[
|
||||
URLPatternFilter(
|
||||
# patterns=["*about*", "*privacy*", "*terms*"],
|
||||
patterns=["*advanced*"],
|
||||
reverse=True
|
||||
),
|
||||
],
|
||||
[
|
||||
ContentRelevanceFilter(
|
||||
query="about faq",
|
||||
threshold=0.2,
|
||||
),
|
||||
],
|
||||
]
|
||||
for idx, (filter_chain, max_pages, timeout) in enumerate(zip(filter_chain_test_cases, max_pages_, timeouts)):
|
||||
docker_passed = await test_with_docker_client(filter_chain=filter_chain, max_pages=max_pages, timeout=timeout)
|
||||
results.append((f"Docker Client w/ filter chain {idx}", docker_passed))
|
||||
|
||||
# Test 2: REST API
|
||||
max_pages_ = [20, 5, 5]
|
||||
timeouts = [30, 60, 60]
|
||||
filters_test_cases = [
|
||||
[
|
||||
{
|
||||
"type": "URLPatternFilter",
|
||||
"params": {
|
||||
"patterns": ["*advanced*"],
|
||||
"reverse": True
|
||||
}
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"type": "ContentRelevanceFilter",
|
||||
"params": {
|
||||
"query": "about faq",
|
||||
"threshold": 0.2,
|
||||
}
|
||||
}
|
||||
],
|
||||
[
|
||||
{
|
||||
"type": "ContentRelevanceFilter",
|
||||
"params": {
|
||||
"query": ["about", "faq"],
|
||||
"threshold": 0.2,
|
||||
}
|
||||
}
|
||||
],
|
||||
]
|
||||
for idx, (filters, max_pages, timeout) in enumerate(zip(filters_test_cases, max_pages_, timeouts)):
|
||||
rest_passed = await test_with_rest_api(filters=filters, max_pages=max_pages, timeout=timeout)
|
||||
results.append((f"REST API w/ filters {idx}", rest_passed))
|
||||
|
||||
# Summary
|
||||
print("\n" + "=" * 60)
|
||||
print("FINAL TEST SUMMARY")
|
||||
print("=" * 60)
|
||||
|
||||
all_passed = True
|
||||
for test_name, passed in results:
|
||||
status = "✅ PASSED" if passed else "❌ FAILED"
|
||||
print(f"{test_name:20} {status}")
|
||||
if not passed:
|
||||
all_passed = False
|
||||
|
||||
print("=" * 60)
|
||||
if all_passed:
|
||||
print("🎉 ALL TESTS PASSED!")
|
||||
else:
|
||||
print("⚠️ Some tests failed. Please check the server logs for details.")
|
||||
|
||||
return 0 if all_passed else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import sys
|
||||
sys.exit(asyncio.run(main()))
|
||||
@@ -0,0 +1,372 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test client for demonstrating user-provided hooks in Crawl4AI Docker API
|
||||
"""
|
||||
|
||||
import requests
|
||||
import json
|
||||
from typing import Dict, Any
|
||||
|
||||
|
||||
API_BASE_URL = "http://localhost:11234" # Adjust if needed
|
||||
|
||||
|
||||
def test_hooks_info():
|
||||
"""Get information about available hooks"""
|
||||
print("=" * 70)
|
||||
print("Testing: GET /hooks/info")
|
||||
print("=" * 70)
|
||||
|
||||
response = requests.get(f"{API_BASE_URL}/hooks/info")
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
print("Available Hook Points:")
|
||||
for hook, info in data['available_hooks'].items():
|
||||
print(f"\n{hook}:")
|
||||
print(f" Parameters: {', '.join(info['parameters'])}")
|
||||
print(f" Description: {info['description']}")
|
||||
else:
|
||||
print(f"Error: {response.status_code}")
|
||||
print(response.text)
|
||||
|
||||
|
||||
def test_basic_crawl_with_hooks():
|
||||
"""Test basic crawling with user-provided hooks"""
|
||||
print("\n" + "=" * 70)
|
||||
print("Testing: POST /crawl with hooks")
|
||||
print("=" * 70)
|
||||
|
||||
# Define hooks as Python code strings
|
||||
hooks_code = {
|
||||
"on_page_context_created": """
|
||||
async def hook(page, context, **kwargs):
|
||||
print("Hook: Setting up page context")
|
||||
# Block images to speed up crawling
|
||||
await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
|
||||
print("Hook: Images blocked")
|
||||
return page
|
||||
""",
|
||||
|
||||
"before_retrieve_html": """
|
||||
async def hook(page, context, **kwargs):
|
||||
print("Hook: Before retrieving HTML")
|
||||
# Scroll to bottom to load lazy content
|
||||
await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
||||
await page.wait_for_timeout(1000)
|
||||
print("Hook: Scrolled to bottom")
|
||||
return page
|
||||
""",
|
||||
|
||||
"before_goto": """
|
||||
async def hook(page, context, url, **kwargs):
|
||||
print(f"Hook: About to navigate to {url}")
|
||||
# Add custom headers
|
||||
await page.set_extra_http_headers({
|
||||
'X-Test-Header': 'crawl4ai-hooks-test'
|
||||
})
|
||||
return page
|
||||
"""
|
||||
}
|
||||
|
||||
# Create request payload
|
||||
payload = {
|
||||
"urls": ["https://httpbin.org/html"],
|
||||
"hooks": {
|
||||
"code": hooks_code,
|
||||
"timeout": 30
|
||||
}
|
||||
}
|
||||
|
||||
print("Sending request with hooks...")
|
||||
response = requests.post(f"{API_BASE_URL}/crawl", json=payload)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
print("\n✅ Crawl successful!")
|
||||
|
||||
# Check hooks status
|
||||
if 'hooks' in data:
|
||||
hooks_info = data['hooks']
|
||||
print("\nHooks Execution Summary:")
|
||||
print(f" Status: {hooks_info['status']['status']}")
|
||||
print(f" Attached hooks: {', '.join(hooks_info['status']['attached_hooks'])}")
|
||||
|
||||
if hooks_info['status']['validation_errors']:
|
||||
print("\n⚠️ Validation Errors:")
|
||||
for error in hooks_info['status']['validation_errors']:
|
||||
print(f" - {error['hook_point']}: {error['error']}")
|
||||
|
||||
if 'summary' in hooks_info:
|
||||
summary = hooks_info['summary']
|
||||
print(f"\nExecution Statistics:")
|
||||
print(f" Total executions: {summary['total_executions']}")
|
||||
print(f" Successful: {summary['successful']}")
|
||||
print(f" Failed: {summary['failed']}")
|
||||
print(f" Timed out: {summary['timed_out']}")
|
||||
print(f" Success rate: {summary['success_rate']:.1f}%")
|
||||
|
||||
if hooks_info['execution_log']:
|
||||
print("\nExecution Log:")
|
||||
for log_entry in hooks_info['execution_log']:
|
||||
status_icon = "✅" if log_entry['status'] == 'success' else "❌"
|
||||
print(f" {status_icon} {log_entry['hook_point']}: {log_entry['status']} ({log_entry.get('execution_time', 0):.2f}s)")
|
||||
|
||||
if hooks_info['errors']:
|
||||
print("\n❌ Hook Errors:")
|
||||
for error in hooks_info['errors']:
|
||||
print(f" - {error['hook_point']}: {error['error']}")
|
||||
|
||||
# Show crawl results
|
||||
if 'results' in data:
|
||||
print(f"\nCrawled {len(data['results'])} URL(s)")
|
||||
for result in data['results']:
|
||||
print(f" - {result['url']}: {'✅' if result['success'] else '❌'}")
|
||||
|
||||
else:
|
||||
print(f"❌ Error: {response.status_code}")
|
||||
print(response.text)
|
||||
|
||||
|
||||
def test_invalid_hook():
|
||||
"""Test with an invalid hook to see error handling"""
|
||||
print("\n" + "=" * 70)
|
||||
print("Testing: Invalid hook handling")
|
||||
print("=" * 70)
|
||||
|
||||
# Intentionally broken hook
|
||||
hooks_code = {
|
||||
"on_page_context_created": """
|
||||
def hook(page, context): # Missing async!
|
||||
return page
|
||||
""",
|
||||
|
||||
"before_retrieve_html": """
|
||||
async def hook(page, context, **kwargs):
|
||||
# This will cause an error
|
||||
await page.non_existent_method()
|
||||
return page
|
||||
"""
|
||||
}
|
||||
|
||||
payload = {
|
||||
"urls": ["https://httpbin.org/html"],
|
||||
"hooks": {
|
||||
"code": hooks_code,
|
||||
"timeout": 5
|
||||
}
|
||||
}
|
||||
|
||||
print("Sending request with invalid hooks...")
|
||||
response = requests.post(f"{API_BASE_URL}/crawl", json=payload)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
|
||||
if 'hooks' in data:
|
||||
hooks_info = data['hooks']
|
||||
print(f"\nHooks Status: {hooks_info['status']['status']}")
|
||||
|
||||
if hooks_info['status']['validation_errors']:
|
||||
print("\n✅ Validation caught errors (as expected):")
|
||||
for error in hooks_info['status']['validation_errors']:
|
||||
print(f" - {error['hook_point']}: {error['error']}")
|
||||
|
||||
if hooks_info['errors']:
|
||||
print("\n✅ Runtime errors handled gracefully:")
|
||||
for error in hooks_info['errors']:
|
||||
print(f" - {error['hook_point']}: {error['error']}")
|
||||
|
||||
# The crawl should still succeed despite hook errors
|
||||
if data.get('success'):
|
||||
print("\n✅ Crawl succeeded despite hook errors (error isolation working!)")
|
||||
|
||||
else:
|
||||
print(f"Error: {response.status_code}")
|
||||
print(response.text)
|
||||
|
||||
|
||||
def test_authentication_hook():
|
||||
"""Test authentication using hooks"""
|
||||
print("\n" + "=" * 70)
|
||||
print("Testing: Authentication with hooks")
|
||||
print("=" * 70)
|
||||
|
||||
hooks_code = {
|
||||
"before_goto": """
|
||||
async def hook(page, context, url, **kwargs):
|
||||
# For httpbin.org basic auth test, set Authorization header
|
||||
import base64
|
||||
|
||||
# httpbin.org/basic-auth/user/passwd expects username="user" and password="passwd"
|
||||
credentials = base64.b64encode(b"user:passwd").decode('ascii')
|
||||
|
||||
await page.set_extra_http_headers({
|
||||
'Authorization': f'Basic {credentials}'
|
||||
})
|
||||
|
||||
print(f"Hook: Set Authorization header for {url}")
|
||||
return page
|
||||
""",
|
||||
"on_page_context_created": """
|
||||
async def hook(page, context, **kwargs):
|
||||
# Example: Add cookies for session tracking
|
||||
await context.add_cookies([
|
||||
{
|
||||
'name': 'session_id',
|
||||
'value': 'test_session_123',
|
||||
'domain': '.httpbin.org',
|
||||
'path': '/',
|
||||
'httpOnly': True,
|
||||
'secure': True
|
||||
}
|
||||
])
|
||||
|
||||
print("Hook: Added session cookie")
|
||||
return page
|
||||
"""
|
||||
}
|
||||
|
||||
payload = {
|
||||
"urls": ["https://httpbin.org/basic-auth/user/passwd"],
|
||||
"hooks": {
|
||||
"code": hooks_code,
|
||||
"timeout": 30
|
||||
}
|
||||
}
|
||||
|
||||
print("Sending request with authentication hook...")
|
||||
response = requests.post(f"{API_BASE_URL}/crawl", json=payload)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
if data.get('success'):
|
||||
print("✅ Crawl with authentication hook successful")
|
||||
|
||||
# Check if hooks executed
|
||||
if 'hooks' in data:
|
||||
hooks_info = data['hooks']
|
||||
if hooks_info.get('summary', {}).get('successful', 0) > 0:
|
||||
print(f"✅ Authentication hooks executed: {hooks_info['summary']['successful']} successful")
|
||||
|
||||
# Check for any hook errors
|
||||
if hooks_info.get('errors'):
|
||||
print("⚠️ Hook errors:")
|
||||
for error in hooks_info['errors']:
|
||||
print(f" - {error}")
|
||||
|
||||
# Check if authentication worked by looking at the result
|
||||
if 'results' in data and len(data['results']) > 0:
|
||||
result = data['results'][0]
|
||||
if result.get('success'):
|
||||
print("✅ Page crawled successfully (authentication worked!)")
|
||||
# httpbin.org/basic-auth returns JSON with authenticated=true when successful
|
||||
if 'authenticated' in str(result.get('html', '')):
|
||||
print("✅ Authentication confirmed in response content")
|
||||
else:
|
||||
print(f"❌ Crawl failed: {result.get('error_message', 'Unknown error')}")
|
||||
else:
|
||||
print("❌ Request failed")
|
||||
print(f"Response: {json.dumps(data, indent=2)}")
|
||||
else:
|
||||
print(f"❌ Error: {response.status_code}")
|
||||
try:
|
||||
error_data = response.json()
|
||||
print(f"Error details: {json.dumps(error_data, indent=2)}")
|
||||
except:
|
||||
print(f"Error text: {response.text[:500]}")
|
||||
|
||||
|
||||
def test_streaming_with_hooks():
|
||||
"""Test streaming endpoint with hooks"""
|
||||
print("\n" + "=" * 70)
|
||||
print("Testing: POST /crawl/stream with hooks")
|
||||
print("=" * 70)
|
||||
|
||||
hooks_code = {
|
||||
"before_retrieve_html": """
|
||||
async def hook(page, context, **kwargs):
|
||||
await page.evaluate("document.querySelectorAll('img').forEach(img => img.remove())")
|
||||
return page
|
||||
"""
|
||||
}
|
||||
|
||||
payload = {
|
||||
"urls": ["https://httpbin.org/html", "https://httpbin.org/json"],
|
||||
"hooks": {
|
||||
"code": hooks_code,
|
||||
"timeout": 10
|
||||
}
|
||||
}
|
||||
|
||||
print("Sending streaming request with hooks...")
|
||||
|
||||
with requests.post(f"{API_BASE_URL}/crawl/stream", json=payload, stream=True) as response:
|
||||
if response.status_code == 200:
|
||||
# Check headers for hooks status
|
||||
hooks_status = response.headers.get('X-Hooks-Status')
|
||||
if hooks_status:
|
||||
print(f"Hooks Status (from header): {hooks_status}")
|
||||
|
||||
print("\nStreaming results:")
|
||||
for line in response.iter_lines():
|
||||
if line:
|
||||
try:
|
||||
result = json.loads(line)
|
||||
if 'url' in result:
|
||||
print(f" Received: {result['url']}")
|
||||
elif 'status' in result:
|
||||
print(f" Stream status: {result['status']}")
|
||||
except json.JSONDecodeError:
|
||||
print(f" Raw: {line.decode()}")
|
||||
else:
|
||||
print(f"Error: {response.status_code}")
|
||||
|
||||
|
||||
def test_basic_without_hooks():
|
||||
"""Test basic crawl without hooks"""
|
||||
print("\n" + "=" * 70)
|
||||
print("Testing: POST /crawl with no hooks")
|
||||
print("=" * 70)
|
||||
|
||||
payload = {
|
||||
"urls": ["https://httpbin.org/html", "https://httpbin.org/json"]
|
||||
}
|
||||
|
||||
response = requests.post(f"{API_BASE_URL}/crawl", json=payload)
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
print(f"Response: {json.dumps(data, indent=2)}")
|
||||
else:
|
||||
print(f"Error: {response.status_code}")
|
||||
|
||||
|
||||
def main():
|
||||
"""Run all tests"""
|
||||
print("🔧 Crawl4AI Docker API - Hooks Testing")
|
||||
print("=" * 70)
|
||||
|
||||
# Test 1: Get hooks information
|
||||
# test_hooks_info()
|
||||
|
||||
# Test 2: Basic crawl with hooks
|
||||
# test_basic_crawl_with_hooks()
|
||||
|
||||
# Test 3: Invalid hooks (error handling)
|
||||
test_invalid_hook()
|
||||
|
||||
# # Test 4: Authentication hook
|
||||
# test_authentication_hook()
|
||||
|
||||
# # Test 5: Streaming with hooks
|
||||
# test_streaming_with_hooks()
|
||||
|
||||
# # Test 6: Basic crawl without hooks
|
||||
# test_basic_without_hooks()
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("✅ All tests completed!")
|
||||
print("=" * 70)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,558 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Comprehensive test demonstrating all hook types from hooks_example.py
|
||||
adapted for the Docker API with real URLs
|
||||
"""
|
||||
|
||||
import requests
|
||||
import json
|
||||
import time
|
||||
from typing import Dict, Optional
|
||||
|
||||
API_BASE_URL = "http://localhost:11235"
|
||||
|
||||
# Global token storage
|
||||
_auth_token: Optional[str] = None
|
||||
|
||||
|
||||
def get_auth_token(email: str = "test@gmail.com") -> str:
|
||||
"""
|
||||
Get a JWT token from the /token endpoint.
|
||||
The email domain must have valid MX records.
|
||||
"""
|
||||
global _auth_token
|
||||
|
||||
if _auth_token:
|
||||
return _auth_token
|
||||
|
||||
print(f"🔐 Requesting JWT token for {email}...")
|
||||
response = requests.post(
|
||||
f"{API_BASE_URL}/token",
|
||||
json={"email": email}
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
_auth_token = data["access_token"]
|
||||
print(f"✅ Token obtained successfully")
|
||||
return _auth_token
|
||||
else:
|
||||
raise Exception(f"Failed to get token: {response.status_code} - {response.text}")
|
||||
|
||||
|
||||
def get_auth_headers() -> Dict[str, str]:
|
||||
"""Get headers with JWT Bearer token."""
|
||||
token = get_auth_token()
|
||||
return {
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
|
||||
def test_all_hooks_demo():
|
||||
"""Demonstrate all 8 hook types with practical examples"""
|
||||
print("=" * 70)
|
||||
print("Testing: All Hooks Comprehensive Demo")
|
||||
print("=" * 70)
|
||||
|
||||
hooks_code = {
|
||||
"on_browser_created": """
|
||||
async def hook(browser, **kwargs):
|
||||
# Hook called after browser is created
|
||||
print("[HOOK] on_browser_created - Browser is ready!")
|
||||
# Browser-level configurations would go here
|
||||
return browser
|
||||
""",
|
||||
|
||||
"on_page_context_created": """
|
||||
async def hook(page, context, **kwargs):
|
||||
# Hook called after a new page and context are created
|
||||
print("[HOOK] on_page_context_created - New page created!")
|
||||
|
||||
# Set viewport size for consistent rendering
|
||||
await page.set_viewport_size({"width": 1920, "height": 1080})
|
||||
|
||||
# Add cookies for the session (using httpbin.org domain)
|
||||
await context.add_cookies([
|
||||
{
|
||||
"name": "test_session",
|
||||
"value": "abc123xyz",
|
||||
"domain": ".httpbin.org",
|
||||
"path": "/",
|
||||
"httpOnly": True,
|
||||
"secure": True
|
||||
}
|
||||
])
|
||||
|
||||
# Block ads and tracking scripts to speed up crawling
|
||||
await context.route("**/*.{png,jpg,jpeg,gif,webp,svg}", lambda route: route.abort())
|
||||
await context.route("**/analytics/*", lambda route: route.abort())
|
||||
await context.route("**/ads/*", lambda route: route.abort())
|
||||
|
||||
print("[HOOK] Viewport set, cookies added, and ads blocked")
|
||||
return page
|
||||
""",
|
||||
|
||||
"on_user_agent_updated": """
|
||||
async def hook(page, context, user_agent, **kwargs):
|
||||
# Hook called when user agent is updated
|
||||
print(f"[HOOK] on_user_agent_updated - User agent: {user_agent[:50]}...")
|
||||
return page
|
||||
""",
|
||||
|
||||
"before_goto": """
|
||||
async def hook(page, context, url, **kwargs):
|
||||
# Hook called before navigating to each URL
|
||||
print(f"[HOOK] before_goto - About to visit: {url}")
|
||||
|
||||
# Add custom headers for the request
|
||||
await page.set_extra_http_headers({
|
||||
"X-Custom-Header": "crawl4ai-test",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"DNT": "1"
|
||||
})
|
||||
|
||||
return page
|
||||
""",
|
||||
|
||||
"after_goto": """
|
||||
async def hook(page, context, url, response, **kwargs):
|
||||
# Hook called after navigating to each URL
|
||||
print(f"[HOOK] after_goto - Successfully loaded: {url}")
|
||||
|
||||
# Wait a moment for dynamic content to load
|
||||
await page.wait_for_timeout(1000)
|
||||
|
||||
# Check if specific elements exist (with error handling)
|
||||
try:
|
||||
# For httpbin.org, wait for body element
|
||||
await page.wait_for_selector("body", timeout=2000)
|
||||
print("[HOOK] Body element found and loaded")
|
||||
except:
|
||||
print("[HOOK] Timeout waiting for body, continuing anyway")
|
||||
|
||||
return page
|
||||
""",
|
||||
|
||||
"on_execution_started": """
|
||||
async def hook(page, context, **kwargs):
|
||||
# Hook called after custom JavaScript execution
|
||||
print("[HOOK] on_execution_started - Custom JS executed!")
|
||||
|
||||
# You could inject additional JavaScript here if needed
|
||||
await page.evaluate("console.log('[INJECTED] Hook JS running');")
|
||||
|
||||
return page
|
||||
""",
|
||||
|
||||
"before_retrieve_html": """
|
||||
async def hook(page, context, **kwargs):
|
||||
# Hook called before retrieving the HTML content
|
||||
print("[HOOK] before_retrieve_html - Preparing to get HTML")
|
||||
|
||||
# Scroll to bottom to trigger lazy loading
|
||||
await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
|
||||
await page.wait_for_timeout(500)
|
||||
|
||||
# Scroll back to top
|
||||
await page.evaluate("window.scrollTo(0, 0);")
|
||||
await page.wait_for_timeout(500)
|
||||
|
||||
# One more scroll to middle for good measure
|
||||
await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 2);")
|
||||
|
||||
print("[HOOK] Scrolling completed for lazy-loaded content")
|
||||
return page
|
||||
""",
|
||||
|
||||
"before_return_html": """
|
||||
async def hook(page, context, html, **kwargs):
|
||||
# Hook called before returning the HTML content
|
||||
print(f"[HOOK] before_return_html - HTML length: {len(html)} characters")
|
||||
|
||||
# Log some page metrics
|
||||
metrics = await page.evaluate('''() => {
|
||||
return {
|
||||
images: document.images.length,
|
||||
links: document.links.length,
|
||||
scripts: document.scripts.length
|
||||
}
|
||||
}''')
|
||||
|
||||
print(f"[HOOK] Page metrics - Images: {metrics['images']}, Links: {metrics['links']}, Scripts: {metrics['scripts']}")
|
||||
|
||||
return page
|
||||
"""
|
||||
}
|
||||
|
||||
# Create request payload
|
||||
payload = {
|
||||
"urls": ["https://httpbin.org/html"],
|
||||
"hooks": {
|
||||
"code": hooks_code,
|
||||
"timeout": 30
|
||||
},
|
||||
"crawler_config": {
|
||||
"js_code": "window.scrollTo(0, document.body.scrollHeight);",
|
||||
"wait_for": "body",
|
||||
"cache_mode": "bypass"
|
||||
}
|
||||
}
|
||||
|
||||
print("\nSending request with all 8 hooks...")
|
||||
start_time = time.time()
|
||||
|
||||
response = requests.post(f"{API_BASE_URL}/crawl", json=payload, headers=get_auth_headers())
|
||||
|
||||
elapsed_time = time.time() - start_time
|
||||
print(f"Request completed in {elapsed_time:.2f} seconds")
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
print("\n✅ Request successful!")
|
||||
|
||||
# Check hooks execution
|
||||
if 'hooks' in data:
|
||||
hooks_info = data['hooks']
|
||||
print("\n📊 Hooks Execution Summary:")
|
||||
print(f" Status: {hooks_info['status']['status']}")
|
||||
print(f" Attached hooks: {len(hooks_info['status']['attached_hooks'])}")
|
||||
|
||||
for hook_name in hooks_info['status']['attached_hooks']:
|
||||
print(f" ✓ {hook_name}")
|
||||
|
||||
if 'summary' in hooks_info:
|
||||
summary = hooks_info['summary']
|
||||
print(f"\n📈 Execution Statistics:")
|
||||
print(f" Total executions: {summary['total_executions']}")
|
||||
print(f" Successful: {summary['successful']}")
|
||||
print(f" Failed: {summary['failed']}")
|
||||
print(f" Timed out: {summary['timed_out']}")
|
||||
print(f" Success rate: {summary['success_rate']:.1f}%")
|
||||
|
||||
if hooks_info.get('execution_log'):
|
||||
print(f"\n📝 Execution Log:")
|
||||
for log_entry in hooks_info['execution_log']:
|
||||
status_icon = "✅" if log_entry['status'] == 'success' else "❌"
|
||||
exec_time = log_entry.get('execution_time', 0)
|
||||
print(f" {status_icon} {log_entry['hook_point']}: {exec_time:.3f}s")
|
||||
|
||||
# Check crawl results
|
||||
if 'results' in data and len(data['results']) > 0:
|
||||
print(f"\n📄 Crawl Results:")
|
||||
for result in data['results']:
|
||||
print(f" URL: {result['url']}")
|
||||
print(f" Success: {result.get('success', False)}")
|
||||
if result.get('html'):
|
||||
print(f" HTML length: {len(result['html'])} characters")
|
||||
|
||||
else:
|
||||
print(f"❌ Error: {response.status_code}")
|
||||
try:
|
||||
error_data = response.json()
|
||||
print(f"Error details: {json.dumps(error_data, indent=2)}")
|
||||
except:
|
||||
print(f"Error text: {response.text[:500]}")
|
||||
|
||||
|
||||
def test_authentication_flow():
|
||||
"""Test a complete authentication flow with multiple hooks"""
|
||||
print("\n" + "=" * 70)
|
||||
print("Testing: Authentication Flow with Multiple Hooks")
|
||||
print("=" * 70)
|
||||
|
||||
hooks_code = {
|
||||
"on_page_context_created": """
|
||||
async def hook(page, context, **kwargs):
|
||||
print("[HOOK] Setting up authentication context")
|
||||
|
||||
# Add authentication cookies
|
||||
await context.add_cookies([
|
||||
{
|
||||
"name": "auth_token",
|
||||
"value": "fake_jwt_token_here",
|
||||
"domain": ".httpbin.org",
|
||||
"path": "/",
|
||||
"httpOnly": True,
|
||||
"secure": True
|
||||
}
|
||||
])
|
||||
|
||||
# Set localStorage items (for SPA authentication)
|
||||
await page.evaluate('''
|
||||
localStorage.setItem('user_id', '12345');
|
||||
localStorage.setItem('auth_time', new Date().toISOString());
|
||||
''')
|
||||
|
||||
return page
|
||||
""",
|
||||
|
||||
"before_goto": """
|
||||
async def hook(page, context, url, **kwargs):
|
||||
print(f"[HOOK] Adding auth headers for {url}")
|
||||
|
||||
# Add Authorization header
|
||||
import base64
|
||||
credentials = base64.b64encode(b"user:passwd").decode('ascii')
|
||||
|
||||
await page.set_extra_http_headers({
|
||||
'Authorization': f'Basic {credentials}',
|
||||
'X-API-Key': 'test-api-key-123'
|
||||
})
|
||||
|
||||
return page
|
||||
"""
|
||||
}
|
||||
|
||||
payload = {
|
||||
"urls": [
|
||||
"https://httpbin.org/basic-auth/user/passwd"
|
||||
],
|
||||
"hooks": {
|
||||
"code": hooks_code,
|
||||
"timeout": 15
|
||||
}
|
||||
}
|
||||
|
||||
print("\nTesting authentication with httpbin endpoints...")
|
||||
response = requests.post(f"{API_BASE_URL}/crawl", json=payload, headers=get_auth_headers())
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
print("✅ Authentication test completed")
|
||||
|
||||
if 'results' in data:
|
||||
for i, result in enumerate(data['results']):
|
||||
print(f"\n URL {i+1}: {result['url']}")
|
||||
if result.get('success'):
|
||||
# Check for authentication success indicators
|
||||
html_content = result.get('html', '')
|
||||
if '"authenticated"' in html_content and 'true' in html_content:
|
||||
print(" ✅ Authentication successful! Basic auth worked.")
|
||||
else:
|
||||
print(" ⚠️ Page loaded but auth status unclear")
|
||||
else:
|
||||
print(f" ❌ Failed: {result.get('error_message', 'Unknown error')}")
|
||||
else:
|
||||
print(f"❌ Error: {response.status_code}")
|
||||
|
||||
|
||||
def test_performance_optimization_hooks():
|
||||
"""Test hooks for performance optimization"""
|
||||
print("\n" + "=" * 70)
|
||||
print("Testing: Performance Optimization Hooks")
|
||||
print("=" * 70)
|
||||
|
||||
hooks_code = {
|
||||
"on_page_context_created": """
|
||||
async def hook(page, context, **kwargs):
|
||||
print("[HOOK] Optimizing page for performance")
|
||||
|
||||
# Block resource-heavy content
|
||||
await context.route("**/*.{png,jpg,jpeg,gif,webp,svg,ico}", lambda route: route.abort())
|
||||
await context.route("**/*.{woff,woff2,ttf,otf}", lambda route: route.abort())
|
||||
await context.route("**/*.{mp4,webm,ogg,mp3,wav}", lambda route: route.abort())
|
||||
await context.route("**/googletagmanager.com/*", lambda route: route.abort())
|
||||
await context.route("**/google-analytics.com/*", lambda route: route.abort())
|
||||
await context.route("**/doubleclick.net/*", lambda route: route.abort())
|
||||
await context.route("**/facebook.com/*", lambda route: route.abort())
|
||||
|
||||
# Disable animations and transitions
|
||||
await page.add_style_tag(content='''
|
||||
*, *::before, *::after {
|
||||
animation-duration: 0s !important;
|
||||
animation-delay: 0s !important;
|
||||
transition-duration: 0s !important;
|
||||
transition-delay: 0s !important;
|
||||
}
|
||||
''')
|
||||
|
||||
print("[HOOK] Performance optimizations applied")
|
||||
return page
|
||||
""",
|
||||
|
||||
"before_retrieve_html": """
|
||||
async def hook(page, context, **kwargs):
|
||||
print("[HOOK] Removing unnecessary elements before extraction")
|
||||
|
||||
# Remove ads, popups, and other unnecessary elements
|
||||
await page.evaluate('''() => {
|
||||
// Remove common ad containers
|
||||
const adSelectors = [
|
||||
'.ad', '.ads', '.advertisement', '[id*="ad-"]', '[class*="ad-"]',
|
||||
'.popup', '.modal', '.overlay', '.cookie-banner', '.newsletter-signup'
|
||||
];
|
||||
|
||||
adSelectors.forEach(selector => {
|
||||
document.querySelectorAll(selector).forEach(el => el.remove());
|
||||
});
|
||||
|
||||
// Remove script tags to clean up HTML
|
||||
document.querySelectorAll('script').forEach(el => el.remove());
|
||||
|
||||
// Remove style tags we don't need
|
||||
document.querySelectorAll('style').forEach(el => el.remove());
|
||||
}''')
|
||||
|
||||
return page
|
||||
"""
|
||||
}
|
||||
|
||||
payload = {
|
||||
"urls": ["https://httpbin.org/html"],
|
||||
"hooks": {
|
||||
"code": hooks_code,
|
||||
"timeout": 10
|
||||
}
|
||||
}
|
||||
|
||||
print("\nTesting performance optimization hooks...")
|
||||
start_time = time.time()
|
||||
|
||||
response = requests.post(f"{API_BASE_URL}/crawl", json=payload, headers=get_auth_headers())
|
||||
|
||||
elapsed_time = time.time() - start_time
|
||||
print(f"Request completed in {elapsed_time:.2f} seconds")
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
print("✅ Performance optimization test completed")
|
||||
|
||||
if 'results' in data and len(data['results']) > 0:
|
||||
result = data['results'][0]
|
||||
if result.get('html'):
|
||||
print(f" HTML size: {len(result['html'])} characters")
|
||||
print(" Resources blocked, ads removed, animations disabled")
|
||||
else:
|
||||
print(f"❌ Error: {response.status_code}")
|
||||
|
||||
|
||||
def test_content_extraction_hooks():
|
||||
"""Test hooks for intelligent content extraction"""
|
||||
print("\n" + "=" * 70)
|
||||
print("Testing: Content Extraction Hooks")
|
||||
print("=" * 70)
|
||||
|
||||
hooks_code = {
|
||||
"after_goto": """
|
||||
async def hook(page, context, url, response, **kwargs):
|
||||
print(f"[HOOK] Waiting for dynamic content on {url}")
|
||||
|
||||
# Wait for any lazy-loaded content
|
||||
await page.wait_for_timeout(2000)
|
||||
|
||||
# Trigger any "Load More" buttons
|
||||
try:
|
||||
load_more = await page.query_selector('[class*="load-more"], [class*="show-more"], button:has-text("Load More")')
|
||||
if load_more:
|
||||
await load_more.click()
|
||||
await page.wait_for_timeout(1000)
|
||||
print("[HOOK] Clicked 'Load More' button")
|
||||
except:
|
||||
pass
|
||||
|
||||
return page
|
||||
""",
|
||||
|
||||
"before_retrieve_html": """
|
||||
async def hook(page, context, **kwargs):
|
||||
print("[HOOK] Extracting structured data")
|
||||
|
||||
# Extract metadata
|
||||
metadata = await page.evaluate('''() => {
|
||||
const getMeta = (name) => {
|
||||
const element = document.querySelector(`meta[name="${name}"], meta[property="${name}"]`);
|
||||
return element ? element.getAttribute('content') : null;
|
||||
};
|
||||
|
||||
return {
|
||||
title: document.title,
|
||||
description: getMeta('description') || getMeta('og:description'),
|
||||
author: getMeta('author'),
|
||||
keywords: getMeta('keywords'),
|
||||
ogTitle: getMeta('og:title'),
|
||||
ogImage: getMeta('og:image'),
|
||||
canonical: document.querySelector('link[rel="canonical"]')?.href,
|
||||
jsonLd: Array.from(document.querySelectorAll('script[type="application/ld+json"]'))
|
||||
.map(el => el.textContent).filter(Boolean)
|
||||
};
|
||||
}''')
|
||||
|
||||
print(f"[HOOK] Extracted metadata: {json.dumps(metadata, indent=2)}")
|
||||
|
||||
# Infinite scroll handling
|
||||
for i in range(3):
|
||||
await page.evaluate("window.scrollTo(0, document.body.scrollHeight);")
|
||||
await page.wait_for_timeout(1000)
|
||||
print(f"[HOOK] Scroll iteration {i+1}/3")
|
||||
|
||||
return page
|
||||
"""
|
||||
}
|
||||
|
||||
payload = {
|
||||
"urls": ["https://httpbin.org/html", "https://httpbin.org/json"],
|
||||
"hooks": {
|
||||
"code": hooks_code,
|
||||
"timeout": 20
|
||||
}
|
||||
}
|
||||
|
||||
print("\nTesting content extraction hooks...")
|
||||
response = requests.post(f"{API_BASE_URL}/crawl", json=payload, headers=get_auth_headers())
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
print("✅ Content extraction test completed")
|
||||
|
||||
if 'hooks' in data and 'summary' in data['hooks']:
|
||||
summary = data['hooks']['summary']
|
||||
print(f" Hooks executed: {summary['successful']}/{summary['total_executions']}")
|
||||
|
||||
if 'results' in data:
|
||||
for result in data['results']:
|
||||
print(f"\n URL: {result['url']}")
|
||||
print(f" Success: {result.get('success', False)}")
|
||||
else:
|
||||
print(f"❌ Error: {response.status_code}")
|
||||
|
||||
|
||||
def main():
|
||||
"""Run comprehensive hook tests"""
|
||||
print("🔧 Crawl4AI Docker API - Comprehensive Hooks Testing")
|
||||
print("Based on docs/examples/hooks_example.py")
|
||||
print("=" * 70)
|
||||
|
||||
# Get JWT token first (required when jwt_enabled=true)
|
||||
try:
|
||||
get_auth_token()
|
||||
print("=" * 70)
|
||||
except Exception as e:
|
||||
print(f"❌ Failed to authenticate: {e}")
|
||||
print("Make sure the server is running and jwt_enabled is configured correctly.")
|
||||
return
|
||||
|
||||
tests = [
|
||||
("All Hooks Demo", test_all_hooks_demo),
|
||||
("Authentication Flow", test_authentication_flow),
|
||||
("Performance Optimization", test_performance_optimization_hooks),
|
||||
("Content Extraction", test_content_extraction_hooks),
|
||||
]
|
||||
|
||||
for i, (name, test_func) in enumerate(tests, 1):
|
||||
print(f"\n📌 Test {i}/{len(tests)}: {name}")
|
||||
try:
|
||||
test_func()
|
||||
print(f"✅ {name} completed")
|
||||
except Exception as e:
|
||||
print(f"❌ {name} failed: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("🎉 All comprehensive hook tests completed!")
|
||||
print("=" * 70)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,193 @@
|
||||
"""
|
||||
Test script demonstrating the hooks_to_string utility and Docker client integration.
|
||||
"""
|
||||
import asyncio
|
||||
from crawl4ai import Crawl4aiDockerClient, hooks_to_string
|
||||
|
||||
|
||||
# Define hook functions as regular Python functions
|
||||
async def auth_hook(page, context, **kwargs):
|
||||
"""Add authentication cookies."""
|
||||
await context.add_cookies([{
|
||||
'name': 'test_cookie',
|
||||
'value': 'test_value',
|
||||
'domain': '.httpbin.org',
|
||||
'path': '/'
|
||||
}])
|
||||
return page
|
||||
|
||||
|
||||
async def scroll_hook(page, context, **kwargs):
|
||||
"""Scroll to load lazy content."""
|
||||
await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
||||
await page.wait_for_timeout(1000)
|
||||
return page
|
||||
|
||||
|
||||
async def viewport_hook(page, context, **kwargs):
|
||||
"""Set custom viewport."""
|
||||
await page.set_viewport_size({"width": 1920, "height": 1080})
|
||||
return page
|
||||
|
||||
|
||||
async def test_hooks_utility():
|
||||
"""Test the hooks_to_string utility function."""
|
||||
print("=" * 60)
|
||||
print("Testing hooks_to_string utility")
|
||||
print("=" * 60)
|
||||
|
||||
# Create hooks dictionary with function objects
|
||||
hooks_dict = {
|
||||
"on_page_context_created": auth_hook,
|
||||
"before_retrieve_html": scroll_hook
|
||||
}
|
||||
|
||||
# Convert to string format
|
||||
hooks_string = hooks_to_string(hooks_dict)
|
||||
|
||||
print("\n✓ Successfully converted function objects to strings")
|
||||
print(f"\n✓ Converted {len(hooks_string)} hooks:")
|
||||
for hook_name in hooks_string.keys():
|
||||
print(f" - {hook_name}")
|
||||
|
||||
print("\n✓ Preview of converted hook:")
|
||||
print("-" * 60)
|
||||
print(hooks_string["on_page_context_created"][:200] + "...")
|
||||
print("-" * 60)
|
||||
|
||||
return hooks_string
|
||||
|
||||
|
||||
async def test_docker_client_with_functions():
|
||||
"""Test Docker client with function objects (automatic conversion)."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Testing Docker Client with Function Objects")
|
||||
print("=" * 60)
|
||||
|
||||
# Note: This requires a running Crawl4AI Docker server
|
||||
# Uncomment the following to test with actual server:
|
||||
|
||||
async with Crawl4aiDockerClient(base_url="http://localhost:11234", verbose=True) as client:
|
||||
# Pass function objects directly - they'll be converted automatically
|
||||
result = await client.crawl(
|
||||
["https://httpbin.org/html"],
|
||||
hooks={
|
||||
"on_page_context_created": auth_hook,
|
||||
"before_retrieve_html": scroll_hook
|
||||
},
|
||||
hooks_timeout=30
|
||||
)
|
||||
print(f"\n✓ Crawl successful: {result.success}")
|
||||
print(f"✓ URL: {result.url}")
|
||||
|
||||
print("\n✓ Docker client accepts function objects directly")
|
||||
print("✓ Automatic conversion happens internally")
|
||||
print("✓ No manual string formatting needed!")
|
||||
|
||||
|
||||
async def test_docker_client_with_strings():
|
||||
"""Test Docker client with pre-converted strings."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Testing Docker Client with String Hooks")
|
||||
print("=" * 60)
|
||||
|
||||
# Convert hooks to strings first
|
||||
hooks_dict = {
|
||||
"on_page_context_created": viewport_hook,
|
||||
"before_retrieve_html": scroll_hook
|
||||
}
|
||||
hooks_string = hooks_to_string(hooks_dict)
|
||||
|
||||
# Note: This requires a running Crawl4AI Docker server
|
||||
# Uncomment the following to test with actual server:
|
||||
|
||||
async with Crawl4aiDockerClient(base_url="http://localhost:11234", verbose=True) as client:
|
||||
# Pass string hooks - they'll be used as-is
|
||||
result = await client.crawl(
|
||||
["https://httpbin.org/html"],
|
||||
hooks=hooks_string,
|
||||
hooks_timeout=30
|
||||
)
|
||||
print(f"\n✓ Crawl successful: {result.success}")
|
||||
|
||||
print("\n✓ Docker client also accepts pre-converted strings")
|
||||
print("✓ Backward compatible with existing code")
|
||||
|
||||
|
||||
async def show_usage_patterns():
|
||||
"""Show different usage patterns."""
|
||||
print("\n" + "=" * 60)
|
||||
print("Usage Patterns")
|
||||
print("=" * 60)
|
||||
|
||||
print("\n1. Direct function usage (simplest):")
|
||||
print("-" * 60)
|
||||
print("""
|
||||
async def my_hook(page, context, **kwargs):
|
||||
await page.set_viewport_size({"width": 1920, "height": 1080})
|
||||
return page
|
||||
|
||||
result = await client.crawl(
|
||||
["https://example.com"],
|
||||
hooks={"on_page_context_created": my_hook}
|
||||
)
|
||||
""")
|
||||
|
||||
print("\n2. Convert then use:")
|
||||
print("-" * 60)
|
||||
print("""
|
||||
hooks_dict = {"on_page_context_created": my_hook}
|
||||
hooks_string = hooks_to_string(hooks_dict)
|
||||
|
||||
result = await client.crawl(
|
||||
["https://example.com"],
|
||||
hooks=hooks_string
|
||||
)
|
||||
""")
|
||||
|
||||
print("\n3. Manual string (backward compatible):")
|
||||
print("-" * 60)
|
||||
print("""
|
||||
hooks_string = {
|
||||
"on_page_context_created": '''
|
||||
async def hook(page, context, **kwargs):
|
||||
await page.set_viewport_size({"width": 1920, "height": 1080})
|
||||
return page
|
||||
'''
|
||||
}
|
||||
|
||||
result = await client.crawl(
|
||||
["https://example.com"],
|
||||
hooks=hooks_string
|
||||
)
|
||||
""")
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all tests."""
|
||||
print("\n🚀 Crawl4AI Hooks Utility Test Suite\n")
|
||||
|
||||
# Test the utility function
|
||||
# await test_hooks_utility()
|
||||
|
||||
# Show usage with Docker client
|
||||
# await test_docker_client_with_functions()
|
||||
await test_docker_client_with_strings()
|
||||
|
||||
# Show different patterns
|
||||
# await show_usage_patterns()
|
||||
|
||||
# print("\n" + "=" * 60)
|
||||
# print("✓ All tests completed successfully!")
|
||||
# print("=" * 60)
|
||||
# print("\nKey Benefits:")
|
||||
# print(" • Write hooks as regular Python functions")
|
||||
# print(" • IDE support with autocomplete and type checking")
|
||||
# print(" • Automatic conversion to API format")
|
||||
# print(" • Backward compatible with string hooks")
|
||||
# print(" • Same utility used everywhere")
|
||||
# print("\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Executable
+349
@@ -0,0 +1,349 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Test script for LLM temperature and base_url parameters in Crawl4AI Docker API.
|
||||
This demonstrates the new hierarchical configuration system:
|
||||
1. Request-level parameters (highest priority)
|
||||
2. Provider-specific environment variables
|
||||
3. Global environment variables
|
||||
4. System defaults (lowest priority)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import httpx
|
||||
import json
|
||||
import os
|
||||
from rich.console import Console
|
||||
from rich.panel import Panel
|
||||
from rich.syntax import Syntax
|
||||
from rich.table import Table
|
||||
|
||||
|
||||
console = Console()
|
||||
|
||||
# Configuration
|
||||
BASE_URL = "http://localhost:11235" # Docker API endpoint
|
||||
TEST_URL = "https://httpbin.org/html" # Simple test page
|
||||
|
||||
# --- Helper Functions ---
|
||||
|
||||
async def check_server_health(client: httpx.AsyncClient) -> bool:
|
||||
"""Check if the server is healthy."""
|
||||
console.print("[bold cyan]Checking server health...[/]", end="")
|
||||
try:
|
||||
response = await client.get("/health", timeout=10.0)
|
||||
response.raise_for_status()
|
||||
console.print(" [bold green]✓ Server is healthy![/]")
|
||||
return True
|
||||
except Exception as e:
|
||||
console.print(f"\n[bold red]✗ Server health check failed: {e}[/]")
|
||||
console.print(f"Is the server running at {BASE_URL}?")
|
||||
return False
|
||||
|
||||
def print_request(endpoint: str, payload: dict, title: str = "Request"):
|
||||
"""Pretty print the request."""
|
||||
syntax = Syntax(json.dumps(payload, indent=2), "json", theme="monokai")
|
||||
console.print(Panel.fit(
|
||||
f"[cyan]POST {endpoint}[/cyan]\n{syntax}",
|
||||
title=f"[bold blue]{title}[/]",
|
||||
border_style="blue"
|
||||
))
|
||||
|
||||
def print_response(response: dict, title: str = "Response"):
|
||||
"""Pretty print relevant parts of the response."""
|
||||
# Extract only the relevant parts
|
||||
relevant = {}
|
||||
if "markdown" in response:
|
||||
relevant["markdown"] = response["markdown"][:200] + "..." if len(response.get("markdown", "")) > 200 else response.get("markdown", "")
|
||||
if "success" in response:
|
||||
relevant["success"] = response["success"]
|
||||
if "url" in response:
|
||||
relevant["url"] = response["url"]
|
||||
if "filter" in response:
|
||||
relevant["filter"] = response["filter"]
|
||||
|
||||
console.print(Panel.fit(
|
||||
Syntax(json.dumps(relevant, indent=2), "json", theme="monokai"),
|
||||
title=f"[bold green]{title}[/]",
|
||||
border_style="green"
|
||||
))
|
||||
|
||||
# --- Test Functions ---
|
||||
|
||||
async def test_default_no_params(client: httpx.AsyncClient):
|
||||
"""Test 1: No temperature or base_url specified - uses defaults"""
|
||||
console.rule("[bold yellow]Test 1: Default Configuration (No Parameters)[/]")
|
||||
|
||||
payload = {
|
||||
"url": TEST_URL,
|
||||
"f": "llm",
|
||||
"q": "What is the main heading of this page? Answer in exactly 5 words."
|
||||
}
|
||||
|
||||
print_request("/md", payload, "Request without temperature/base_url")
|
||||
|
||||
try:
|
||||
response = await client.post("/md", json=payload, timeout=30.0)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
print_response(data, "Response (using system defaults)")
|
||||
console.print("[dim]→ This used system defaults or environment variables if set[/]")
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/]")
|
||||
|
||||
async def test_request_temperature(client: httpx.AsyncClient):
|
||||
"""Test 2: Request-level temperature (highest priority)"""
|
||||
console.rule("[bold yellow]Test 2: Request-Level Temperature[/]")
|
||||
|
||||
# Test with low temperature (more focused)
|
||||
payload_low = {
|
||||
"url": TEST_URL,
|
||||
"f": "llm",
|
||||
"q": "What is the main heading? Be creative and poetic.",
|
||||
"temperature": 0.1 # Very low - should be less creative
|
||||
}
|
||||
|
||||
print_request("/md", payload_low, "Low Temperature (0.1)")
|
||||
|
||||
try:
|
||||
response = await client.post("/md", json=payload_low, timeout=30.0)
|
||||
response.raise_for_status()
|
||||
data_low = response.json()
|
||||
print_response(data_low, "Response with Low Temperature")
|
||||
console.print("[dim]→ Low temperature (0.1) should produce focused, less creative output[/]")
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/]")
|
||||
|
||||
console.print()
|
||||
|
||||
# Test with high temperature (more creative)
|
||||
payload_high = {
|
||||
"url": TEST_URL,
|
||||
"f": "llm",
|
||||
"q": "What is the main heading? Be creative and poetic.",
|
||||
"temperature": 1.5 # High - should be more creative
|
||||
}
|
||||
|
||||
print_request("/md", payload_high, "High Temperature (1.5)")
|
||||
|
||||
try:
|
||||
response = await client.post("/md", json=payload_high, timeout=30.0)
|
||||
response.raise_for_status()
|
||||
data_high = response.json()
|
||||
print_response(data_high, "Response with High Temperature")
|
||||
console.print("[dim]→ High temperature (1.5) should produce more creative, varied output[/]")
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/]")
|
||||
|
||||
async def test_provider_override(client: httpx.AsyncClient):
|
||||
"""Test 3: Provider override with temperature"""
|
||||
console.rule("[bold yellow]Test 3: Provider Override with Temperature[/]")
|
||||
|
||||
provider = "gemini/gemini-2.5-flash-lite"
|
||||
payload = {
|
||||
"url": TEST_URL,
|
||||
"f": "llm",
|
||||
"q": "Summarize this page in one sentence.",
|
||||
"provider": provider, # Explicitly set provider
|
||||
"temperature": 0.7
|
||||
}
|
||||
|
||||
print_request("/md", payload, "Provider + Temperature Override")
|
||||
|
||||
try:
|
||||
response = await client.post("/md", json=payload, timeout=30.0)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
print_response(data, "Response with Provider Override")
|
||||
console.print(f"[dim]→ This explicitly uses {provider} with temperature 0.7[/]")
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/]")
|
||||
|
||||
async def test_base_url_custom(client: httpx.AsyncClient):
|
||||
"""Test 4: Custom base_url (will fail unless you have a custom endpoint)"""
|
||||
console.rule("[bold yellow]Test 4: Custom Base URL (Demo Only)[/]")
|
||||
|
||||
payload = {
|
||||
"url": TEST_URL,
|
||||
"f": "llm",
|
||||
"q": "What is this page about?",
|
||||
"base_url": "https://api.custom-endpoint.com/v1", # Custom endpoint
|
||||
"temperature": 0.5
|
||||
}
|
||||
|
||||
print_request("/md", payload, "Custom Base URL Request")
|
||||
console.print("[yellow]Note: This will fail unless you have a custom endpoint set up[/]")
|
||||
|
||||
try:
|
||||
response = await client.post("/md", json=payload, timeout=10.0)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
print_response(data, "Response from Custom Endpoint")
|
||||
except httpx.HTTPStatusError as e:
|
||||
console.print(f"[yellow]Expected failure (no custom endpoint): Status {e.response.status_code}[/]")
|
||||
except Exception as e:
|
||||
console.print(f"[yellow]Expected error: {e}[/]")
|
||||
|
||||
async def test_llm_job_endpoint(client: httpx.AsyncClient):
|
||||
"""Test 5: Test the /llm/job endpoint with temperature and base_url"""
|
||||
console.rule("[bold yellow]Test 5: LLM Job Endpoint with Parameters[/]")
|
||||
|
||||
payload = {
|
||||
"url": TEST_URL,
|
||||
"q": "Extract the main title and any key information",
|
||||
"temperature": 0.3,
|
||||
# "base_url": "https://api.openai.com/v1" # Optional
|
||||
}
|
||||
|
||||
print_request("/llm/job", payload, "LLM Job with Temperature")
|
||||
|
||||
try:
|
||||
# Submit the job
|
||||
response = await client.post("/llm/job", json=payload, timeout=30.0)
|
||||
response.raise_for_status()
|
||||
job_data = response.json()
|
||||
|
||||
if "task_id" in job_data:
|
||||
task_id = job_data["task_id"]
|
||||
console.print(f"[green]Job created with task_id: {task_id}[/]")
|
||||
|
||||
# Poll for result (simplified - in production use proper polling)
|
||||
await asyncio.sleep(3)
|
||||
|
||||
status_response = await client.get(f"/llm/job/{task_id}")
|
||||
status_data = status_response.json()
|
||||
|
||||
if status_data.get("status") == "completed":
|
||||
console.print("[green]Job completed successfully![/]")
|
||||
if "result" in status_data:
|
||||
console.print(Panel.fit(
|
||||
Syntax(json.dumps(status_data["result"], indent=2), "json", theme="monokai"),
|
||||
title="Extraction Result",
|
||||
border_style="green"
|
||||
))
|
||||
else:
|
||||
console.print(f"[yellow]Job status: {status_data.get('status', 'unknown')}[/]")
|
||||
else:
|
||||
console.print(f"[red]Unexpected response: {job_data}[/]")
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error: {e}[/]")
|
||||
|
||||
|
||||
async def test_llm_endpoint(client: httpx.AsyncClient):
|
||||
"""
|
||||
Quick QA round-trip with /llm.
|
||||
Asks a trivial question against SIMPLE_URL just to show wiring.
|
||||
"""
|
||||
import time
|
||||
import urllib.parse
|
||||
|
||||
page_url = "https://kidocode.com"
|
||||
question = "What is the title of this page?"
|
||||
|
||||
enc = urllib.parse.quote_plus(page_url, safe="")
|
||||
console.print(f"GET /llm/{enc}?q={question}")
|
||||
|
||||
try:
|
||||
t0 = time.time()
|
||||
resp = await client.get(f"/llm/{enc}", params={"q": question})
|
||||
dt = time.time() - t0
|
||||
console.print(
|
||||
f"Response Status: [bold {'green' if resp.is_success else 'red'}]{resp.status_code}[/] (took {dt:.2f}s)")
|
||||
resp.raise_for_status()
|
||||
answer = resp.json().get("answer", "")
|
||||
console.print(Panel(answer or "No answer returned",
|
||||
title="LLM answer", border_style="magenta", expand=False))
|
||||
except Exception as e:
|
||||
console.print(f"[bold red]Error hitting /llm:[/] {e}")
|
||||
|
||||
|
||||
async def show_environment_info():
|
||||
"""Display current environment configuration"""
|
||||
console.rule("[bold cyan]Current Environment Configuration[/]")
|
||||
|
||||
table = Table(title="LLM Environment Variables", show_header=True, header_style="bold magenta")
|
||||
table.add_column("Variable", style="cyan", width=30)
|
||||
table.add_column("Value", style="yellow")
|
||||
table.add_column("Description", style="dim")
|
||||
|
||||
env_vars = [
|
||||
("LLM_PROVIDER", "Global default provider"),
|
||||
("LLM_TEMPERATURE", "Global default temperature"),
|
||||
("LLM_BASE_URL", "Global custom API endpoint"),
|
||||
("OPENAI_API_KEY", "OpenAI API key"),
|
||||
("OPENAI_TEMPERATURE", "OpenAI-specific temperature"),
|
||||
("OPENAI_BASE_URL", "OpenAI-specific endpoint"),
|
||||
("ANTHROPIC_API_KEY", "Anthropic API key"),
|
||||
("ANTHROPIC_TEMPERATURE", "Anthropic-specific temperature"),
|
||||
("GROQ_API_KEY", "Groq API key"),
|
||||
("GROQ_TEMPERATURE", "Groq-specific temperature"),
|
||||
]
|
||||
|
||||
for var, desc in env_vars:
|
||||
value = os.environ.get(var, "[not set]")
|
||||
if "API_KEY" in var and value != "[not set]":
|
||||
# Mask API keys for security
|
||||
value = value[:10] + "..." if len(value) > 10 else "***"
|
||||
table.add_row(var, value, desc)
|
||||
|
||||
console.print(table)
|
||||
console.print()
|
||||
|
||||
# --- Main Test Runner ---
|
||||
|
||||
async def main():
|
||||
"""Run all tests"""
|
||||
console.print(Panel.fit(
|
||||
"[bold cyan]Crawl4AI LLM Parameters Test Suite[/]\n" +
|
||||
"Testing temperature and base_url configuration hierarchy",
|
||||
border_style="cyan"
|
||||
))
|
||||
|
||||
# Show current environment
|
||||
# await show_environment_info()
|
||||
|
||||
# Create HTTP client
|
||||
async with httpx.AsyncClient(base_url=BASE_URL, timeout=60.0) as client:
|
||||
# Check server health
|
||||
if not await check_server_health(client):
|
||||
console.print("[red]Server is not available. Please ensure the Docker container is running.[/]")
|
||||
return
|
||||
|
||||
# Run tests
|
||||
tests = [
|
||||
("Default Configuration", test_default_no_params),
|
||||
("Request Temperature", test_request_temperature),
|
||||
("Provider Override", test_provider_override),
|
||||
("Custom Base URL", test_base_url_custom),
|
||||
("LLM Job Endpoint", test_llm_job_endpoint),
|
||||
("LLM Endpoint", test_llm_endpoint),
|
||||
]
|
||||
|
||||
for i, (name, test_func) in enumerate(tests, 1):
|
||||
if i > 1:
|
||||
console.print() # Add spacing between tests
|
||||
|
||||
try:
|
||||
await test_func(client)
|
||||
except Exception as e:
|
||||
console.print(f"[red]Test '{name}' failed with error: {e}[/]")
|
||||
console.print_exception(show_locals=False)
|
||||
|
||||
console.rule("[bold green]All Tests Complete![/]", style="green")
|
||||
|
||||
# Summary
|
||||
console.print("\n[bold cyan]Configuration Hierarchy Summary:[/]")
|
||||
console.print("1. [yellow]Request parameters[/] - Highest priority (temperature, base_url in API call)")
|
||||
console.print("2. [yellow]Provider-specific env[/] - e.g., OPENAI_TEMPERATURE, GROQ_BASE_URL")
|
||||
console.print("3. [yellow]Global env variables[/] - LLM_TEMPERATURE, LLM_BASE_URL")
|
||||
console.print("4. [yellow]System defaults[/] - Lowest priority (provider/litellm defaults)")
|
||||
console.print()
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except KeyboardInterrupt:
|
||||
console.print("\n[yellow]Tests interrupted by user.[/]")
|
||||
except Exception as e:
|
||||
console.print(f"\n[bold red]An error occurred:[/]")
|
||||
console.print_exception(show_locals=False)
|
||||
@@ -0,0 +1,155 @@
|
||||
"""Tests for crawler pool release_crawler() and active_requests tracking.
|
||||
|
||||
These tests validate the pool lifecycle without requiring Docker or a running
|
||||
server. They test the release logic directly using mock crawler objects.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import pytest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Standalone release_crawler implementation for testing
|
||||
# (mirrors the logic that will be added to deploy/docker/crawler_pool.py)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_TEST_LOCK = asyncio.Lock()
|
||||
|
||||
|
||||
async def _release_crawler(crawler, lock=None):
|
||||
"""Standalone release logic matching crawler_pool.release_crawler()."""
|
||||
lock = lock or _TEST_LOCK
|
||||
async with lock:
|
||||
if hasattr(crawler, "active_requests"):
|
||||
crawler.active_requests = max(0, crawler.active_requests - 1)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestReleaseCrawler:
|
||||
"""Tests for the release_crawler function."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_decrements_active_requests(self):
|
||||
"""release_crawler should decrement active_requests by 1."""
|
||||
crawler = MagicMock()
|
||||
crawler.active_requests = 3
|
||||
|
||||
await _release_crawler(crawler)
|
||||
assert crawler.active_requests == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_floors_at_zero(self):
|
||||
"""active_requests should never go below 0."""
|
||||
crawler = MagicMock()
|
||||
crawler.active_requests = 0
|
||||
|
||||
await _release_crawler(crawler)
|
||||
assert crawler.active_requests == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_from_one_to_zero(self):
|
||||
"""Standard case: single request finishes."""
|
||||
crawler = MagicMock()
|
||||
crawler.active_requests = 1
|
||||
|
||||
await _release_crawler(crawler)
|
||||
assert crawler.active_requests == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_handles_missing_attribute(self):
|
||||
"""Should not crash if crawler has no active_requests attribute."""
|
||||
crawler = MagicMock(spec=[]) # no attributes at all
|
||||
# Should not raise
|
||||
await _release_crawler(crawler)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_releases_decrement_correctly(self):
|
||||
"""Multiple sequential releases should each decrement by 1."""
|
||||
crawler = MagicMock()
|
||||
crawler.active_requests = 5
|
||||
|
||||
for expected in [4, 3, 2, 1, 0, 0]: # last one should floor at 0
|
||||
await _release_crawler(crawler)
|
||||
assert crawler.active_requests == expected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_releases_are_safe(self):
|
||||
"""Concurrent releases should not corrupt the counter."""
|
||||
crawler = MagicMock()
|
||||
crawler.active_requests = 100
|
||||
lock = asyncio.Lock()
|
||||
|
||||
async def release_n_times(n):
|
||||
for _ in range(n):
|
||||
await _release_crawler(crawler, lock=lock)
|
||||
|
||||
# 10 concurrent tasks each releasing 10 times = 100 total
|
||||
tasks = [asyncio.create_task(release_n_times(10)) for _ in range(10)]
|
||||
await asyncio.gather(*tasks)
|
||||
|
||||
assert crawler.active_requests == 0
|
||||
|
||||
|
||||
class TestActiveRequestsTracking:
|
||||
"""Tests for the get/release lifecycle pattern."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_sets_active_requests(self):
|
||||
"""Simulated get_crawler should set active_requests to 1 for new crawlers."""
|
||||
crawler = MagicMock()
|
||||
# Simulate what get_crawler does for a new browser
|
||||
crawler.active_requests = 1
|
||||
|
||||
assert crawler.active_requests == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_increments_existing(self):
|
||||
"""Simulated get_crawler should increment for existing pooled crawlers."""
|
||||
crawler = MagicMock()
|
||||
crawler.active_requests = 2
|
||||
|
||||
# Simulate another get_crawler call returning same browser
|
||||
crawler.active_requests += 1
|
||||
assert crawler.active_requests == 3
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_full_get_release_lifecycle(self):
|
||||
"""Full lifecycle: get -> use -> release -> get -> release."""
|
||||
crawler = MagicMock()
|
||||
|
||||
# First request gets the crawler
|
||||
crawler.active_requests = 1
|
||||
|
||||
# Second concurrent request gets same crawler
|
||||
crawler.active_requests += 1
|
||||
assert crawler.active_requests == 2
|
||||
|
||||
# First request finishes
|
||||
await _release_crawler(crawler)
|
||||
assert crawler.active_requests == 1
|
||||
|
||||
# Second request finishes
|
||||
await _release_crawler(crawler)
|
||||
assert crawler.active_requests == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_janitor_safety_check(self):
|
||||
"""Janitor should only close browsers with active_requests == 0."""
|
||||
crawler = MagicMock()
|
||||
crawler.active_requests = 1
|
||||
|
||||
# Janitor check: should NOT close
|
||||
should_close = getattr(crawler, "active_requests", 0) == 0
|
||||
assert should_close is False
|
||||
|
||||
# Request finishes
|
||||
await _release_crawler(crawler)
|
||||
|
||||
# Janitor check: now safe to close
|
||||
should_close = getattr(crawler, "active_requests", 0) == 0
|
||||
assert should_close is True
|
||||
@@ -0,0 +1,596 @@
|
||||
# ==== File: test_rest_api_deep_crawl.py ====
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import httpx
|
||||
import json
|
||||
import asyncio
|
||||
import os
|
||||
from typing import List, Dict, Any, AsyncGenerator
|
||||
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv() # Load environment variables from .env file if present
|
||||
|
||||
# --- Test Configuration ---
|
||||
BASE_URL = os.getenv("CRAWL4AI_TEST_URL", "http://localhost:11235") # If server is running in Docker, use the host's IP
|
||||
BASE_URL = os.getenv("CRAWL4AI_TEST_URL", "http://localhost:8020") # If server is running in dev debug mode
|
||||
DEEP_CRAWL_BASE_URL = "https://docs.crawl4ai.com/samples/deepcrawl/"
|
||||
DEEP_CRAWL_DOMAIN = "docs.crawl4ai.com" # Used for domain filter
|
||||
|
||||
# --- Helper Functions ---
|
||||
def load_proxies_from_env() -> List[Dict]:
|
||||
"""Load proxies from PROXIES environment variable"""
|
||||
proxies = []
|
||||
proxies_str = os.getenv("PROXIES", "")
|
||||
if not proxies_str:
|
||||
print("PROXIES environment variable not set or empty.")
|
||||
return proxies
|
||||
try:
|
||||
proxy_list = proxies_str.split(",")
|
||||
for proxy in proxy_list:
|
||||
proxy = proxy.strip()
|
||||
if not proxy:
|
||||
continue
|
||||
parts = proxy.split(":")
|
||||
if len(parts) == 4:
|
||||
ip, port, username, password = parts
|
||||
proxies.append({
|
||||
"server": f"http://{ip}:{port}", # Assuming http, adjust if needed
|
||||
"username": username,
|
||||
"password": password,
|
||||
"ip": ip # Store original IP if available
|
||||
})
|
||||
elif len(parts) == 2: # ip:port only
|
||||
ip, port = parts
|
||||
proxies.append({
|
||||
"server": f"http://{ip}:{port}",
|
||||
"ip": ip
|
||||
})
|
||||
else:
|
||||
print(f"Skipping invalid proxy string format: {proxy}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error loading proxies from environment: {e}")
|
||||
return proxies
|
||||
|
||||
|
||||
async def check_server_health(client: httpx.AsyncClient):
|
||||
"""Check if the server is healthy before running tests."""
|
||||
try:
|
||||
response = await client.get("/health")
|
||||
response.raise_for_status()
|
||||
print(f"\nServer healthy: {response.json()}")
|
||||
return True
|
||||
except (httpx.RequestError, httpx.HTTPStatusError) as e:
|
||||
pytest.fail(f"Server health check failed: {e}. Is the server running at {BASE_URL}?", pytrace=False)
|
||||
|
||||
async def assert_crawl_result_structure(result: Dict[str, Any], check_ssl=False):
|
||||
"""Asserts the basic structure of a single crawl result."""
|
||||
assert isinstance(result, dict)
|
||||
assert "url" in result
|
||||
assert "success" in result
|
||||
assert "html" in result # Basic crawls should return HTML
|
||||
assert "metadata" in result
|
||||
assert isinstance(result["metadata"], dict)
|
||||
assert "depth" in result["metadata"] # Deep crawls add depth
|
||||
|
||||
if check_ssl:
|
||||
assert "ssl_certificate" in result # Check if SSL info is present
|
||||
assert isinstance(result["ssl_certificate"], dict) or result["ssl_certificate"] is None
|
||||
|
||||
|
||||
async def process_streaming_response(response: httpx.Response) -> List[Dict[str, Any]]:
|
||||
"""Processes an NDJSON streaming response."""
|
||||
results = []
|
||||
completed = False
|
||||
async for line in response.aiter_lines():
|
||||
if line:
|
||||
try:
|
||||
data = json.loads(line)
|
||||
if data.get("status") == "completed":
|
||||
completed = True
|
||||
break # Stop processing after completion marker
|
||||
elif data.get("url"): # Ensure it looks like a result object
|
||||
results.append(data)
|
||||
else:
|
||||
print(f"Received non-result JSON line: {data}") # Log other status messages if needed
|
||||
except json.JSONDecodeError:
|
||||
pytest.fail(f"Failed to decode JSON line: {line}")
|
||||
assert completed, "Streaming response did not end with a completion marker."
|
||||
return results
|
||||
|
||||
|
||||
# --- Pytest Fixtures ---
|
||||
@pytest_asyncio.fixture(scope="function")
|
||||
async def async_client() -> AsyncGenerator[httpx.AsyncClient, None]:
|
||||
"""Provides an async HTTP client"""
|
||||
# Increased timeout for potentially longer deep crawls
|
||||
async with httpx.AsyncClient(base_url=BASE_URL, timeout=300.0) as client:
|
||||
yield client
|
||||
# No explicit close needed with 'async with'
|
||||
|
||||
# --- Test Class ---
|
||||
@pytest.mark.asyncio
|
||||
class TestDeepCrawlEndpoints:
|
||||
|
||||
@pytest_asyncio.fixture(autouse=True)
|
||||
async def check_health_before_tests(self, async_client: httpx.AsyncClient):
|
||||
"""Fixture to ensure server is healthy before each test in the class."""
|
||||
await check_server_health(async_client)
|
||||
|
||||
# 1. Basic Deep Crawl
|
||||
async def test_deep_crawl_basic_bfs(self, async_client: httpx.AsyncClient):
|
||||
"""Test BFS deep crawl with limited depth and pages."""
|
||||
max_depth = 1
|
||||
max_pages = 3 # start_url + 2 more
|
||||
payload = {
|
||||
"urls": [DEEP_CRAWL_BASE_URL],
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"stream": False,
|
||||
"cache_mode": "BYPASS", # Use string value for CacheMode
|
||||
"deep_crawl_strategy": {
|
||||
"type": "BFSDeepCrawlStrategy",
|
||||
"params": {
|
||||
"max_depth": max_depth,
|
||||
"max_pages": max_pages,
|
||||
# Minimal filters for basic test
|
||||
"filter_chain": {
|
||||
"type": "FilterChain",
|
||||
"params": {
|
||||
"filters": [
|
||||
{
|
||||
"type": "DomainFilter",
|
||||
"params": {"allowed_domains": [DEEP_CRAWL_DOMAIN]}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
assert data["success"] is True
|
||||
assert isinstance(data["results"], list)
|
||||
assert len(data["results"]) > 1 # Should be more than just the start URL
|
||||
assert len(data["results"]) <= max_pages # Respect max_pages
|
||||
|
||||
found_depth_0 = False
|
||||
found_depth_1 = False
|
||||
for result in data["results"]:
|
||||
await assert_crawl_result_structure(result)
|
||||
assert result["success"] is True
|
||||
assert DEEP_CRAWL_DOMAIN in result["url"]
|
||||
depth = result["metadata"]["depth"]
|
||||
assert depth <= max_depth
|
||||
if depth == 0: found_depth_0 = True
|
||||
if depth == 1: found_depth_1 = True
|
||||
|
||||
assert found_depth_0
|
||||
assert found_depth_1
|
||||
|
||||
# 2. Deep Crawl with Filtering
|
||||
async def test_deep_crawl_with_filters(self, async_client: httpx.AsyncClient):
|
||||
"""Test BFS deep crawl with content type and domain filters."""
|
||||
max_depth = 1
|
||||
max_pages = 5
|
||||
payload = {
|
||||
"urls": [DEEP_CRAWL_BASE_URL],
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"stream": False,
|
||||
"cache_mode": "BYPASS",
|
||||
"deep_crawl_strategy": {
|
||||
"type": "BFSDeepCrawlStrategy",
|
||||
"params": {
|
||||
"max_depth": max_depth,
|
||||
"max_pages": max_pages,
|
||||
"filter_chain": {
|
||||
"type": "FilterChain",
|
||||
"params": {
|
||||
"filters": [
|
||||
{
|
||||
"type": "DomainFilter",
|
||||
"params": {"allowed_domains": [DEEP_CRAWL_DOMAIN]}
|
||||
},
|
||||
{
|
||||
"type": "ContentTypeFilter",
|
||||
"params": {"allowed_types": ["text/html"]}
|
||||
},
|
||||
# Example: Exclude specific paths using regex
|
||||
{
|
||||
"type": "URLPatternFilter",
|
||||
"params": {
|
||||
"patterns": ["*/category-3/*"], # Block category 3
|
||||
"reverse": True # Block if match
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
assert data["success"] is True
|
||||
assert len(data["results"]) > 0
|
||||
assert len(data["results"]) <= max_pages
|
||||
|
||||
for result in data["results"]:
|
||||
await assert_crawl_result_structure(result)
|
||||
assert result["success"] is True
|
||||
assert DEEP_CRAWL_DOMAIN in result["url"]
|
||||
assert "category-3" not in result["url"] # Check if filter worked
|
||||
assert result["metadata"]["depth"] <= max_depth
|
||||
|
||||
# 3. Deep Crawl with Scoring
|
||||
async def test_deep_crawl_with_scoring(self, async_client: httpx.AsyncClient):
|
||||
"""Test BFS deep crawl with URL scoring."""
|
||||
max_depth = 1
|
||||
max_pages = 4
|
||||
payload = {
|
||||
"urls": [DEEP_CRAWL_BASE_URL],
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"stream": False,
|
||||
"cache_mode": "BYPASS",
|
||||
"deep_crawl_strategy": {
|
||||
"type": "BFSDeepCrawlStrategy",
|
||||
"params": {
|
||||
"max_depth": max_depth,
|
||||
"max_pages": max_pages,
|
||||
"filter_chain": { # Keep basic domain filter
|
||||
"type": "FilterChain",
|
||||
"params": { "filters": [{"type": "DomainFilter", "params": {"allowed_domains": [DEEP_CRAWL_DOMAIN]}}]}
|
||||
},
|
||||
"url_scorer": { # Add scorer
|
||||
"type": "CompositeScorer",
|
||||
"params": {
|
||||
"scorers": [
|
||||
{ # Favor pages with 'product' in the URL
|
||||
"type": "KeywordRelevanceScorer",
|
||||
"params": {"keywords": ["product"], "weight": 1.0}
|
||||
},
|
||||
{ # Penalize deep paths slightly
|
||||
"type": "PathDepthScorer",
|
||||
"params": {"optimal_depth": 2, "weight": -0.2}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
# Set a threshold if needed: "score_threshold": 0.1
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
assert data["success"] is True
|
||||
assert len(data["results"]) > 0
|
||||
assert len(data["results"]) <= max_pages
|
||||
|
||||
# Check if results seem biased towards products (harder to assert strictly without knowing exact scores)
|
||||
product_urls_found = any("product_" in result["url"] for result in data["results"] if result["metadata"]["depth"] > 0)
|
||||
print(f"Product URLs found among depth > 0 results: {product_urls_found}")
|
||||
# We expect scoring to prioritize product pages if available within limits
|
||||
# assert product_urls_found # This might be too strict depending on site structure and limits
|
||||
|
||||
for result in data["results"]:
|
||||
await assert_crawl_result_structure(result)
|
||||
assert result["success"] is True
|
||||
assert result["metadata"]["depth"] <= max_depth
|
||||
|
||||
# 4. Deep Crawl with CSS Extraction
|
||||
async def test_deep_crawl_with_css_extraction(self, async_client: httpx.AsyncClient):
|
||||
"""Test BFS deep crawl combined with JsonCssExtractionStrategy."""
|
||||
max_depth = 6 # Go deep enough to reach product pages
|
||||
max_pages = 20
|
||||
# Schema to extract product details
|
||||
product_schema = {
|
||||
"name": "ProductDetails",
|
||||
"baseSelector": "div.container", # Base for product page
|
||||
"fields": [
|
||||
{"name": "product_title", "selector": "h1", "type": "text"},
|
||||
{"name": "price", "selector": ".product-price", "type": "text"},
|
||||
{"name": "description", "selector": ".product-description p", "type": "text"},
|
||||
{"name": "specs", "selector": ".product-specs li", "type": "list", "fields":[
|
||||
{"name": "spec_name", "selector": ".spec-name", "type": "text"},
|
||||
{"name": "spec_value", "selector": ".spec-value", "type": "text"}
|
||||
]}
|
||||
]
|
||||
}
|
||||
payload = {
|
||||
"urls": [DEEP_CRAWL_BASE_URL],
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"stream": False,
|
||||
"cache_mode": "BYPASS",
|
||||
"extraction_strategy": { # Apply extraction to ALL crawled pages
|
||||
"type": "JsonCssExtractionStrategy",
|
||||
"params": {"schema": {"type": "dict", "value": product_schema}}
|
||||
},
|
||||
"deep_crawl_strategy": {
|
||||
"type": "BFSDeepCrawlStrategy",
|
||||
"params": {
|
||||
"max_depth": max_depth,
|
||||
"max_pages": max_pages,
|
||||
"filter_chain": { # Only crawl HTML on our domain
|
||||
"type": "FilterChain",
|
||||
"params": {
|
||||
"filters": [
|
||||
{"type": "DomainFilter", "params": {"allowed_domains": [DEEP_CRAWL_DOMAIN]}},
|
||||
{"type": "ContentTypeFilter", "params": {"allowed_types": ["text/html"]}}
|
||||
]
|
||||
}
|
||||
}
|
||||
# Optional: Add scoring to prioritize product pages for extraction
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
assert data["success"] is True
|
||||
assert len(data["results"]) > 0
|
||||
# assert len(data["results"]) <= max_pages
|
||||
|
||||
found_extracted_product = False
|
||||
for result in data["results"]:
|
||||
await assert_crawl_result_structure(result)
|
||||
assert result["success"] is True
|
||||
assert "extracted_content" in result
|
||||
if "product_" in result["url"]: # Check product pages specifically
|
||||
assert result["extracted_content"] is not None
|
||||
try:
|
||||
extracted = json.loads(result["extracted_content"])
|
||||
# Schema returns list even if one base match
|
||||
assert isinstance(extracted, list)
|
||||
if extracted:
|
||||
item = extracted[0]
|
||||
assert "product_title" in item and item["product_title"]
|
||||
assert "price" in item and item["price"]
|
||||
# Specs might be empty list if not found
|
||||
assert "specs" in item and isinstance(item["specs"], list)
|
||||
found_extracted_product = True
|
||||
print(f"Extracted product: {item.get('product_title')}")
|
||||
except (json.JSONDecodeError, AssertionError, IndexError) as e:
|
||||
pytest.fail(f"Extraction validation failed for {result['url']}: {e}\nContent: {result['extracted_content']}")
|
||||
# else:
|
||||
# # Non-product pages might have None or empty list depending on schema match
|
||||
# assert result["extracted_content"] is None or json.loads(result["extracted_content"]) == []
|
||||
|
||||
assert found_extracted_product, "Did not find any pages where product data was successfully extracted."
|
||||
|
||||
# 5. Deep Crawl with LLM Extraction (Requires Server LLM Setup)
|
||||
async def test_deep_crawl_with_llm_extraction(self, async_client: httpx.AsyncClient):
|
||||
"""Test BFS deep crawl combined with LLMExtractionStrategy."""
|
||||
max_depth = 1 # Limit depth to keep LLM calls manageable
|
||||
max_pages = 3
|
||||
payload = {
|
||||
"urls": [DEEP_CRAWL_BASE_URL],
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"stream": False,
|
||||
"cache_mode": "BYPASS",
|
||||
"extraction_strategy": { # Apply LLM extraction to crawled pages
|
||||
"type": "LLMExtractionStrategy",
|
||||
"params": {
|
||||
"instruction": "Extract the main H1 title and the text content of the first paragraph.",
|
||||
"llm_config": { # Example override, rely on server default if possible
|
||||
"type": "LLMConfig",
|
||||
"params": {"provider": "openai/gpt-4.1-mini"} # Use a cheaper model for testing
|
||||
},
|
||||
"schema": { # Expected JSON output
|
||||
"type": "dict",
|
||||
"value": {
|
||||
"title": "PageContent", "type": "object",
|
||||
"properties": {
|
||||
"h1_title": {"type": "string"},
|
||||
"first_paragraph": {"type": "string"}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"deep_crawl_strategy": {
|
||||
"type": "BFSDeepCrawlStrategy",
|
||||
"params": {
|
||||
"max_depth": max_depth,
|
||||
"max_pages": max_pages,
|
||||
"filter_chain": {
|
||||
"type": "FilterChain",
|
||||
"params": {
|
||||
"filters": [
|
||||
{"type": "DomainFilter", "params": {"allowed_domains": [DEEP_CRAWL_DOMAIN]}},
|
||||
{"type": "ContentTypeFilter", "params": {"allowed_types": ["text/html"]}}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try:
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
pytest.fail(f"Deep Crawl + LLM extraction request failed: {e}. Response: {e.response.text}. Check server logs and LLM API key setup.")
|
||||
except httpx.RequestError as e:
|
||||
pytest.fail(f"Deep Crawl + LLM extraction request failed: {e}.")
|
||||
|
||||
|
||||
assert data["success"] is True
|
||||
assert len(data["results"]) > 0
|
||||
assert len(data["results"]) <= max_pages
|
||||
|
||||
found_llm_extraction = False
|
||||
for result in data["results"]:
|
||||
await assert_crawl_result_structure(result)
|
||||
assert result["success"] is True
|
||||
assert "extracted_content" in result
|
||||
assert result["extracted_content"] is not None
|
||||
try:
|
||||
extracted = json.loads(result["extracted_content"])
|
||||
if isinstance(extracted, list): extracted = extracted[0] # Handle list output
|
||||
assert isinstance(extracted, dict)
|
||||
assert "h1_title" in extracted # Check keys based on schema
|
||||
assert "first_paragraph" in extracted
|
||||
found_llm_extraction = True
|
||||
print(f"LLM extracted from {result['url']}: Title='{extracted.get('h1_title')}'")
|
||||
except (json.JSONDecodeError, AssertionError, IndexError, TypeError) as e:
|
||||
pytest.fail(f"LLM extraction validation failed for {result['url']}: {e}\nContent: {result['extracted_content']}")
|
||||
|
||||
assert found_llm_extraction, "LLM extraction did not yield expected data on any crawled page."
|
||||
|
||||
|
||||
# 6. Deep Crawl with SSL Certificate Fetching
|
||||
async def test_deep_crawl_with_ssl(self, async_client: httpx.AsyncClient):
|
||||
"""Test BFS deep crawl with fetch_ssl_certificate enabled."""
|
||||
max_depth = 0 # Only fetch for start URL to keep test fast
|
||||
max_pages = 1
|
||||
payload = {
|
||||
"urls": [DEEP_CRAWL_BASE_URL],
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"stream": False,
|
||||
"cache_mode": "BYPASS",
|
||||
"fetch_ssl_certificate": True, # <-- Enable SSL fetching
|
||||
"deep_crawl_strategy": {
|
||||
"type": "BFSDeepCrawlStrategy",
|
||||
"params": {
|
||||
"max_depth": max_depth,
|
||||
"max_pages": max_pages,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
|
||||
assert data["success"] is True
|
||||
assert len(data["results"]) == 1
|
||||
result = data["results"][0]
|
||||
|
||||
await assert_crawl_result_structure(result, check_ssl=True) # <-- Tell helper to check SSL field
|
||||
assert result["success"] is True
|
||||
# Check if SSL info was actually retrieved
|
||||
if result["ssl_certificate"]:
|
||||
# Assert directly using dictionary keys
|
||||
assert isinstance(result["ssl_certificate"], dict) # Verify it's a dict
|
||||
assert "issuer" in result["ssl_certificate"]
|
||||
assert "subject" in result["ssl_certificate"]
|
||||
# --- MODIFIED ASSERTIONS ---
|
||||
assert "not_before" in result["ssl_certificate"] # Check for the actual key
|
||||
assert "not_after" in result["ssl_certificate"] # Check for the actual key
|
||||
# --- END MODIFICATIONS ---
|
||||
assert "fingerprint" in result["ssl_certificate"] # Check another key
|
||||
|
||||
# This print statement using .get() already works correctly with dictionaries
|
||||
print(f"SSL Issuer Org: {result['ssl_certificate'].get('issuer', {}).get('O', 'N/A')}")
|
||||
print(f"SSL Valid From: {result['ssl_certificate'].get('not_before', 'N/A')}")
|
||||
else:
|
||||
# This part remains the same
|
||||
print("SSL Certificate was null in the result.")
|
||||
|
||||
|
||||
# 7. Deep Crawl with Proxy Rotation (Requires PROXIES env var)
|
||||
async def test_deep_crawl_with_proxies(self, async_client: httpx.AsyncClient):
|
||||
"""Test BFS deep crawl using proxy rotation."""
|
||||
proxies = load_proxies_from_env()
|
||||
if not proxies:
|
||||
pytest.skip("Skipping proxy test: PROXIES environment variable not set or empty.")
|
||||
|
||||
print(f"\nTesting with {len(proxies)} proxies loaded from environment.")
|
||||
|
||||
max_depth = 1
|
||||
max_pages = 3
|
||||
payload = {
|
||||
"urls": [DEEP_CRAWL_BASE_URL], # Use the dummy site
|
||||
# Use a BrowserConfig that *might* pick up proxy if set, but rely on CrawlerRunConfig
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"stream": False,
|
||||
"cache_mode": "BYPASS",
|
||||
"proxy_rotation_strategy": { # <-- Define the strategy
|
||||
"type": "RoundRobinProxyStrategy",
|
||||
"params": {
|
||||
# Convert ProxyConfig dicts back to the serialized format expected by server
|
||||
"proxies": [{"type": "ProxyConfig", "params": p} for p in proxies]
|
||||
}
|
||||
},
|
||||
"deep_crawl_strategy": {
|
||||
"type": "BFSDeepCrawlStrategy",
|
||||
"params": {
|
||||
"max_depth": max_depth,
|
||||
"max_pages": max_pages,
|
||||
"filter_chain": {
|
||||
"type": "FilterChain",
|
||||
"params": { "filters": [{"type": "DomainFilter", "params": {"allowed_domains": [DEEP_CRAWL_DOMAIN]}}]}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
try:
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
# Proxies often cause connection errors, catch them
|
||||
pytest.fail(f"Proxy deep crawl failed: {e}. Response: {e.response.text}. Are proxies valid and accessible by the server?")
|
||||
except httpx.RequestError as e:
|
||||
pytest.fail(f"Proxy deep crawl request failed: {e}. Are proxies valid and accessible?")
|
||||
|
||||
assert data["success"] is True
|
||||
assert len(data["results"]) > 0
|
||||
assert len(data["results"]) <= max_pages
|
||||
# Primary assertion is that the crawl succeeded *with* proxy config
|
||||
print(f"Proxy deep crawl completed successfully for {len(data['results'])} pages.")
|
||||
|
||||
# Verifying specific proxy usage requires server logs or custom headers/responses
|
||||
|
||||
|
||||
# --- Main Execution Block (for running script directly) ---
|
||||
if __name__ == "__main__":
|
||||
pytest_args = ["-v", "-s", __file__]
|
||||
# Example: Run only proxy test
|
||||
# pytest_args.append("-k test_deep_crawl_with_proxies")
|
||||
print(f"Running pytest with args: {pytest_args}")
|
||||
exit_code = pytest.main(pytest_args)
|
||||
print(f"Pytest finished with exit code: {exit_code}")
|
||||
@@ -0,0 +1,255 @@
|
||||
import inspect
|
||||
from typing import Any, Dict
|
||||
from enum import Enum
|
||||
|
||||
from crawl4ai import LLMConfig
|
||||
|
||||
def to_serializable_dict(obj: Any) -> Dict:
|
||||
"""
|
||||
Recursively convert an object to a serializable dictionary using {type, params} structure
|
||||
for complex objects.
|
||||
"""
|
||||
if obj is None:
|
||||
return None
|
||||
|
||||
# Handle basic types
|
||||
if isinstance(obj, (str, int, float, bool)):
|
||||
return obj
|
||||
|
||||
# Handle Enum
|
||||
if isinstance(obj, Enum):
|
||||
return {
|
||||
"type": obj.__class__.__name__,
|
||||
"params": obj.value
|
||||
}
|
||||
|
||||
# Handle datetime objects
|
||||
if hasattr(obj, 'isoformat'):
|
||||
return obj.isoformat()
|
||||
|
||||
# Handle lists, tuples, and sets
|
||||
if isinstance(obj, (list, tuple, set)):
|
||||
return [to_serializable_dict(item) for item in obj]
|
||||
|
||||
# Handle dictionaries - preserve them as-is
|
||||
if isinstance(obj, dict):
|
||||
return {
|
||||
"type": "dict", # Mark as plain dictionary
|
||||
"value": {str(k): to_serializable_dict(v) for k, v in obj.items()}
|
||||
}
|
||||
|
||||
# Handle class instances
|
||||
if hasattr(obj, '__class__'):
|
||||
# Get constructor signature
|
||||
sig = inspect.signature(obj.__class__.__init__)
|
||||
params = sig.parameters
|
||||
|
||||
# Get current values
|
||||
current_values = {}
|
||||
for name, param in params.items():
|
||||
if name == 'self':
|
||||
continue
|
||||
|
||||
value = getattr(obj, name, param.default)
|
||||
|
||||
# Only include if different from default, considering empty values
|
||||
if not (is_empty_value(value) and is_empty_value(param.default)):
|
||||
if value != param.default:
|
||||
current_values[name] = to_serializable_dict(value)
|
||||
|
||||
return {
|
||||
"type": obj.__class__.__name__,
|
||||
"params": current_values
|
||||
}
|
||||
|
||||
return str(obj)
|
||||
|
||||
def from_serializable_dict(data: Any) -> Any:
|
||||
"""
|
||||
Recursively convert a serializable dictionary back to an object instance.
|
||||
"""
|
||||
if data is None:
|
||||
return None
|
||||
|
||||
# Handle basic types
|
||||
if isinstance(data, (str, int, float, bool)):
|
||||
return data
|
||||
|
||||
# Handle typed data
|
||||
if isinstance(data, dict) and "type" in data:
|
||||
# Handle plain dictionaries
|
||||
if data["type"] == "dict":
|
||||
return {k: from_serializable_dict(v) for k, v in data["value"].items()}
|
||||
|
||||
# Import from crawl4ai for class instances
|
||||
import crawl4ai
|
||||
cls = getattr(crawl4ai, data["type"])
|
||||
|
||||
# Handle Enum
|
||||
if issubclass(cls, Enum):
|
||||
return cls(data["params"])
|
||||
|
||||
# Handle class instances
|
||||
constructor_args = {
|
||||
k: from_serializable_dict(v) for k, v in data["params"].items()
|
||||
}
|
||||
return cls(**constructor_args)
|
||||
|
||||
# Handle lists
|
||||
if isinstance(data, list):
|
||||
return [from_serializable_dict(item) for item in data]
|
||||
|
||||
# Handle raw dictionaries (legacy support)
|
||||
if isinstance(data, dict):
|
||||
return {k: from_serializable_dict(v) for k, v in data.items()}
|
||||
|
||||
return data
|
||||
|
||||
def is_empty_value(value: Any) -> bool:
|
||||
"""Check if a value is effectively empty/null."""
|
||||
if value is None:
|
||||
return True
|
||||
if isinstance(value, (list, tuple, set, dict, str)) and len(value) == 0:
|
||||
return True
|
||||
return False
|
||||
|
||||
# if __name__ == "__main__":
|
||||
# from crawl4ai import (
|
||||
# CrawlerRunConfig, CacheMode, DefaultMarkdownGenerator,
|
||||
# PruningContentFilter, BM25ContentFilter, LLMContentFilter,
|
||||
# JsonCssExtractionStrategy, CosineStrategy, RegexChunking,
|
||||
# WebScrapingStrategy, LXMLWebScrapingStrategy
|
||||
# )
|
||||
|
||||
# # Test Case 1: BM25 content filtering through markdown generator
|
||||
# config1 = CrawlerRunConfig(
|
||||
# cache_mode=CacheMode.BYPASS,
|
||||
# markdown_generator=DefaultMarkdownGenerator(
|
||||
# content_filter=BM25ContentFilter(
|
||||
# user_query="technology articles",
|
||||
# bm25_threshold=1.2,
|
||||
# language="english"
|
||||
# )
|
||||
# ),
|
||||
# chunking_strategy=RegexChunking(patterns=[r"\n\n", r"\.\s+"]),
|
||||
# excluded_tags=["nav", "footer", "aside"],
|
||||
# remove_overlay_elements=True
|
||||
# )
|
||||
|
||||
# # Serialize
|
||||
# serialized = to_serializable_dict(config1)
|
||||
# print("\nSerialized Config:")
|
||||
# print(serialized)
|
||||
|
||||
# # Example output structure would now look like:
|
||||
# """
|
||||
# {
|
||||
# "type": "CrawlerRunConfig",
|
||||
# "params": {
|
||||
# "cache_mode": {
|
||||
# "type": "CacheMode",
|
||||
# "params": "bypass"
|
||||
# },
|
||||
# "markdown_generator": {
|
||||
# "type": "DefaultMarkdownGenerator",
|
||||
# "params": {
|
||||
# "content_filter": {
|
||||
# "type": "BM25ContentFilter",
|
||||
# "params": {
|
||||
# "user_query": "technology articles",
|
||||
# "bm25_threshold": 1.2,
|
||||
# "language": "english"
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
# """
|
||||
|
||||
# # Deserialize
|
||||
# deserialized = from_serializable_dict(serialized)
|
||||
# print("\nDeserialized Config:")
|
||||
# print(to_serializable_dict(deserialized))
|
||||
|
||||
# # Verify they match
|
||||
# assert to_serializable_dict(config1) == to_serializable_dict(deserialized)
|
||||
# print("\nVerification passed: Configuration matches after serialization/deserialization!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
from crawl4ai import (
|
||||
CrawlerRunConfig, CacheMode, DefaultMarkdownGenerator,
|
||||
PruningContentFilter, BM25ContentFilter, LLMContentFilter,
|
||||
JsonCssExtractionStrategy, RegexChunking,
|
||||
WebScrapingStrategy, LXMLWebScrapingStrategy
|
||||
)
|
||||
|
||||
# Test Case 1: BM25 content filtering through markdown generator
|
||||
config1 = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
markdown_generator=DefaultMarkdownGenerator(
|
||||
content_filter=BM25ContentFilter(
|
||||
user_query="technology articles",
|
||||
bm25_threshold=1.2,
|
||||
language="english"
|
||||
)
|
||||
),
|
||||
chunking_strategy=RegexChunking(patterns=[r"\n\n", r"\.\s+"]),
|
||||
excluded_tags=["nav", "footer", "aside"],
|
||||
remove_overlay_elements=True
|
||||
)
|
||||
|
||||
# Test Case 2: LLM-based extraction with pruning filter
|
||||
schema = {
|
||||
"baseSelector": "article.post",
|
||||
"fields": [
|
||||
{"name": "title", "selector": "h1", "type": "text"},
|
||||
{"name": "content", "selector": ".content", "type": "html"}
|
||||
]
|
||||
}
|
||||
config2 = CrawlerRunConfig(
|
||||
extraction_strategy=JsonCssExtractionStrategy(schema=schema),
|
||||
markdown_generator=DefaultMarkdownGenerator(
|
||||
content_filter=PruningContentFilter(
|
||||
threshold=0.48,
|
||||
threshold_type="fixed",
|
||||
min_word_threshold=0
|
||||
),
|
||||
options={"ignore_links": True}
|
||||
),
|
||||
scraping_strategy=LXMLWebScrapingStrategy()
|
||||
)
|
||||
|
||||
# Test Case 3:LLM content filter
|
||||
config3 = CrawlerRunConfig(
|
||||
markdown_generator=DefaultMarkdownGenerator(
|
||||
content_filter=LLMContentFilter(
|
||||
llm_config = LLMConfig(provider="openai/gpt-4"),
|
||||
instruction="Extract key technical concepts",
|
||||
chunk_token_threshold=2000,
|
||||
overlap_rate=0.1
|
||||
),
|
||||
options={"ignore_images": True}
|
||||
),
|
||||
scraping_strategy=WebScrapingStrategy()
|
||||
)
|
||||
|
||||
# Test all configurations
|
||||
test_configs = [config1, config2, config3]
|
||||
|
||||
for i, config in enumerate(test_configs, 1):
|
||||
print(f"\nTesting Configuration {i}:")
|
||||
|
||||
# Serialize
|
||||
serialized = to_serializable_dict(config)
|
||||
print(f"\nSerialized Config {i}:")
|
||||
print(serialized)
|
||||
|
||||
# Deserialize
|
||||
deserialized = from_serializable_dict(serialized)
|
||||
print(f"\nDeserialized Config {i}:")
|
||||
print(to_serializable_dict(deserialized)) # Convert back to dict for comparison
|
||||
|
||||
# Verify they match
|
||||
assert to_serializable_dict(config) == to_serializable_dict(deserialized)
|
||||
print(f"\nVerification passed: Configuration {i} matches after serialization/deserialization!")
|
||||
@@ -0,0 +1,146 @@
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Optional
|
||||
from urllib.parse import quote
|
||||
|
||||
async def test_endpoint(
|
||||
endpoint: str,
|
||||
url: str,
|
||||
params: Optional[dict] = None,
|
||||
expected_status: int = 200
|
||||
) -> None:
|
||||
"""Test an endpoint and print results"""
|
||||
import aiohttp
|
||||
|
||||
params = params or {}
|
||||
param_str = "&".join(f"{k}={v}" for k, v in params.items())
|
||||
full_url = f"http://localhost:8000/{endpoint}/{quote(url)}"
|
||||
if param_str:
|
||||
full_url += f"?{param_str}"
|
||||
|
||||
print(f"\nTesting: {full_url}")
|
||||
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(full_url) as response:
|
||||
status = response.status
|
||||
try:
|
||||
data = await response.json()
|
||||
except:
|
||||
data = await response.text()
|
||||
|
||||
print(f"Status: {status} (Expected: {expected_status})")
|
||||
if isinstance(data, dict):
|
||||
print(f"Response: {json.dumps(data, indent=2)}")
|
||||
else:
|
||||
print(f"Response: {data[:500]}...") # First 500 chars
|
||||
assert status == expected_status
|
||||
return data
|
||||
except Exception as e:
|
||||
print(f"Error: {str(e)}")
|
||||
return None
|
||||
|
||||
async def test_llm_task_completion(task_id: str) -> None:
|
||||
"""Poll task until completion"""
|
||||
for _ in range(10): # Try 10 times
|
||||
result = await test_endpoint("llm", task_id)
|
||||
if result and result.get("status") in ["completed", "failed"]:
|
||||
return result
|
||||
print("Task still processing, waiting 5 seconds...")
|
||||
await asyncio.sleep(5)
|
||||
print("Task timed out")
|
||||
|
||||
async def run_tests():
|
||||
print("Starting API Tests...")
|
||||
|
||||
# Test URLs
|
||||
urls = [
|
||||
"example.com",
|
||||
"https://www.python.org",
|
||||
"https://news.ycombinator.com/news",
|
||||
"https://github.com/trending"
|
||||
]
|
||||
|
||||
print("\n=== Testing Markdown Endpoint ===")
|
||||
for url in[] : #urls:
|
||||
# Test different filter types
|
||||
for filter_type in ["raw", "fit", "bm25", "llm"]:
|
||||
params = {"f": filter_type}
|
||||
if filter_type in ["bm25", "llm"]:
|
||||
params["q"] = "extract main content"
|
||||
|
||||
# Test with and without cache
|
||||
for cache in ["0", "1"]:
|
||||
params["c"] = cache
|
||||
await test_endpoint("md", url, params)
|
||||
await asyncio.sleep(1) # Be nice to the server
|
||||
|
||||
print("\n=== Testing LLM Endpoint ===")
|
||||
for url in []: # urls:
|
||||
# Test basic extraction
|
||||
result = await test_endpoint(
|
||||
"llm",
|
||||
url,
|
||||
{"q": "Extract title and main content"}
|
||||
)
|
||||
if result and "task_id" in result:
|
||||
print("\nChecking task completion...")
|
||||
await test_llm_task_completion(result["task_id"])
|
||||
|
||||
# Test with schema
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"content": {"type": "string"},
|
||||
"links": {"type": "array", "items": {"type": "string"}}
|
||||
}
|
||||
}
|
||||
result = await test_endpoint(
|
||||
"llm",
|
||||
url,
|
||||
{
|
||||
"q": "Extract content with links",
|
||||
"s": json.dumps(schema),
|
||||
"c": "1" # Test with cache
|
||||
}
|
||||
)
|
||||
if result and "task_id" in result:
|
||||
print("\nChecking schema task completion...")
|
||||
await test_llm_task_completion(result["task_id"])
|
||||
|
||||
await asyncio.sleep(2) # Be nice to the server
|
||||
|
||||
print("\n=== Testing Error Cases ===")
|
||||
# Test invalid URL
|
||||
await test_endpoint(
|
||||
"md",
|
||||
"not_a_real_url",
|
||||
expected_status=500
|
||||
)
|
||||
|
||||
# Test invalid filter type
|
||||
await test_endpoint(
|
||||
"md",
|
||||
"example.com",
|
||||
{"f": "invalid"},
|
||||
expected_status=422
|
||||
)
|
||||
|
||||
# Test LLM without query
|
||||
await test_endpoint(
|
||||
"llm",
|
||||
"example.com"
|
||||
)
|
||||
|
||||
# Test invalid task ID
|
||||
await test_endpoint(
|
||||
"llm",
|
||||
"llm_invalid_task",
|
||||
expected_status=404
|
||||
)
|
||||
|
||||
print("\nAll tests completed!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_tests())
|
||||
@@ -0,0 +1,890 @@
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import httpx
|
||||
import json
|
||||
import asyncio
|
||||
import os
|
||||
from typing import List, Dict, Any, AsyncGenerator
|
||||
|
||||
from dotenv import load_dotenv
|
||||
load_dotenv()
|
||||
|
||||
|
||||
# Optional: Import crawl4ai classes directly for reference/easier payload creation aid
|
||||
# You don't strictly NEED these imports for the tests to run against the server,
|
||||
# but they help in understanding the structure you are mimicking in JSON.
|
||||
from crawl4ai import (
|
||||
BrowserConfig,
|
||||
CrawlerRunConfig,
|
||||
CacheMode,
|
||||
DefaultMarkdownGenerator,
|
||||
PruningContentFilter,
|
||||
BM25ContentFilter,
|
||||
BFSDeepCrawlStrategy,
|
||||
FilterChain,
|
||||
ContentTypeFilter,
|
||||
DomainFilter,
|
||||
CompositeScorer,
|
||||
KeywordRelevanceScorer,
|
||||
PathDepthScorer,
|
||||
JsonCssExtractionStrategy,
|
||||
LLMExtractionStrategy,
|
||||
LLMConfig
|
||||
)
|
||||
|
||||
# --- Test Configuration ---
|
||||
# BASE_URL = os.getenv("CRAWL4AI_TEST_URL", "http://localhost:8020") # Make base URL configurable
|
||||
BASE_URL = os.getenv("CRAWL4AI_TEST_URL", "http://localhost:11235") # Make base URL configurable
|
||||
# Use a known simple HTML page for basic tests
|
||||
SIMPLE_HTML_URL = "https://httpbin.org/html"
|
||||
# Use a site suitable for scraping tests
|
||||
SCRAPE_TARGET_URL = "http://books.toscrape.com/"
|
||||
# Use a site with internal links for deep crawl tests
|
||||
DEEP_CRAWL_URL = "https://python.org"
|
||||
|
||||
# --- Pytest Fixtures ---
|
||||
|
||||
# Use the built-in event_loop fixture from pytest_asyncio
|
||||
# The custom implementation was causing issues with closing the loop
|
||||
|
||||
@pytest_asyncio.fixture(scope="function") # Changed to function scope to avoid event loop issues
|
||||
async def async_client() -> AsyncGenerator[httpx.AsyncClient, None]:
|
||||
"""Provides an async HTTP client"""
|
||||
client = httpx.AsyncClient(base_url=BASE_URL, timeout=120.0)
|
||||
yield client
|
||||
await client.aclose()
|
||||
|
||||
# --- Helper Functions ---
|
||||
|
||||
async def check_server_health(client: httpx.AsyncClient):
|
||||
"""Check if the server is healthy before running tests."""
|
||||
try:
|
||||
response = await client.get("/health")
|
||||
response.raise_for_status()
|
||||
print(f"\nServer healthy: {response.json()}")
|
||||
return True
|
||||
except (httpx.RequestError, httpx.HTTPStatusError) as e:
|
||||
pytest.fail(f"Server health check failed: {e}. Is the server running at {BASE_URL}?", pytrace=False)
|
||||
|
||||
async def assert_crawl_result_structure(result: Dict[str, Any]):
|
||||
"""Asserts the basic structure of a single crawl result."""
|
||||
assert isinstance(result, dict)
|
||||
assert "url" in result
|
||||
assert "success" in result
|
||||
assert "html" in result
|
||||
# Add more common checks if needed
|
||||
|
||||
async def process_streaming_response(response: httpx.Response) -> List[Dict[str, Any]]:
|
||||
"""Processes an NDJSON streaming response."""
|
||||
results = []
|
||||
completed = False
|
||||
async for line in response.aiter_lines():
|
||||
if line:
|
||||
try:
|
||||
data = json.loads(line)
|
||||
if data.get("status") == "completed":
|
||||
completed = True
|
||||
break # Stop processing after completion marker
|
||||
else:
|
||||
results.append(data)
|
||||
except json.JSONDecodeError:
|
||||
pytest.fail(f"Failed to decode JSON line: {line}")
|
||||
assert completed, "Streaming response did not end with a completion marker."
|
||||
return results
|
||||
|
||||
|
||||
# --- Test Class ---
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestCrawlEndpoints:
|
||||
|
||||
@pytest_asyncio.fixture(autouse=True)
|
||||
async def check_health_before_tests(self, async_client: httpx.AsyncClient):
|
||||
"""Fixture to ensure server is healthy before each test in the class."""
|
||||
await check_server_health(async_client)
|
||||
|
||||
# 1. Simple Requests (Primitives)
|
||||
async def test_simple_crawl_single_url(self, async_client: httpx.AsyncClient):
|
||||
"""Test /crawl with a single URL and simple config values."""
|
||||
payload = {
|
||||
"urls": [SIMPLE_HTML_URL],
|
||||
"browser_config": {
|
||||
"type": "BrowserConfig",
|
||||
"params": {
|
||||
"headless": True,
|
||||
}
|
||||
},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"stream": False, # Explicitly false for /crawl
|
||||
"screenshot": False,
|
||||
"cache_mode": CacheMode.BYPASS.value # Use enum value
|
||||
}
|
||||
}
|
||||
}
|
||||
try:
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
print(f"Response status: {response.status_code}")
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
print(f"Server error: {e}")
|
||||
print(f"Response content: {e.response.text}")
|
||||
raise
|
||||
|
||||
assert data["success"] is True
|
||||
assert isinstance(data["results"], list)
|
||||
assert len(data["results"]) == 1
|
||||
result = data["results"][0]
|
||||
await assert_crawl_result_structure(result)
|
||||
assert result["success"] is True
|
||||
assert result["url"] == SIMPLE_HTML_URL
|
||||
assert "<h1>Herman Melville - Moby-Dick</h1>" in result["html"]
|
||||
# We don't specify a markdown generator in this test, so don't make assumptions about markdown field
|
||||
# It might be null, missing, or populated depending on the server's default behavior
|
||||
async def test_crawl_with_stream_direct(self, async_client: httpx.AsyncClient):
|
||||
"""Test that /crawl endpoint handles stream=True directly without redirect."""
|
||||
payload = {
|
||||
"urls": [SIMPLE_HTML_URL],
|
||||
"browser_config": {
|
||||
"type": "BrowserConfig",
|
||||
"params": {
|
||||
"headless": True,
|
||||
}
|
||||
},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"stream": True, # Set stream to True for direct streaming
|
||||
"screenshot": False,
|
||||
"cache_mode": CacheMode.BYPASS.value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Send a request to the /crawl endpoint - should handle streaming directly
|
||||
async with async_client.stream("POST", "/crawl", json=payload) as response:
|
||||
assert response.status_code == 200
|
||||
assert response.headers["content-type"] == "application/x-ndjson"
|
||||
assert response.headers.get("x-stream-status") == "active"
|
||||
|
||||
results = await process_streaming_response(response)
|
||||
|
||||
assert len(results) == 1
|
||||
result = results[0]
|
||||
await assert_crawl_result_structure(result)
|
||||
assert result["success"] is True
|
||||
assert result["url"] == SIMPLE_HTML_URL
|
||||
assert "<h1>Herman Melville - Moby-Dick</h1>" in result["html"]
|
||||
async def test_simple_crawl_single_url_streaming(self, async_client: httpx.AsyncClient):
|
||||
"""Test /crawl/stream with a single URL and simple config values."""
|
||||
payload = {
|
||||
"urls": [SIMPLE_HTML_URL],
|
||||
"browser_config": {
|
||||
"type": "BrowserConfig",
|
||||
"params": {
|
||||
"headless": True,
|
||||
}
|
||||
},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"stream": True, # Must be true for /crawl/stream
|
||||
"screenshot": False,
|
||||
"cache_mode": CacheMode.BYPASS.value
|
||||
}
|
||||
}
|
||||
}
|
||||
async with async_client.stream("POST", "/crawl/stream", json=payload) as response:
|
||||
response.raise_for_status()
|
||||
results = await process_streaming_response(response)
|
||||
|
||||
assert len(results) == 1
|
||||
result = results[0]
|
||||
await assert_crawl_result_structure(result)
|
||||
assert result["success"] is True
|
||||
assert result["url"] == SIMPLE_HTML_URL
|
||||
assert "<h1>Herman Melville - Moby-Dick</h1>" in result["html"]
|
||||
|
||||
|
||||
# 2. Multi-URL and Dispatcher
|
||||
async def test_multi_url_crawl(self, async_client: httpx.AsyncClient):
|
||||
"""Test /crawl with multiple URLs, implicitly testing dispatcher."""
|
||||
urls = [SIMPLE_HTML_URL, "https://httpbin.org/links/10/0"]
|
||||
payload = {
|
||||
"urls": urls,
|
||||
"browser_config": {
|
||||
"type": "BrowserConfig",
|
||||
"params": {"headless": True}
|
||||
},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {"stream": False, "cache_mode": CacheMode.BYPASS.value}
|
||||
}
|
||||
}
|
||||
try:
|
||||
print(f"Sending deep crawl request to server...")
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
print(f"Response status: {response.status_code}")
|
||||
|
||||
if response.status_code >= 400:
|
||||
error_detail = response.json().get('detail', 'No detail provided')
|
||||
print(f"Error detail: {error_detail}")
|
||||
print(f"Full response: {response.text}")
|
||||
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
print(f"Server error status: {e.response.status_code}")
|
||||
print(f"Server error response: {e.response.text}")
|
||||
try:
|
||||
error_json = e.response.json()
|
||||
print(f"Parsed error: {error_json}")
|
||||
except:
|
||||
print("Could not parse error response as JSON")
|
||||
raise
|
||||
|
||||
assert data["success"] is True
|
||||
assert isinstance(data["results"], list)
|
||||
assert len(data["results"]) == len(urls)
|
||||
for result in data["results"]:
|
||||
await assert_crawl_result_structure(result)
|
||||
assert result["success"] is True
|
||||
assert result["url"] in urls
|
||||
|
||||
async def test_multi_url_crawl_streaming(self, async_client: httpx.AsyncClient):
|
||||
"""Test /crawl/stream with multiple URLs."""
|
||||
urls = [SIMPLE_HTML_URL, "https://httpbin.org/links/10/0"]
|
||||
payload = {
|
||||
"urls": urls,
|
||||
"browser_config": {
|
||||
"type": "BrowserConfig",
|
||||
"params": {"headless": True}
|
||||
},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {"stream": True, "cache_mode": CacheMode.BYPASS.value}
|
||||
}
|
||||
}
|
||||
async with async_client.stream("POST", "/crawl/stream", json=payload) as response:
|
||||
response.raise_for_status()
|
||||
results = await process_streaming_response(response)
|
||||
|
||||
assert len(results) == len(urls)
|
||||
processed_urls = set()
|
||||
for result in results:
|
||||
await assert_crawl_result_structure(result)
|
||||
assert result["success"] is True
|
||||
assert result["url"] in urls
|
||||
processed_urls.add(result["url"])
|
||||
assert processed_urls == set(urls) # Ensure all URLs were processed
|
||||
|
||||
|
||||
# 3. Class Values and Nested Classes (Markdown Generator)
|
||||
async def test_crawl_with_markdown_pruning_filter(self, async_client: httpx.AsyncClient):
|
||||
"""Test /crawl with MarkdownGenerator using PruningContentFilter."""
|
||||
payload = {
|
||||
"urls": [SIMPLE_HTML_URL],
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"cache_mode": CacheMode.ENABLED.value, # Test different cache mode
|
||||
"markdown_generator": {
|
||||
"type": "DefaultMarkdownGenerator",
|
||||
"params": {
|
||||
"content_filter": {
|
||||
"type": "PruningContentFilter",
|
||||
"params": {
|
||||
"threshold": 0.5, # Example param
|
||||
"threshold_type": "relative"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
try:
|
||||
print(f"Sending deep crawl request to server...")
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
print(f"Response status: {response.status_code}")
|
||||
|
||||
if response.status_code >= 400:
|
||||
error_detail = response.json().get('detail', 'No detail provided')
|
||||
print(f"Error detail: {error_detail}")
|
||||
print(f"Full response: {response.text}")
|
||||
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
print(f"Server error status: {e.response.status_code}")
|
||||
print(f"Server error response: {e.response.text}")
|
||||
try:
|
||||
error_json = e.response.json()
|
||||
print(f"Parsed error: {error_json}")
|
||||
except:
|
||||
print("Could not parse error response as JSON")
|
||||
raise
|
||||
|
||||
assert data["success"] is True
|
||||
assert len(data["results"]) == 1
|
||||
result = data["results"][0]
|
||||
await assert_crawl_result_structure(result)
|
||||
assert result["success"] is True
|
||||
assert "markdown" in result
|
||||
assert isinstance(result["markdown"], dict)
|
||||
assert "raw_markdown" in result["markdown"]
|
||||
assert "fit_markdown" in result["markdown"] # Pruning creates fit_markdown
|
||||
assert "Moby-Dick" in result["markdown"]["raw_markdown"]
|
||||
# Fit markdown content might be different/shorter due to pruning
|
||||
assert len(result["markdown"]["fit_markdown"]) <= len(result["markdown"]["raw_markdown"])
|
||||
|
||||
async def test_crawl_with_markdown_bm25_filter(self, async_client: httpx.AsyncClient):
|
||||
"""Test /crawl with MarkdownGenerator using BM25ContentFilter."""
|
||||
payload = {
|
||||
"urls": [SIMPLE_HTML_URL],
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"markdown_generator": {
|
||||
"type": "DefaultMarkdownGenerator",
|
||||
"params": {
|
||||
"content_filter": {
|
||||
"type": "BM25ContentFilter",
|
||||
"params": {
|
||||
"user_query": "Herman Melville", # Query for BM25
|
||||
"bm25_threshold": 0.1, # Lower threshold to increase matches
|
||||
"language": "english" # Valid parameters
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
try:
|
||||
print(f"Payload for BM25 test: {json.dumps(payload)}")
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
print(f"Response status: {response.status_code}")
|
||||
|
||||
if response.status_code >= 400:
|
||||
error_detail = response.json().get('detail', 'No detail provided')
|
||||
print(f"Error detail: {error_detail}")
|
||||
print(f"Full response: {response.text}")
|
||||
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
print(f"Server error status: {e.response.status_code}")
|
||||
print(f"Server error response: {e.response.text}")
|
||||
try:
|
||||
error_json = e.response.json()
|
||||
print(f"Parsed error: {error_json}")
|
||||
except:
|
||||
print("Could not parse error response as JSON")
|
||||
raise
|
||||
|
||||
assert data["success"] is True
|
||||
assert len(data["results"]) == 1
|
||||
result = data["results"][0]
|
||||
await assert_crawl_result_structure(result)
|
||||
assert result["success"] is True
|
||||
assert "markdown" in result
|
||||
assert isinstance(result["markdown"], dict)
|
||||
assert "raw_markdown" in result["markdown"]
|
||||
assert "fit_markdown" in result["markdown"] # BM25 creates fit_markdown
|
||||
|
||||
# Print values for debug
|
||||
print(f"Raw markdown length: {len(result['markdown']['raw_markdown'])}")
|
||||
print(f"Fit markdown length: {len(result['markdown']['fit_markdown'])}")
|
||||
|
||||
# Either fit_markdown has content (possibly including our query terms)
|
||||
# or it might be empty if no good BM25 matches were found
|
||||
# Don't assert specific content since it can be environment-dependent
|
||||
|
||||
|
||||
# 4. Deep Crawling
|
||||
async def test_deep_crawl(self, async_client: httpx.AsyncClient):
|
||||
"""Test /crawl with a deep crawl strategy."""
|
||||
payload = {
|
||||
"urls": [DEEP_CRAWL_URL], # Start URL
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"stream": False,
|
||||
"cache_mode": CacheMode.BYPASS.value,
|
||||
"deep_crawl_strategy": {
|
||||
"type": "BFSDeepCrawlStrategy",
|
||||
"params": {
|
||||
"max_depth": 1, # Limit depth for testing speed
|
||||
"max_pages": 5, # Limit pages to crawl
|
||||
"filter_chain": {
|
||||
"type": "FilterChain",
|
||||
"params": {
|
||||
"filters": [
|
||||
{
|
||||
"type": "ContentTypeFilter",
|
||||
"params": {"allowed_types": ["text/html"]}
|
||||
},
|
||||
{
|
||||
"type": "DomainFilter",
|
||||
"params": {"allowed_domains": ["python.org", "docs.python.org"]} # Include important subdomains
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"url_scorer": {
|
||||
"type": "CompositeScorer",
|
||||
"params": {
|
||||
"scorers": [
|
||||
{
|
||||
"type": "KeywordRelevanceScorer",
|
||||
"params": {"keywords": ["documentation", "tutorial"]}
|
||||
},
|
||||
{
|
||||
"type": "PathDepthScorer",
|
||||
"params": {"weight": 0.5, "optimal_depth": 2}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
try:
|
||||
print(f"Sending deep crawl request to server...")
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
print(f"Response status: {response.status_code}")
|
||||
|
||||
if response.status_code >= 400:
|
||||
error_detail = response.json().get('detail', 'No detail provided')
|
||||
print(f"Error detail: {error_detail}")
|
||||
print(f"Full response: {response.text}")
|
||||
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
print(f"Server error status: {e.response.status_code}")
|
||||
print(f"Server error response: {e.response.text}")
|
||||
try:
|
||||
error_json = e.response.json()
|
||||
print(f"Parsed error: {error_json}")
|
||||
except:
|
||||
print("Could not parse error response as JSON")
|
||||
raise
|
||||
|
||||
assert data["success"] is True
|
||||
assert isinstance(data["results"], list)
|
||||
# Expect more than 1 result due to deep crawl (start URL + crawled links)
|
||||
assert len(data["results"]) > 1
|
||||
assert len(data["results"]) <= 6 # Start URL + max_links=5
|
||||
|
||||
start_url_found = False
|
||||
crawled_urls_found = False
|
||||
for result in data["results"]:
|
||||
await assert_crawl_result_structure(result)
|
||||
assert result["success"] is True
|
||||
|
||||
# Print URL for debugging
|
||||
print(f"Crawled URL: {result['url']}")
|
||||
|
||||
# Allow URLs that contain python.org (including subdomains like docs.python.org)
|
||||
assert "python.org" in result["url"]
|
||||
if result["url"] == DEEP_CRAWL_URL:
|
||||
start_url_found = True
|
||||
else:
|
||||
crawled_urls_found = True
|
||||
|
||||
assert start_url_found
|
||||
assert crawled_urls_found
|
||||
|
||||
|
||||
# 5. Extraction without LLM (JSON/CSS)
|
||||
async def test_json_css_extraction(self, async_client: httpx.AsyncClient):
|
||||
"""Test /crawl with JsonCssExtractionStrategy."""
|
||||
payload = {
|
||||
"urls": [SCRAPE_TARGET_URL],
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"cache_mode": CacheMode.BYPASS.value,
|
||||
"extraction_strategy": {
|
||||
"type": "JsonCssExtractionStrategy",
|
||||
"params": {
|
||||
"schema": {
|
||||
"type": "dict", # IMPORTANT: Wrap schema dict with type/value structure
|
||||
"value": {
|
||||
"name": "BookList",
|
||||
"baseSelector": "ol.row li.col-xs-6", # Select each book item
|
||||
"fields": [
|
||||
{"name": "title", "selector": "article.product_pod h3 a", "type": "attribute", "attribute": "title"},
|
||||
{"name": "price", "selector": "article.product_pod .price_color", "type": "text"},
|
||||
{"name": "rating", "selector": "article.product_pod p.star-rating", "type": "attribute", "attribute": "class"}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
try:
|
||||
print(f"Sending deep crawl request to server...")
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
print(f"Response status: {response.status_code}")
|
||||
|
||||
if response.status_code >= 400:
|
||||
error_detail = response.json().get('detail', 'No detail provided')
|
||||
print(f"Error detail: {error_detail}")
|
||||
print(f"Full response: {response.text}")
|
||||
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
print(f"Server error status: {e.response.status_code}")
|
||||
print(f"Server error response: {e.response.text}")
|
||||
try:
|
||||
error_json = e.response.json()
|
||||
print(f"Parsed error: {error_json}")
|
||||
except:
|
||||
print("Could not parse error response as JSON")
|
||||
raise
|
||||
|
||||
assert data["success"] is True
|
||||
assert len(data["results"]) == 1
|
||||
result = data["results"][0]
|
||||
await assert_crawl_result_structure(result)
|
||||
assert result["success"] is True
|
||||
assert "extracted_content" in result
|
||||
assert result["extracted_content"] is not None
|
||||
|
||||
# Extracted content should be a JSON string representing a list of dicts
|
||||
try:
|
||||
extracted_data = json.loads(result["extracted_content"])
|
||||
assert isinstance(extracted_data, list)
|
||||
assert len(extracted_data) > 0 # Should find some books
|
||||
# Check structure of the first extracted item
|
||||
first_item = extracted_data[0]
|
||||
assert "title" in first_item
|
||||
assert "price" in first_item
|
||||
assert "rating" in first_item
|
||||
assert "star-rating" in first_item["rating"] # e.g., "star-rating Three"
|
||||
except (json.JSONDecodeError, AssertionError) as e:
|
||||
pytest.fail(f"Extracted content parsing or validation failed: {e}\nContent: {result['extracted_content']}")
|
||||
|
||||
|
||||
# 6. Extraction with LLM
|
||||
async def test_llm_extraction(self, async_client: httpx.AsyncClient):
|
||||
"""
|
||||
Test /crawl with LLMExtractionStrategy.
|
||||
NOTE: Requires the server to have appropriate LLM API keys (e.g., OPENAI_API_KEY)
|
||||
configured via .llm.env or environment variables.
|
||||
This test uses the default provider configured in the server's config.yml.
|
||||
"""
|
||||
payload = {
|
||||
"urls": [SIMPLE_HTML_URL],
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"cache_mode": CacheMode.BYPASS.value,
|
||||
"extraction_strategy": {
|
||||
"type": "LLMExtractionStrategy",
|
||||
"params": {
|
||||
"instruction": "Extract the main title and the author mentioned in the text into JSON.",
|
||||
# LLMConfig is implicitly defined by server's config.yml and .llm.env
|
||||
# If you needed to override provider/token PER REQUEST:
|
||||
"llm_config": {
|
||||
"type": "LLMConfig",
|
||||
"params": {
|
||||
"provider": "openai/gpt-4o", # Example override
|
||||
"api_token": os.getenv("OPENAI_API_KEY") # Example override
|
||||
}
|
||||
},
|
||||
"schema": { # Optional: Provide a schema for structured output
|
||||
"type": "dict", # IMPORTANT: Wrap schema dict
|
||||
"value": {
|
||||
"title": "Book Info",
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string", "description": "The main title of the work"},
|
||||
"author": {"type": "string", "description": "The author of the work"}
|
||||
},
|
||||
"required": ["title", "author"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try:
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
response.raise_for_status() # Will raise if server returns 500 (e.g., bad API key)
|
||||
data = response.json()
|
||||
except httpx.HTTPStatusError as e:
|
||||
# Catch potential server errors (like 500 due to missing/invalid API keys)
|
||||
pytest.fail(f"LLM extraction request failed: {e}. Response: {e.response.text}. Check server logs and ensure API keys are correctly configured for the server.")
|
||||
except httpx.RequestError as e:
|
||||
pytest.fail(f"LLM extraction request failed: {e}.")
|
||||
|
||||
assert data["success"] is True
|
||||
assert len(data["results"]) == 1
|
||||
result = data["results"][0]
|
||||
await assert_crawl_result_structure(result)
|
||||
assert result["success"] is True
|
||||
assert "extracted_content" in result
|
||||
assert result["extracted_content"] is not None
|
||||
|
||||
# Extracted content should be JSON (because we provided a schema)
|
||||
try:
|
||||
extracted_data = json.loads(result["extracted_content"])
|
||||
print(f"\nLLM Extracted Data: {extracted_data}") # Print for verification
|
||||
|
||||
# Handle both dict and list formats (server returns a list)
|
||||
if isinstance(extracted_data, list):
|
||||
assert len(extracted_data) > 0
|
||||
extracted_item = extracted_data[0] # Take first item
|
||||
assert isinstance(extracted_item, dict)
|
||||
assert "title" in extracted_item
|
||||
assert "author" in extracted_item
|
||||
assert "Moby-Dick" in extracted_item.get("title", "")
|
||||
assert "Herman Melville" in extracted_item.get("author", "")
|
||||
else:
|
||||
assert isinstance(extracted_data, dict)
|
||||
assert "title" in extracted_data
|
||||
assert "author" in extracted_data
|
||||
assert "Moby-Dick" in extracted_data.get("title", "")
|
||||
assert "Herman Melville" in extracted_data.get("author", "")
|
||||
except (json.JSONDecodeError, AssertionError) as e:
|
||||
pytest.fail(f"LLM extracted content parsing or validation failed: {e}\nContent: {result['extracted_content']}")
|
||||
except Exception as e: # Catch any other unexpected error
|
||||
pytest.fail(f"An unexpected error occurred during LLM result processing: {e}\nContent: {result['extracted_content']}")
|
||||
|
||||
|
||||
# 7. Error Handling Tests
|
||||
async def test_invalid_url_handling(self, async_client: httpx.AsyncClient):
|
||||
"""Test error handling for invalid URLs."""
|
||||
payload = {
|
||||
"urls": ["invalid-url", "https://nonexistent-domain-12345.com"],
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {"type": "CrawlerRunConfig", "params": {"cache_mode": CacheMode.BYPASS.value}}
|
||||
}
|
||||
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
# Should return 200 with failed results, not 500
|
||||
print(f"Status code: {response.status_code}")
|
||||
print(f"Response: {response.text}")
|
||||
assert response.status_code == 500
|
||||
data = response.json()
|
||||
assert data["detail"].startswith("Crawl request failed:")
|
||||
|
||||
async def test_mixed_success_failure_urls(self, async_client: httpx.AsyncClient):
|
||||
"""Test handling of mixed success/failure URLs."""
|
||||
payload = {
|
||||
"urls": [
|
||||
SIMPLE_HTML_URL, # Should succeed
|
||||
"https://nonexistent-domain-12345.com", # Should fail
|
||||
"https://invalid-url-with-special-chars-!@#$%^&*()", # Should fail
|
||||
],
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"cache_mode": CacheMode.BYPASS.value,
|
||||
"markdown_generator": {
|
||||
"type": "DefaultMarkdownGenerator",
|
||||
"params": {
|
||||
"content_filter": {
|
||||
"type": "PruningContentFilter",
|
||||
"params": {"threshold": 0.5}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
response = await async_client.post("/crawl", json=payload)
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["success"] is True
|
||||
assert len(data["results"]) == 3
|
||||
|
||||
success_count = 0
|
||||
failure_count = 0
|
||||
|
||||
for result in data["results"]:
|
||||
if result["success"]:
|
||||
success_count += 1
|
||||
else:
|
||||
failure_count += 1
|
||||
assert "error_message" in result
|
||||
assert len(result["error_message"]) > 0
|
||||
|
||||
assert success_count >= 1 # At least one should succeed
|
||||
assert failure_count >= 1 # At least one should fail
|
||||
|
||||
async def test_streaming_mixed_urls(self, async_client: httpx.AsyncClient):
|
||||
"""Test streaming with mixed success/failure URLs."""
|
||||
payload = {
|
||||
"urls": [
|
||||
SIMPLE_HTML_URL, # Should succeed
|
||||
"https://nonexistent-domain-12345.com", # Should fail
|
||||
],
|
||||
"browser_config": {"type": "BrowserConfig", "params": {"headless": True}},
|
||||
"crawler_config": {
|
||||
"type": "CrawlerRunConfig",
|
||||
"params": {
|
||||
"stream": True,
|
||||
"cache_mode": CacheMode.BYPASS.value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async with async_client.stream("POST", "/crawl/stream", json=payload) as response:
|
||||
response.raise_for_status()
|
||||
results = await process_streaming_response(response)
|
||||
|
||||
assert len(results) == 2
|
||||
|
||||
success_count = 0
|
||||
failure_count = 0
|
||||
|
||||
for result in results:
|
||||
if result["success"]:
|
||||
success_count += 1
|
||||
assert result["url"] == SIMPLE_HTML_URL
|
||||
else:
|
||||
failure_count += 1
|
||||
assert "error_message" in result
|
||||
assert result["error_message"] is not None
|
||||
|
||||
assert success_count == 1
|
||||
assert failure_count == 1
|
||||
|
||||
async def test_markdown_endpoint_error_handling(self, async_client: httpx.AsyncClient):
|
||||
"""Test error handling for markdown endpoint."""
|
||||
# Test invalid URL
|
||||
invalid_payload = {"url": "invalid-url", "f": "fit"}
|
||||
response = await async_client.post("/md", json=invalid_payload)
|
||||
# Should return 400 for invalid URL format
|
||||
assert response.status_code == 400
|
||||
|
||||
# Test non-existent URL
|
||||
nonexistent_payload = {"url": "https://nonexistent-domain-12345.com", "f": "fit"}
|
||||
response = await async_client.post("/md", json=nonexistent_payload)
|
||||
# Should return 500 for crawl failure
|
||||
assert response.status_code == 500
|
||||
|
||||
async def test_html_endpoint_error_handling(self, async_client: httpx.AsyncClient):
|
||||
"""Test error handling for HTML endpoint."""
|
||||
# Test invalid URL
|
||||
invalid_payload = {"url": "invalid-url"}
|
||||
response = await async_client.post("/html", json=invalid_payload)
|
||||
# Should return 500 for crawl failure
|
||||
assert response.status_code == 500
|
||||
|
||||
async def test_screenshot_endpoint_error_handling(self, async_client: httpx.AsyncClient):
|
||||
"""Test error handling for screenshot endpoint."""
|
||||
# Test invalid URL
|
||||
invalid_payload = {"url": "invalid-url"}
|
||||
response = await async_client.post("/screenshot", json=invalid_payload)
|
||||
# Should return 500 for crawl failure
|
||||
assert response.status_code == 500
|
||||
|
||||
async def test_pdf_endpoint_error_handling(self, async_client: httpx.AsyncClient):
|
||||
"""Test error handling for PDF endpoint."""
|
||||
# Test invalid URL
|
||||
invalid_payload = {"url": "invalid-url"}
|
||||
response = await async_client.post("/pdf", json=invalid_payload)
|
||||
# Should return 500 for crawl failure
|
||||
assert response.status_code == 500
|
||||
|
||||
async def test_execute_js_endpoint_error_handling(self, async_client: httpx.AsyncClient):
|
||||
"""Test error handling for execute_js endpoint."""
|
||||
# Test invalid URL
|
||||
invalid_payload = {"url": "invalid-url", "scripts": ["return document.title;"]}
|
||||
response = await async_client.post("/execute_js", json=invalid_payload)
|
||||
# Should return 500 for crawl failure
|
||||
assert response.status_code == 500
|
||||
|
||||
async def test_llm_endpoint_error_handling(self, async_client: httpx.AsyncClient):
|
||||
"""Test error handling for LLM endpoint."""
|
||||
# Test missing query parameter
|
||||
response = await async_client.get("/llm/https://example.com")
|
||||
assert response.status_code == 422 # FastAPI validation error, not 400
|
||||
|
||||
# Test invalid URL
|
||||
response = await async_client.get("/llm/invalid-url?q=test")
|
||||
# Should return 500 for crawl failure
|
||||
assert response.status_code == 500
|
||||
|
||||
async def test_ask_endpoint_error_handling(self, async_client: httpx.AsyncClient):
|
||||
"""Test error handling for ask endpoint."""
|
||||
# Test invalid context_type
|
||||
response = await async_client.get("/ask?context_type=invalid")
|
||||
assert response.status_code == 422 # Validation error
|
||||
|
||||
# Test invalid score_ratio
|
||||
response = await async_client.get("/ask?score_ratio=2.0") # > 1.0
|
||||
assert response.status_code == 422 # Validation error
|
||||
|
||||
# Test invalid max_results
|
||||
response = await async_client.get("/ask?max_results=0") # < 1
|
||||
assert response.status_code == 422 # Validation error
|
||||
|
||||
async def test_config_dump_error_handling(self, async_client: httpx.AsyncClient):
|
||||
"""Test error handling for config dump endpoint."""
|
||||
# Test invalid code
|
||||
invalid_payload = {"code": "invalid_code"}
|
||||
response = await async_client.post("/config/dump", json=invalid_payload)
|
||||
assert response.status_code == 400
|
||||
|
||||
# Test nested function calls (not allowed)
|
||||
nested_payload = {"code": "CrawlerRunConfig(BrowserConfig())"}
|
||||
response = await async_client.post("/config/dump", json=nested_payload)
|
||||
assert response.status_code == 400
|
||||
|
||||
async def test_malformed_request_handling(self, async_client: httpx.AsyncClient):
|
||||
"""Test handling of malformed requests."""
|
||||
# Test missing required fields
|
||||
malformed_payload = {"urls": []} # Missing browser_config and crawler_config
|
||||
response = await async_client.post("/crawl", json=malformed_payload)
|
||||
print(f"Response: {response.text}")
|
||||
assert response.status_code == 422 # Validation error
|
||||
|
||||
# Test empty URLs list
|
||||
empty_urls_payload = {
|
||||
"urls": [],
|
||||
"browser_config": {"type": "BrowserConfig", "params": {}},
|
||||
"crawler_config": {"type": "CrawlerRunConfig", "params": {}}
|
||||
}
|
||||
response = await async_client.post("/crawl", json=empty_urls_payload)
|
||||
assert response.status_code == 422 # "At least one URL required"
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Define arguments for pytest programmatically
|
||||
# -v: verbose output
|
||||
# -s: show print statements immediately (useful for debugging)
|
||||
# __file__: tells pytest to run tests in the current file
|
||||
pytest_args = ["-v", "-s", __file__]
|
||||
|
||||
# You can add more pytest arguments here if needed, for example:
|
||||
# '-k test_llm_extraction': Run only the LLM test function
|
||||
# pytest_args.append("-k test_llm_extraction")
|
||||
|
||||
print(f"Running pytest with args: {pytest_args}")
|
||||
|
||||
# Execute pytest
|
||||
exit_code = pytest.main(pytest_args)
|
||||
|
||||
print(f"Pytest finished with exit code: {exit_code}")
|
||||
@@ -0,0 +1,212 @@
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Optional
|
||||
from urllib.parse import quote
|
||||
|
||||
async def get_token(session, email: str = "test@example.com") -> str:
|
||||
"""Fetch a JWT token from the /token endpoint."""
|
||||
url = "http://localhost:8000/token"
|
||||
payload = {"email": email}
|
||||
print(f"\nFetching token from {url} with email: {email}")
|
||||
try:
|
||||
async with session.post(url, json=payload) as response:
|
||||
status = response.status
|
||||
data = await response.json()
|
||||
print(f"Token Response Status: {status}")
|
||||
print(f"Token Response: {json.dumps(data, indent=2)}")
|
||||
if status == 200:
|
||||
return data["access_token"]
|
||||
else:
|
||||
raise Exception(f"Failed to get token: {data.get('detail', 'Unknown error')}")
|
||||
except Exception as e:
|
||||
print(f"Error fetching token: {str(e)}")
|
||||
raise
|
||||
|
||||
async def test_endpoint(
|
||||
session,
|
||||
endpoint: str,
|
||||
url: str,
|
||||
token: str,
|
||||
params: Optional[dict] = None,
|
||||
expected_status: int = 200
|
||||
) -> Optional[dict]:
|
||||
"""Test an endpoint with token and print results."""
|
||||
params = params or {}
|
||||
param_str = "&".join(f"{k}={v}" for k, v in params.items())
|
||||
full_url = f"http://localhost:8000/{endpoint}/{quote(url)}"
|
||||
if param_str:
|
||||
full_url += f"?{param_str}"
|
||||
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
print(f"\nTesting: {full_url}")
|
||||
|
||||
try:
|
||||
async with session.get(full_url, headers=headers) as response:
|
||||
status = response.status
|
||||
try:
|
||||
data = await response.json()
|
||||
except:
|
||||
data = await response.text()
|
||||
|
||||
print(f"Status: {status} (Expected: {expected_status})")
|
||||
if isinstance(data, dict):
|
||||
print(f"Response: {json.dumps(data, indent=2)}")
|
||||
else:
|
||||
print(f"Response: {data[:500]}...") # First 500 chars
|
||||
assert status == expected_status, f"Expected {expected_status}, got {status}"
|
||||
return data
|
||||
except Exception as e:
|
||||
print(f"Error: {str(e)}")
|
||||
return None
|
||||
|
||||
|
||||
async def test_stream_crawl(session, token: str):
|
||||
"""Test the /crawl/stream endpoint with multiple URLs."""
|
||||
url = "http://localhost:8000/crawl/stream"
|
||||
payload = {
|
||||
"urls": [
|
||||
"https://example.com",
|
||||
"https://example.com/page1", # Replicated example.com with variation
|
||||
"https://example.com/page2", # Replicated example.com with variation
|
||||
"https://example.com/page3", # Replicated example.com with variation
|
||||
# "https://www.python.org",
|
||||
# "https://news.ycombinator.com/news"
|
||||
],
|
||||
"browser_config": {"headless": True, "viewport": {"width": 1200}},
|
||||
"crawler_config": {"stream": True, "cache_mode": "bypass"}
|
||||
}
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
print(f"\nTesting Streaming Crawl: {url}")
|
||||
print(f"Payload: {json.dumps(payload, indent=2)}")
|
||||
|
||||
try:
|
||||
async with session.post(url, json=payload, headers=headers) as response:
|
||||
status = response.status
|
||||
print(f"Status: {status} (Expected: 200)")
|
||||
assert status == 200, f"Expected 200, got {status}"
|
||||
|
||||
# Read streaming response line-by-line (NDJSON)
|
||||
async for line in response.content:
|
||||
if line:
|
||||
data = json.loads(line.decode('utf-8').strip())
|
||||
print(f"Streamed Result: {json.dumps(data, indent=2)}")
|
||||
except Exception as e:
|
||||
print(f"Error in streaming crawl test: {str(e)}")
|
||||
|
||||
async def run_tests():
|
||||
import aiohttp
|
||||
print("Starting API Tests...")
|
||||
|
||||
# Test URLs
|
||||
urls = [
|
||||
"example.com",
|
||||
"https://www.python.org",
|
||||
"https://news.ycombinator.com/news",
|
||||
"https://github.com/trending"
|
||||
]
|
||||
|
||||
async with aiohttp.ClientSession() as session:
|
||||
# Fetch token once and reuse it
|
||||
token = await get_token(session)
|
||||
if not token:
|
||||
print("Aborting tests due to token failure!")
|
||||
return
|
||||
|
||||
print("\n=== Testing Crawl Endpoint ===")
|
||||
crawl_payload = {
|
||||
"urls": ["https://example.com"],
|
||||
"browser_config": {"headless": True},
|
||||
"crawler_config": {"stream": False}
|
||||
}
|
||||
async with session.post(
|
||||
"http://localhost:8000/crawl",
|
||||
json=crawl_payload,
|
||||
headers={"Authorization": f"Bearer {token}"}
|
||||
) as response:
|
||||
status = response.status
|
||||
data = await response.json()
|
||||
print(f"\nCrawl Endpoint Status: {status}")
|
||||
print(f"Crawl Response: {json.dumps(data, indent=2)}")
|
||||
|
||||
|
||||
print("\n=== Testing Crawl Stream Endpoint ===")
|
||||
await test_stream_crawl(session, token)
|
||||
|
||||
print("\n=== Testing Markdown Endpoint ===")
|
||||
for url in []: #urls:
|
||||
for filter_type in ["raw", "fit", "bm25", "llm"]:
|
||||
params = {"f": filter_type}
|
||||
if filter_type in ["bm25", "llm"]:
|
||||
params["q"] = "extract main content"
|
||||
|
||||
for cache in ["0", "1"]:
|
||||
params["c"] = cache
|
||||
await test_endpoint(session, "md", url, token, params)
|
||||
await asyncio.sleep(1) # Be nice to the server
|
||||
|
||||
print("\n=== Testing LLM Endpoint ===")
|
||||
for url in urls:
|
||||
# Test basic extraction (direct response now)
|
||||
result = await test_endpoint(
|
||||
session,
|
||||
"llm",
|
||||
url,
|
||||
token,
|
||||
{"q": "Extract title and main content"}
|
||||
)
|
||||
|
||||
# Test with schema (direct response)
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string"},
|
||||
"content": {"type": "string"},
|
||||
"links": {"type": "array", "items": {"type": "string"}}
|
||||
}
|
||||
}
|
||||
result = await test_endpoint(
|
||||
session,
|
||||
"llm",
|
||||
url,
|
||||
token,
|
||||
{
|
||||
"q": "Extract content with links",
|
||||
"s": json.dumps(schema),
|
||||
"c": "1" # Test with cache
|
||||
}
|
||||
)
|
||||
await asyncio.sleep(2) # Be nice to the server
|
||||
|
||||
print("\n=== Testing Error Cases ===")
|
||||
# Test invalid URL
|
||||
await test_endpoint(
|
||||
session,
|
||||
"md",
|
||||
"not_a_real_url",
|
||||
token,
|
||||
expected_status=500
|
||||
)
|
||||
|
||||
# Test invalid filter type
|
||||
await test_endpoint(
|
||||
session,
|
||||
"md",
|
||||
"example.com",
|
||||
token,
|
||||
{"f": "invalid"},
|
||||
expected_status=422
|
||||
)
|
||||
|
||||
# Test LLM without query (should fail per your server logic)
|
||||
await test_endpoint(
|
||||
session,
|
||||
"llm",
|
||||
"example.com",
|
||||
token,
|
||||
expected_status=400
|
||||
)
|
||||
|
||||
print("\nAll tests completed!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(run_tests())
|
||||
Reference in New Issue
Block a user