chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,151 @@
|
||||
import pytest
|
||||
import asyncio
|
||||
import time
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig, CacheMode
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_for_timeout_separate_from_page_timeout():
|
||||
"""Test that wait_for has its own timeout separate from page_timeout"""
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
|
||||
# Test with short wait_for_timeout but longer page_timeout
|
||||
config = CrawlerRunConfig(
|
||||
wait_for="css:.nonexistent-element",
|
||||
wait_for_timeout=2000, # 2 seconds
|
||||
page_timeout=10000, # 10 seconds
|
||||
cache_mode=CacheMode.BYPASS
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
start_time = time.time()
|
||||
result = await crawler.arun("https://example.com", config=config)
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
# Should timeout after ~2 seconds (wait_for_timeout), not 10 seconds
|
||||
assert elapsed < 5, f"Expected timeout around 2s, but took {elapsed:.2f}s"
|
||||
assert result.success, "Crawl should still succeed even if wait_for times out"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_for_timeout_with_existing_element():
|
||||
"""Test that wait_for_timeout works correctly when element exists"""
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
wait_for="css:body", # This should exist quickly
|
||||
wait_for_timeout=5000,
|
||||
cache_mode=CacheMode.BYPASS
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
start_time = time.time()
|
||||
result = await crawler.arun("https://example.com", config=config)
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
# Should complete quickly since body element exists
|
||||
assert elapsed < 3, f"Expected quick completion, but took {elapsed:.2f}s"
|
||||
assert result.success
|
||||
assert "<body" in result.html
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_javascript_wait_for_timeout():
|
||||
"""Test wait_for_timeout with JavaScript condition"""
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
wait_for="js:() => window.nonExistentVariable === true",
|
||||
wait_for_timeout=2000,
|
||||
cache_mode=CacheMode.BYPASS
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
start_time = time.time()
|
||||
result = await crawler.arun("https://example.com", config=config)
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
# Should timeout after ~2 seconds
|
||||
assert elapsed < 4, f"Expected timeout around 2s, but took {elapsed:.2f}s"
|
||||
assert result.success
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_google_analytics_integration():
|
||||
"""Test that Google Analytics scripts are properly integrated"""
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
|
||||
|
||||
# Test with a simple HTML page that we can control
|
||||
html_content = """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Test GA Integration</title>
|
||||
<!-- Google tag (gtag.js) -->
|
||||
<script async src="https://www.googletagmanager.com/gtag/js?id=GA_MEASUREMENT_ID"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('config', 'GA_MEASUREMENT_ID');
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Test Page</h1>
|
||||
<p>Testing Google Analytics integration</p>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(f"raw://{html_content}", config=config)
|
||||
|
||||
assert result.success
|
||||
# Check that GA scripts are preserved in the HTML
|
||||
assert "googletagmanager.com/gtag/js" in result.html
|
||||
assert "dataLayer" in result.html
|
||||
assert "gtag('config'" in result.html
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mkdocs_no_duplicate_gtag():
|
||||
"""Test that there are no duplicate gtag.js entries in documentation"""
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
|
||||
|
||||
# Simulate MkDocs-like HTML structure
|
||||
html_content = """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Crawl4AI Documentation</title>
|
||||
<script async src="https://www.googletagmanager.com/gtag/js?id=G-XXXXXXXXXX"></script>
|
||||
<script>
|
||||
window.dataLayer = window.dataLayer || [];
|
||||
function gtag(){dataLayer.push(arguments);}
|
||||
gtag('config', 'G-XXXXXXXXXX');
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Crawl4AI Documentation</h1>
|
||||
<p>Welcome to the documentation</p>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(f"raw://{html_content}", config=config)
|
||||
|
||||
assert result.success
|
||||
# Count occurrences of gtag.js to ensure no duplicates
|
||||
gtag_count = result.html.count("googletagmanager.com/gtag/js")
|
||||
assert gtag_count <= 1, f"Found {gtag_count} gtag.js scripts, expected at most 1"
|
||||
|
||||
# Ensure the analytics functionality is still there
|
||||
if gtag_count == 1:
|
||||
assert "dataLayer" in result.html
|
||||
assert "gtag('config'" in result.html
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -0,0 +1,317 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import asyncio
|
||||
import pytest
|
||||
import os
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
|
||||
from crawl4ai import JsonCssExtractionStrategy, LLMExtractionStrategy, LLMConfig
|
||||
from crawl4ai.content_filter_strategy import BM25ContentFilter
|
||||
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
|
||||
from crawl4ai.async_url_seeder import AsyncUrlSeeder
|
||||
from crawl4ai.utils import RobotsParser
|
||||
|
||||
|
||||
class TestCrawl4AIv070:
|
||||
"""Test suite for Crawl4AI v0.7.0 changes"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_url_parsing(self):
|
||||
"""Test raw:// URL parsing logic fix"""
|
||||
html_content = "<html><body><h1>Test Content</h1><p>This is a test paragraph.</p></body></html>"
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
# Test raw:// prefix
|
||||
result1 = await crawler.arun(f"raw://{html_content}")
|
||||
assert result1.success
|
||||
assert "Test Content" in result1.markdown
|
||||
|
||||
# Test raw: prefix
|
||||
result2 = await crawler.arun(f"raw:{html_content}")
|
||||
assert result2.success
|
||||
assert "Test Content" in result2.markdown
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_max_pages_limit_batch_processing(self):
|
||||
"""Test max_pages limit is respected during batch processing"""
|
||||
urls = [
|
||||
"https://httpbin.org/html",
|
||||
"https://httpbin.org/json",
|
||||
"https://httpbin.org/xml"
|
||||
]
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
max_pages=2
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
results = await crawler.arun_many(urls, config=config)
|
||||
# Should only process 2 pages due to max_pages limit
|
||||
successful_results = [r for r in results if r.success]
|
||||
assert len(successful_results) <= 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_navigation_abort_handling(self):
|
||||
"""Test handling of navigation aborts during file downloads"""
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
# Test with a URL that might cause navigation issues
|
||||
result = await crawler.arun(
|
||||
"https://httpbin.org/status/404",
|
||||
config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
|
||||
)
|
||||
# Should not crash even with navigation issues
|
||||
assert result is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_screenshot_capture_fix(self):
|
||||
"""Test screenshot capture improvements"""
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
screenshot=True
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun("https://httpbin.org/html", config=config)
|
||||
assert result.success
|
||||
assert result.screenshot is not None
|
||||
assert len(result.screenshot) > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_redirect_status_codes(self):
|
||||
"""Test that real redirect status codes are surfaced"""
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
# Test with a redirect URL
|
||||
result = await crawler.arun(
|
||||
"https://httpbin.org/redirect/1",
|
||||
config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
|
||||
)
|
||||
assert result.success
|
||||
# Should have redirect information
|
||||
assert result.status_code in [200, 301, 302, 303, 307, 308]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_local_file_processing(self):
|
||||
"""Test local file processing with captured_console initialization"""
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.html', delete=False) as f:
|
||||
f.write("<html><body><h1>Local File Test</h1></body></html>")
|
||||
temp_file = f.name
|
||||
|
||||
try:
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun(f"file://{temp_file}")
|
||||
assert result.success
|
||||
assert "Local File Test" in result.markdown
|
||||
finally:
|
||||
os.unlink(temp_file)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_robots_txt_wildcard_support(self):
|
||||
"""Test robots.txt wildcard rules support"""
|
||||
parser = RobotsParser()
|
||||
|
||||
# Test wildcard patterns
|
||||
robots_content = "User-agent: *\nDisallow: /admin/*\nDisallow: *.pdf"
|
||||
|
||||
# This should work without throwing exceptions
|
||||
assert parser is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exclude_external_images(self):
|
||||
"""Test exclude_external_images flag"""
|
||||
html_with_images = '''
|
||||
<html><body>
|
||||
<img src="/local-image.jpg" alt="Local">
|
||||
<img src="https://external.com/image.jpg" alt="External">
|
||||
</body></html>
|
||||
'''
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
exclude_external_images=True
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun(f"raw://{html_with_images}", config=config)
|
||||
assert result.success
|
||||
# External images should be excluded
|
||||
assert "external.com" not in result.cleaned_html
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_llm_extraction_strategy_fix(self):
|
||||
"""Test LLM extraction strategy choices error fix"""
|
||||
if not os.getenv("OPENAI_API_KEY"):
|
||||
pytest.skip("OpenAI API key not available")
|
||||
|
||||
llm_config = LLMConfig(
|
||||
provider="openai/gpt-4o-mini",
|
||||
api_token=os.getenv("OPENAI_API_KEY")
|
||||
)
|
||||
|
||||
strategy = LLMExtractionStrategy(
|
||||
llm_config=llm_config,
|
||||
instruction="Extract the main heading",
|
||||
extraction_type="block"
|
||||
)
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
extraction_strategy=strategy
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun("https://httpbin.org/html", config=config)
|
||||
assert result.success
|
||||
# Should not throw 'str' object has no attribute 'choices' error
|
||||
assert result.extracted_content is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_for_timeout(self):
|
||||
"""Test separate timeout for wait_for condition"""
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
wait_for="css:non-existent-element",
|
||||
wait_for_timeout=1000 # 1 second timeout
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun("https://httpbin.org/html", config=config)
|
||||
# Should timeout gracefully and still return result
|
||||
assert result is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bm25_content_filter_language_parameter(self):
|
||||
"""Test BM25 filter with language parameter for stemming"""
|
||||
content_filter = BM25ContentFilter(
|
||||
user_query="test content",
|
||||
language="english",
|
||||
use_stemming=True
|
||||
)
|
||||
|
||||
markdown_generator = DefaultMarkdownGenerator(
|
||||
content_filter=content_filter
|
||||
)
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
markdown_generator=markdown_generator
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun("https://httpbin.org/html", config=config)
|
||||
assert result.success
|
||||
assert result.markdown is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_url_normalization(self):
|
||||
"""Test URL normalization for invalid schemes and trailing slashes"""
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
# Test with trailing slash
|
||||
result = await crawler.arun(
|
||||
"https://httpbin.org/html/",
|
||||
config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
|
||||
)
|
||||
assert result.success
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_max_scroll_steps(self):
|
||||
"""Test max_scroll_steps parameter for full page scanning"""
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
scan_full_page=True,
|
||||
max_scroll_steps=3
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun("https://httpbin.org/html", config=config)
|
||||
assert result.success
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_url_seeder(self):
|
||||
"""Test AsyncUrlSeeder functionality"""
|
||||
seeder = AsyncUrlSeeder(
|
||||
base_url="https://httpbin.org",
|
||||
max_depth=1,
|
||||
max_urls=5
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
urls = await seeder.seed(crawler)
|
||||
assert isinstance(urls, list)
|
||||
assert len(urls) <= 5
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pdf_processing_timeout(self):
|
||||
"""Test PDF processing with timeout"""
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
pdf=True,
|
||||
pdf_timeout=10000 # 10 seconds
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun("https://httpbin.org/html", config=config)
|
||||
assert result.success
|
||||
# PDF might be None for HTML pages, but should not hang
|
||||
assert result.pdf is not None or result.pdf is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_session_management(self):
|
||||
"""Test improved browser session management"""
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
use_persistent_context=True
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(
|
||||
"https://httpbin.org/html",
|
||||
config=CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
|
||||
)
|
||||
assert result.success
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_memory_management(self):
|
||||
"""Test memory management features"""
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
memory_threshold_percent=80.0,
|
||||
check_interval=1.0,
|
||||
memory_wait_timeout=600 # 10 minutes default
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun("https://httpbin.org/html", config=config)
|
||||
assert result.success
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_virtual_scroll_support(self):
|
||||
"""Test virtual scroll support for modern web scraping"""
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
scan_full_page=True,
|
||||
virtual_scroll=True
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun("https://httpbin.org/html", config=config)
|
||||
assert result.success
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_adaptive_crawling(self):
|
||||
"""Test adaptive crawling feature"""
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
adaptive_crawling=True
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun("https://httpbin.org/html", config=config)
|
||||
assert result.success
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Run the tests
|
||||
pytest.main([__file__, "-v"])
|
||||
Reference in New Issue
Block a user