chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,335 @@
|
||||
# ==== File: build_dummy_site.py ====
|
||||
|
||||
import os
|
||||
import random
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
# --- Configuration ---
|
||||
NUM_CATEGORIES = 3
|
||||
NUM_SUBCATEGORIES_PER_CAT = 2 # Results in NUM_CATEGORIES * NUM_SUBCATEGORIES_PER_CAT total L2 categories
|
||||
NUM_PRODUCTS_PER_SUBCAT = 5 # Products listed on L3 pages
|
||||
MAX_DEPTH_TARGET = 5 # Explicitly set target depth
|
||||
|
||||
# --- Helper Functions ---
|
||||
|
||||
def generate_lorem(words=20):
|
||||
"""Generates simple placeholder text."""
|
||||
lorem_words = ["lorem", "ipsum", "dolor", "sit", "amet", "consectetur",
|
||||
"adipiscing", "elit", "sed", "do", "eiusmod", "tempor",
|
||||
"incididunt", "ut", "labore", "et", "dolore", "magna", "aliqua"]
|
||||
return " ".join(random.choice(lorem_words) for _ in range(words)).capitalize() + "."
|
||||
|
||||
def create_html_page(filepath: Path, title: str, body_content: str, breadcrumbs: list = [], head_extras: str = ""):
|
||||
"""Creates an HTML file with basic structure and inline CSS."""
|
||||
os.makedirs(filepath.parent, exist_ok=True)
|
||||
|
||||
# Generate breadcrumb HTML using the 'link' provided in the breadcrumbs list
|
||||
breadcrumb_html = ""
|
||||
if breadcrumbs:
|
||||
links_html = " » ".join(f'<a href="{bc["link"]}">{bc["name"]}</a>' for bc in breadcrumbs)
|
||||
breadcrumb_html = f"<nav class='breadcrumbs'>{links_html} » {title}</nav>"
|
||||
|
||||
# Basic CSS for structure identification (kept the same)
|
||||
css = """
|
||||
<style>
|
||||
body {
|
||||
font-family: sans-serif;
|
||||
padding: 20px;
|
||||
background-color: #1e1e1e;
|
||||
color: #d1d1d1;
|
||||
}
|
||||
|
||||
.container {
|
||||
max-width: 960px;
|
||||
margin: auto;
|
||||
background: #2c2c2c;
|
||||
padding: 20px;
|
||||
border-radius: 5px;
|
||||
box-shadow: 0 2px 5px rgba(0, 0, 0, 0.5);
|
||||
}
|
||||
|
||||
h1, h2 {
|
||||
color: #ccc;
|
||||
}
|
||||
|
||||
a {
|
||||
color: #9bcdff;
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
a:hover {
|
||||
text-decoration: underline;
|
||||
}
|
||||
|
||||
ul {
|
||||
list-style: none;
|
||||
padding-left: 0;
|
||||
}
|
||||
|
||||
li {
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.category-link,
|
||||
.subcategory-link,
|
||||
.product-link,
|
||||
.details-link,
|
||||
.reviews-link {
|
||||
display: block;
|
||||
padding: 8px;
|
||||
background-color: #3a3a3a;
|
||||
border-radius: 3px;
|
||||
}
|
||||
|
||||
.product-preview {
|
||||
border: 1px solid #444;
|
||||
padding: 10px;
|
||||
margin-bottom: 10px;
|
||||
border-radius: 4px;
|
||||
background-color: #2a2a2a;
|
||||
}
|
||||
|
||||
.product-title {
|
||||
color: #d1d1d1;
|
||||
}
|
||||
|
||||
.product-price {
|
||||
font-weight: bold;
|
||||
color: #85e085;
|
||||
}
|
||||
|
||||
.product-description,
|
||||
.product-specs,
|
||||
.product-reviews {
|
||||
margin-top: 15px;
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.product-specs li {
|
||||
margin-bottom: 5px;
|
||||
font-size: 0.9em;
|
||||
}
|
||||
|
||||
.spec-name {
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.breadcrumbs {
|
||||
margin-bottom: 20px;
|
||||
font-size: 0.9em;
|
||||
color: #888;
|
||||
}
|
||||
|
||||
.breadcrumbs a {
|
||||
color: #9bcdff;
|
||||
}
|
||||
</style>
|
||||
"""
|
||||
html_content = f"""<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>{title} - FakeShop</title>
|
||||
{head_extras}
|
||||
{css}
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
{breadcrumb_html}
|
||||
<h1>{title}</h1>
|
||||
{body_content}
|
||||
</div>
|
||||
</body>
|
||||
</html>"""
|
||||
with open(filepath, "w", encoding="utf-8") as f:
|
||||
f.write(html_content)
|
||||
# Keep print statement concise for clarity
|
||||
# print(f"Created: {filepath}")
|
||||
|
||||
def generate_site(base_dir: Path, site_name: str = "FakeShop", base_path: str = ""):
|
||||
"""Generates the dummy website structure."""
|
||||
base_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# --- Clean and prepare the base path for URL construction ---
|
||||
# Ensure it starts with '/' if not empty, and remove any trailing '/'
|
||||
if base_path:
|
||||
full_base_path = "/" + base_path.strip('/')
|
||||
else:
|
||||
full_base_path = "" # Represents the root
|
||||
|
||||
print(f"Using base path for links: '{full_base_path}'")
|
||||
|
||||
# --- Level 0: Homepage ---
|
||||
home_body = "<h2>Welcome to FakeShop!</h2><p>Your one-stop shop for imaginary items.</p><h3>Categories:</h3>\n<ul>"
|
||||
# Define the *actual* link path for the homepage breadcrumb
|
||||
home_link_path = f"{full_base_path}/index.html"
|
||||
breadcrumbs_home = [{"name": "Home", "link": home_link_path}] # Base breadcrumb
|
||||
|
||||
# Links *within* the page content should remain relative
|
||||
for i in range(NUM_CATEGORIES):
|
||||
cat_name = f"Category-{i+1}"
|
||||
cat_folder_name = quote(cat_name.lower().replace(" ", "-"))
|
||||
# This path is relative to the current directory (index.html)
|
||||
cat_relative_page_path = f"{cat_folder_name}/index.html"
|
||||
home_body += f'<li><a class="category-link" href="{cat_relative_page_path}">{cat_name}</a> - {generate_lorem(10)}</li>'
|
||||
home_body += "</ul>"
|
||||
create_html_page(base_dir / "index.html", "Homepage", home_body, []) # No breadcrumbs *on* the homepage itself
|
||||
|
||||
# --- Levels 1-5 ---
|
||||
for i in range(NUM_CATEGORIES):
|
||||
cat_name = f"Category-{i+1}"
|
||||
cat_folder_name = quote(cat_name.lower().replace(" ", "-"))
|
||||
cat_dir = base_dir / cat_folder_name
|
||||
# This is the *absolute* path for the breadcrumb link
|
||||
cat_link_path = f"{full_base_path}/{cat_folder_name}/index.html"
|
||||
# Update breadcrumbs list for this level
|
||||
breadcrumbs_cat = breadcrumbs_home + [{"name": cat_name, "link": cat_link_path}]
|
||||
|
||||
# --- Level 1: Category Page ---
|
||||
cat_body = f"<p>{generate_lorem(15)} for {cat_name}.</p><h3>Sub-Categories:</h3>\n<ul>"
|
||||
for j in range(NUM_SUBCATEGORIES_PER_CAT):
|
||||
subcat_name = f"{cat_name}-Sub-{j+1}"
|
||||
subcat_folder_name = quote(subcat_name.lower().replace(" ", "-"))
|
||||
# Path relative to the category page
|
||||
subcat_relative_page_path = f"{subcat_folder_name}/index.html"
|
||||
cat_body += f'<li><a class="subcategory-link" href="{subcat_relative_page_path}">{subcat_name}</a> - {generate_lorem(8)}</li>'
|
||||
cat_body += "</ul>"
|
||||
# Pass the updated breadcrumbs list
|
||||
create_html_page(cat_dir / "index.html", cat_name, cat_body, breadcrumbs_home) # Parent breadcrumb needed here
|
||||
|
||||
for j in range(NUM_SUBCATEGORIES_PER_CAT):
|
||||
subcat_name = f"{cat_name}-Sub-{j+1}"
|
||||
subcat_folder_name = quote(subcat_name.lower().replace(" ", "-"))
|
||||
subcat_dir = cat_dir / subcat_folder_name
|
||||
# Absolute path for the breadcrumb link
|
||||
subcat_link_path = f"{full_base_path}/{cat_folder_name}/{subcat_folder_name}/index.html"
|
||||
# Update breadcrumbs list for this level
|
||||
breadcrumbs_subcat = breadcrumbs_cat + [{"name": subcat_name, "link": subcat_link_path}]
|
||||
|
||||
# --- Level 2: Sub-Category Page (Product List) ---
|
||||
subcat_body = f"<p>Explore products in {subcat_name}. {generate_lorem(12)}</p><h3>Products:</h3>\n<ul class='product-list'>"
|
||||
for k in range(NUM_PRODUCTS_PER_SUBCAT):
|
||||
prod_id = f"P{i+1}{j+1}{k+1:03d}" # e.g., P11001
|
||||
prod_name = f"{subcat_name} Product {k+1} ({prod_id})"
|
||||
# Filename relative to the subcategory page
|
||||
prod_filename = f"product_{prod_id}.html"
|
||||
# Absolute path for the breadcrumb link
|
||||
prod_link_path = f"{full_base_path}/{cat_folder_name}/{subcat_folder_name}/{prod_filename}"
|
||||
|
||||
# Preview on list page (link remains relative)
|
||||
subcat_body += f"""
|
||||
<li>
|
||||
<div class="product-preview">
|
||||
<a class="product-link" href="{prod_filename}"><strong>{prod_name}</strong></a>
|
||||
<p>{generate_lorem(10)}</p>
|
||||
<span class="product-price">£{random.uniform(10, 500):.2f}</span>
|
||||
</div>
|
||||
</li>"""
|
||||
|
||||
# --- Level 3: Product Page ---
|
||||
prod_price = random.uniform(10, 500)
|
||||
prod_desc = generate_lorem(40)
|
||||
prod_specs = {f"Spec {s+1}": generate_lorem(3) for s in range(random.randint(3,6))}
|
||||
prod_reviews_count = random.randint(0, 150)
|
||||
# Relative filenames for links on this page
|
||||
details_filename_relative = f"product_{prod_id}_details.html"
|
||||
reviews_filename_relative = f"product_{prod_id}_reviews.html"
|
||||
|
||||
prod_body = f"""
|
||||
<p class="product-price">Price: £{prod_price:.2f}</p>
|
||||
<div class="product-description">
|
||||
<h2>Description</h2>
|
||||
<p>{prod_desc}</p>
|
||||
</div>
|
||||
<div class="product-specs">
|
||||
<h2>Specifications</h2>
|
||||
<ul>
|
||||
{''.join(f'<li><span class="spec-name">{name}</span>: <span class="spec-value">{value}</span></li>' for name, value in prod_specs.items())}
|
||||
</ul>
|
||||
</div>
|
||||
<div class="product-reviews">
|
||||
<h2>Reviews</h2>
|
||||
<p>Total Reviews: <span class="review-count">{prod_reviews_count}</span></p>
|
||||
</div>
|
||||
<hr>
|
||||
<p>
|
||||
<a class="details-link" href="{details_filename_relative}">View More Details</a> |
|
||||
<a class="reviews-link" href="{reviews_filename_relative}">See All Reviews</a>
|
||||
</p>
|
||||
"""
|
||||
# Update breadcrumbs list for this level
|
||||
breadcrumbs_prod = breadcrumbs_subcat + [{"name": prod_name, "link": prod_link_path}]
|
||||
# Pass the updated breadcrumbs list
|
||||
create_html_page(subcat_dir / prod_filename, prod_name, prod_body, breadcrumbs_subcat) # Parent breadcrumb needed here
|
||||
|
||||
# --- Level 4: Product Details Page ---
|
||||
details_filename = f"product_{prod_id}_details.html" # Actual filename
|
||||
# Absolute path for the breadcrumb link
|
||||
details_link_path = f"{full_base_path}/{cat_folder_name}/{subcat_folder_name}/{details_filename}"
|
||||
details_body = f"<p>This page contains extremely detailed information about {prod_name}.</p>{generate_lorem(100)}"
|
||||
# Update breadcrumbs list for this level
|
||||
breadcrumbs_details = breadcrumbs_prod + [{"name": "Details", "link": details_link_path}]
|
||||
# Pass the updated breadcrumbs list
|
||||
create_html_page(subcat_dir / details_filename, f"{prod_name} - Details", details_body, breadcrumbs_prod) # Parent breadcrumb needed here
|
||||
|
||||
# --- Level 5: Product Reviews Page ---
|
||||
reviews_filename = f"product_{prod_id}_reviews.html" # Actual filename
|
||||
# Absolute path for the breadcrumb link
|
||||
reviews_link_path = f"{full_base_path}/{cat_folder_name}/{subcat_folder_name}/{reviews_filename}"
|
||||
reviews_body = f"<p>All {prod_reviews_count} reviews for {prod_name} are listed here.</p><ul>"
|
||||
for r in range(prod_reviews_count):
|
||||
reviews_body += f"<li>Review {r+1}: {generate_lorem(random.randint(15, 50))}</li>"
|
||||
reviews_body += "</ul>"
|
||||
# Update breadcrumbs list for this level
|
||||
breadcrumbs_reviews = breadcrumbs_prod + [{"name": "Reviews", "link": reviews_link_path}]
|
||||
# Pass the updated breadcrumbs list
|
||||
create_html_page(subcat_dir / reviews_filename, f"{prod_name} - Reviews", reviews_body, breadcrumbs_prod) # Parent breadcrumb needed here
|
||||
|
||||
|
||||
subcat_body += "</ul>" # Close product-list ul
|
||||
# Pass the correct breadcrumbs list for the subcategory index page
|
||||
create_html_page(subcat_dir / "index.html", subcat_name, subcat_body, breadcrumbs_cat) # Parent breadcrumb needed here
|
||||
|
||||
|
||||
# --- Main Execution ---
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Generate a dummy multi-level retail website.")
|
||||
parser.add_argument(
|
||||
"-o", "--output-dir",
|
||||
type=str,
|
||||
default="dummy_retail_site",
|
||||
help="Directory to generate the website in."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-n", "--site-name",
|
||||
type=str,
|
||||
default="FakeShop",
|
||||
help="Name of the fake shop."
|
||||
)
|
||||
parser.add_argument(
|
||||
"-b", "--base-path",
|
||||
type=str,
|
||||
default="",
|
||||
help="Base path for hosting the site (e.g., 'samples/deepcrawl'). Leave empty if hosted at the root."
|
||||
)
|
||||
# Optional: Add more args to configure counts if needed
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
output_directory = Path(args.output_dir)
|
||||
site_name = args.site_name
|
||||
base_path = args.base_path
|
||||
|
||||
print(f"Generating dummy site '{site_name}' in '{output_directory}'...")
|
||||
# Pass the base_path to the generation function
|
||||
generate_site(output_directory, site_name, base_path)
|
||||
print(f"\nCreated {sum(1 for _ in output_directory.rglob('*.html'))} HTML pages.")
|
||||
print("Dummy site generation complete.")
|
||||
print(f"To serve locally (example): python -m http.server --directory {output_directory} 8000")
|
||||
if base_path:
|
||||
print(f"Access the site at: http://localhost:8000/{base_path.strip('/')}/index.html")
|
||||
else:
|
||||
print(f"Access the site at: http://localhost:8000/index.html")
|
||||
@@ -0,0 +1,56 @@
|
||||
import asyncio
|
||||
from crawl4ai import (
|
||||
AsyncWebCrawler,
|
||||
CrawlerRunConfig,
|
||||
HTTPCrawlerConfig,
|
||||
CacheMode,
|
||||
DefaultMarkdownGenerator,
|
||||
PruningContentFilter
|
||||
)
|
||||
from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy
|
||||
from crawl4ai.async_logger import AsyncLogger
|
||||
|
||||
async def main():
|
||||
# Initialize HTTP crawler strategy
|
||||
http_strategy = AsyncHTTPCrawlerStrategy(
|
||||
browser_config=HTTPCrawlerConfig(
|
||||
method="GET",
|
||||
verify_ssl=True,
|
||||
follow_redirects=True
|
||||
),
|
||||
logger=AsyncLogger(verbose=True)
|
||||
)
|
||||
|
||||
# Initialize web crawler with HTTP strategy
|
||||
async with AsyncWebCrawler(crawler_strategy=http_strategy) as crawler:
|
||||
crawler_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
markdown_generator=DefaultMarkdownGenerator(
|
||||
content_filter=PruningContentFilter(
|
||||
threshold=0.48,
|
||||
threshold_type="fixed",
|
||||
min_word_threshold=0
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
# Test different URLs
|
||||
urls = [
|
||||
"https://example.com",
|
||||
"https://httpbin.org/get",
|
||||
"raw://<html><body>Test content</body></html>"
|
||||
]
|
||||
|
||||
for url in urls:
|
||||
print(f"\n=== Testing {url} ===")
|
||||
try:
|
||||
result = await crawler.arun(url=url, config=crawler_config)
|
||||
print(f"Status: {result.status_code}")
|
||||
print(f"Raw HTML length: {len(result.html)}")
|
||||
if hasattr(result, 'markdown'):
|
||||
print(f"Markdown length: {len(result.markdown.raw_markdown)}")
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,46 @@
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
|
||||
from crawl4ai import CrawlerRunConfig, AsyncWebCrawler, CacheMode
|
||||
from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
|
||||
from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, BestFirstCrawlingStrategy
|
||||
from crawl4ai.deep_crawling.filters import FilterChain, URLPatternFilter, DomainFilter, ContentTypeFilter, ContentRelevanceFilter
|
||||
from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer
|
||||
# from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, BestFirstCrawlingStrategy
|
||||
|
||||
|
||||
async def main():
|
||||
"""Example deep crawl of documentation site."""
|
||||
filter_chain = FilterChain([
|
||||
URLPatternFilter(patterns=["*2025*"]),
|
||||
DomainFilter(allowed_domains=["techcrunch.com"]),
|
||||
ContentRelevanceFilter(query="Use of artificial intelligence in Defence applications", threshold=1),
|
||||
ContentTypeFilter(allowed_types=["text/html","application/javascript"])
|
||||
])
|
||||
config = CrawlerRunConfig(
|
||||
deep_crawl_strategy = BestFirstCrawlingStrategy(
|
||||
max_depth=2,
|
||||
include_external=False,
|
||||
filter_chain=filter_chain,
|
||||
url_scorer=KeywordRelevanceScorer(keywords=["anduril", "defence", "AI"]),
|
||||
),
|
||||
stream=False,
|
||||
verbose=True,
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
scraping_strategy=LXMLWebScrapingStrategy()
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
print("Starting deep crawl in streaming mode:")
|
||||
config.stream = True
|
||||
start_time = time.perf_counter()
|
||||
async for result in await crawler.arun(
|
||||
url="https://techcrunch.com",
|
||||
config=config
|
||||
):
|
||||
print(f"→ {result.url} (Depth: {result.metadata.get('depth', 0)})")
|
||||
print(f"Duration: {time.perf_counter() - start_time:.2f} seconds")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,382 @@
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import asyncio
|
||||
from typing import Dict, Any
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, patch
|
||||
import os
|
||||
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy
|
||||
from crawl4ai.models import AsyncCrawlResponse
|
||||
from crawl4ai.async_logger import AsyncLogger, LogLevel
|
||||
|
||||
CRAWL4AI_HOME_DIR = Path(os.path.expanduser("~")).joinpath(".crawl4ai")
|
||||
|
||||
if not CRAWL4AI_HOME_DIR.joinpath("profiles", "test_profile").exists():
|
||||
CRAWL4AI_HOME_DIR.joinpath("profiles", "test_profile").mkdir(parents=True)
|
||||
|
||||
@pytest.fixture
|
||||
def basic_html():
|
||||
return """
|
||||
<html lang="en">
|
||||
<head>
|
||||
<title>Basic HTML</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Main Heading</h1>
|
||||
<main>
|
||||
<div class="container">
|
||||
<p>Basic HTML document for testing purposes.</p>
|
||||
</div>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
# Test Config Files
|
||||
@pytest.fixture
|
||||
def basic_browser_config():
|
||||
return BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def advanced_browser_config():
|
||||
return BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True,
|
||||
use_managed_browser=True,
|
||||
user_data_dir=CRAWL4AI_HOME_DIR.joinpath("profiles", "test_profile"),
|
||||
# proxy="http://localhost:8080",
|
||||
viewport_width=1920,
|
||||
viewport_height=1080,
|
||||
user_agent_mode="random"
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def basic_crawler_config():
|
||||
return CrawlerRunConfig(
|
||||
word_count_threshold=100,
|
||||
wait_until="domcontentloaded",
|
||||
page_timeout=30000
|
||||
)
|
||||
|
||||
@pytest.fixture
|
||||
def logger():
|
||||
return AsyncLogger(verbose=True, log_level=LogLevel.DEBUG)
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def crawler_strategy(basic_browser_config, logger):
|
||||
strategy = AsyncPlaywrightCrawlerStrategy(browser_config=basic_browser_config, logger=logger)
|
||||
await strategy.start()
|
||||
yield strategy
|
||||
await strategy.close()
|
||||
|
||||
# Browser Configuration Tests
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_config_initialization():
|
||||
config = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
user_agent_mode="random"
|
||||
)
|
||||
assert config.browser_type == "chromium"
|
||||
assert config.user_agent is not None
|
||||
assert config.headless is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_persistent_browser_config():
|
||||
config = BrowserConfig(
|
||||
use_persistent_context=True,
|
||||
user_data_dir="/tmp/test_dir"
|
||||
)
|
||||
assert config.use_managed_browser is True
|
||||
assert config.user_data_dir == "/tmp/test_dir"
|
||||
|
||||
# Crawler Strategy Tests
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_page_load(crawler_strategy):
|
||||
response = await crawler_strategy.crawl(
|
||||
"https://example.com",
|
||||
CrawlerRunConfig()
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert len(response.html) > 0
|
||||
assert "Example Domain" in response.html
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_screenshot_capture(crawler_strategy):
|
||||
config = CrawlerRunConfig(screenshot=True)
|
||||
response = await crawler_strategy.crawl(
|
||||
"https://example.com",
|
||||
config
|
||||
)
|
||||
assert response.screenshot is not None
|
||||
assert len(response.screenshot) > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pdf_generation(crawler_strategy):
|
||||
config = CrawlerRunConfig(pdf=True)
|
||||
response = await crawler_strategy.crawl(
|
||||
"https://example.com",
|
||||
config
|
||||
)
|
||||
assert response.pdf_data is not None
|
||||
assert len(response.pdf_data) > 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_js_execution(crawler_strategy):
|
||||
config = CrawlerRunConfig(
|
||||
js_code="document.body.style.backgroundColor = 'red';"
|
||||
)
|
||||
response = await crawler_strategy.crawl(
|
||||
"https://example.com",
|
||||
config
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert 'background-color: red' in response.html.lower()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_js_commands(crawler_strategy):
|
||||
js_commands = [
|
||||
"document.body.style.backgroundColor = 'blue';",
|
||||
"document.title = 'Modified Title';",
|
||||
"const div = document.createElement('div'); div.id = 'test'; div.textContent = 'Test Content'; document.body.appendChild(div);"
|
||||
]
|
||||
config = CrawlerRunConfig(js_code=js_commands)
|
||||
response = await crawler_strategy.crawl(
|
||||
"https://example.com",
|
||||
config
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert 'background-color: blue' in response.html.lower()
|
||||
assert 'id="test"' in response.html
|
||||
assert '>Test Content<' in response.html
|
||||
assert '<title>Modified Title</title>' in response.html
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_complex_dom_manipulation(crawler_strategy):
|
||||
js_code = """
|
||||
// Create a complex structure
|
||||
const container = document.createElement('div');
|
||||
container.className = 'test-container';
|
||||
|
||||
const list = document.createElement('ul');
|
||||
list.className = 'test-list';
|
||||
|
||||
for (let i = 1; i <= 3; i++) {
|
||||
const item = document.createElement('li');
|
||||
item.textContent = `Item ${i}`;
|
||||
item.className = `item-${i}`;
|
||||
list.appendChild(item);
|
||||
}
|
||||
|
||||
container.appendChild(list);
|
||||
document.body.appendChild(container);
|
||||
"""
|
||||
config = CrawlerRunConfig(js_code=js_code)
|
||||
response = await crawler_strategy.crawl(
|
||||
"https://example.com",
|
||||
config
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert 'class="test-container"' in response.html
|
||||
assert 'class="test-list"' in response.html
|
||||
assert 'class="item-1"' in response.html
|
||||
assert '>Item 1<' in response.html
|
||||
assert '>Item 2<' in response.html
|
||||
assert '>Item 3<' in response.html
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_style_modifications(crawler_strategy):
|
||||
js_code = """
|
||||
const testDiv = document.createElement('div');
|
||||
testDiv.id = 'style-test';
|
||||
testDiv.style.cssText = 'color: green; font-size: 20px; margin: 10px;';
|
||||
testDiv.textContent = 'Styled Content';
|
||||
document.body.appendChild(testDiv);
|
||||
"""
|
||||
config = CrawlerRunConfig(js_code=js_code)
|
||||
response = await crawler_strategy.crawl(
|
||||
"https://example.com",
|
||||
config
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert 'id="style-test"' in response.html
|
||||
assert 'color: green' in response.html.lower()
|
||||
assert 'font-size: 20px' in response.html.lower()
|
||||
assert 'margin: 10px' in response.html.lower()
|
||||
assert '>Styled Content<' in response.html
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dynamic_content_loading(crawler_strategy):
|
||||
js_code = """
|
||||
// Simulate dynamic content loading
|
||||
setTimeout(() => {
|
||||
const dynamic = document.createElement('div');
|
||||
dynamic.id = 'dynamic-content';
|
||||
dynamic.textContent = 'Dynamically Loaded';
|
||||
document.body.appendChild(dynamic);
|
||||
}, 1000);
|
||||
|
||||
// Add a loading indicator immediately
|
||||
const loading = document.createElement('div');
|
||||
loading.id = 'loading';
|
||||
loading.textContent = 'Loading...';
|
||||
document.body.appendChild(loading);
|
||||
"""
|
||||
config = CrawlerRunConfig(js_code=js_code, delay_before_return_html=2.0)
|
||||
response = await crawler_strategy.crawl(
|
||||
"https://example.com",
|
||||
config
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert 'id="loading"' in response.html
|
||||
assert '>Loading...</' in response.html
|
||||
assert 'dynamic-content' in response.html
|
||||
assert '>Dynamically Loaded<' in response.html
|
||||
|
||||
# @pytest.mark.asyncio
|
||||
# async def test_js_return_values(crawler_strategy):
|
||||
# js_code = """
|
||||
# return {
|
||||
# title: document.title,
|
||||
# metaCount: document.getElementsByTagName('meta').length,
|
||||
# bodyClass: document.body.className
|
||||
# };
|
||||
# """
|
||||
# config = CrawlerRunConfig(js_code=js_code)
|
||||
# response = await crawler_strategy.crawl(
|
||||
# "https://example.com",
|
||||
# config
|
||||
# )
|
||||
# assert response.status_code == 200
|
||||
# assert 'Example Domain' in response.html
|
||||
# assert 'meta name="viewport"' in response.html
|
||||
# assert 'class="main"' in response.html
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_js_execution(crawler_strategy):
|
||||
js_code = """
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
document.body.style.color = 'green';
|
||||
const computedStyle = window.getComputedStyle(document.body);
|
||||
return computedStyle.color;
|
||||
"""
|
||||
config = CrawlerRunConfig(js_code=js_code)
|
||||
response = await crawler_strategy.crawl(
|
||||
"https://example.com",
|
||||
config
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert 'color: green' in response.html.lower()
|
||||
|
||||
# @pytest.mark.asyncio
|
||||
# async def test_js_error_handling(crawler_strategy):
|
||||
# js_code = """
|
||||
# // Intentionally cause different types of errors
|
||||
# const results = [];
|
||||
# try {
|
||||
# nonExistentFunction();
|
||||
# } catch (e) {
|
||||
# results.push(e.name);
|
||||
# }
|
||||
# try {
|
||||
# JSON.parse('{invalid}');
|
||||
# } catch (e) {
|
||||
# results.push(e.name);
|
||||
# }
|
||||
# return results;
|
||||
# """
|
||||
# config = CrawlerRunConfig(js_code=js_code)
|
||||
# response = await crawler_strategy.crawl(
|
||||
# "https://example.com",
|
||||
# config
|
||||
# )
|
||||
# assert response.status_code == 200
|
||||
# assert 'ReferenceError' in response.html
|
||||
# assert 'SyntaxError' in response.html
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_navigation_timeout():
|
||||
config = CrawlerRunConfig(page_timeout=1) # 1ms timeout
|
||||
with pytest.raises(Exception):
|
||||
async with AsyncPlaywrightCrawlerStrategy() as strategy:
|
||||
await strategy.crawl("https://example.com", config)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_management(crawler_strategy):
|
||||
config = CrawlerRunConfig(session_id="test_session")
|
||||
response1 = await crawler_strategy.crawl(
|
||||
"https://example.com",
|
||||
config
|
||||
)
|
||||
response2 = await crawler_strategy.crawl(
|
||||
"https://example.com",
|
||||
config
|
||||
)
|
||||
assert response1.status_code == 200
|
||||
assert response2.status_code == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_iframes(crawler_strategy):
|
||||
config = CrawlerRunConfig(
|
||||
process_iframes=True,
|
||||
wait_for_images=True
|
||||
)
|
||||
response = await crawler_strategy.crawl(
|
||||
"https://example.com",
|
||||
config
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stealth_mode(crawler_strategy):
|
||||
config = CrawlerRunConfig(
|
||||
simulate_user=True,
|
||||
override_navigator=True
|
||||
)
|
||||
response = await crawler_strategy.crawl(
|
||||
"https://bot.sannysoft.com",
|
||||
config
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("prefix", ("raw:", "raw://"))
|
||||
async def test_raw_urls(crawler_strategy, basic_html, prefix):
|
||||
url = f"{prefix}{basic_html}"
|
||||
response = await crawler_strategy.crawl(url, CrawlerRunConfig())
|
||||
assert response.html == basic_html
|
||||
|
||||
# Error Handling Tests
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_url():
|
||||
with pytest.raises(ValueError):
|
||||
async with AsyncPlaywrightCrawlerStrategy() as strategy:
|
||||
await strategy.crawl("not_a_url", CrawlerRunConfig())
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_network_error_handling():
|
||||
config = CrawlerRunConfig()
|
||||
with pytest.raises(Exception):
|
||||
async with AsyncPlaywrightCrawlerStrategy() as strategy:
|
||||
await strategy.crawl("https://invalid.example.com", config)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_remove_overlay_elements(crawler_strategy):
|
||||
config = CrawlerRunConfig(
|
||||
remove_overlay_elements=True,
|
||||
delay_before_return_html=5,
|
||||
)
|
||||
|
||||
response = await crawler_strategy.crawl(
|
||||
"https://www2.hm.com/en_us/index.html",
|
||||
config
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert "Accept all cookies" not in response.html
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
@@ -0,0 +1,171 @@
|
||||
import asyncio
|
||||
from typing import Dict
|
||||
from crawl4ai.content_filter_strategy import BM25ContentFilter, PruningContentFilter
|
||||
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
|
||||
import time
|
||||
|
||||
# Test HTML samples
|
||||
TEST_HTML_SAMPLES = {
|
||||
"basic": """
|
||||
<body>
|
||||
<h1>Test Title</h1>
|
||||
<p>This is a test paragraph with <a href="http://example.com">a link</a>.</p>
|
||||
<div class="content">
|
||||
<h2>Section 1</h2>
|
||||
<p>More content here with <b>bold text</b>.</p>
|
||||
</div>
|
||||
</body>
|
||||
""",
|
||||
|
||||
"complex": """
|
||||
<body>
|
||||
<nav>Navigation menu that should be removed</nav>
|
||||
<header>Header content to remove</header>
|
||||
<main>
|
||||
<article>
|
||||
<h1>Main Article</h1>
|
||||
<p>Important content paragraph with <a href="http://test.com">useful link</a>.</p>
|
||||
<section>
|
||||
<h2>Key Section</h2>
|
||||
<p>Detailed explanation with multiple sentences. This should be kept
|
||||
in the final output. Very important information here.</p>
|
||||
</section>
|
||||
</article>
|
||||
<aside>Sidebar content to remove</aside>
|
||||
</main>
|
||||
<footer>Footer content to remove</footer>
|
||||
</body>
|
||||
""",
|
||||
|
||||
"edge_cases": """
|
||||
<body>
|
||||
<div>
|
||||
<p></p>
|
||||
<p> </p>
|
||||
<script>alert('remove me');</script>
|
||||
<div class="advertisement">Ad content to remove</div>
|
||||
<p class="social-share">Share buttons to remove</p>
|
||||
<h1>!!Special>> Characters## Title!!</h1>
|
||||
<pre><code>def test(): pass</code></pre>
|
||||
</div>
|
||||
</body>
|
||||
""",
|
||||
|
||||
"links_citations": """
|
||||
<body>
|
||||
<h1>Document with Links</h1>
|
||||
<p>First link to <a href="http://example.com/1">Example 1</a></p>
|
||||
<p>Second link to <a href="http://example.com/2" title="Example 2">Test 2</a></p>
|
||||
<p>Image link: <img src="test.jpg" alt="test image"></p>
|
||||
<p>Repeated link to <a href="http://example.com/1">Example 1 again</a></p>
|
||||
</body>
|
||||
""",
|
||||
}
|
||||
|
||||
def test_content_filters() -> Dict[str, Dict[str, int]]:
|
||||
"""Test various content filtering strategies and return length comparisons."""
|
||||
results = {}
|
||||
|
||||
# Initialize filters
|
||||
pruning_filter = PruningContentFilter(
|
||||
threshold=0.48,
|
||||
threshold_type="fixed",
|
||||
min_word_threshold=2
|
||||
)
|
||||
|
||||
bm25_filter = BM25ContentFilter(
|
||||
bm25_threshold=1.0,
|
||||
user_query="test article content important"
|
||||
)
|
||||
|
||||
# Test each HTML sample
|
||||
for test_name, html in TEST_HTML_SAMPLES.items():
|
||||
# Store results for this test case
|
||||
results[test_name] = {}
|
||||
|
||||
# Test PruningContentFilter
|
||||
start_time = time.time()
|
||||
pruned_content = pruning_filter.filter_content(html)
|
||||
pruning_time = time.time() - start_time
|
||||
|
||||
# Test BM25ContentFilter
|
||||
start_time = time.time()
|
||||
bm25_content = bm25_filter.filter_content(html)
|
||||
bm25_time = time.time() - start_time
|
||||
|
||||
# Store results
|
||||
results[test_name] = {
|
||||
"original_length": len(html),
|
||||
"pruned_length": sum(len(c) for c in pruned_content),
|
||||
"bm25_length": sum(len(c) for c in bm25_content),
|
||||
"pruning_time": pruning_time,
|
||||
"bm25_time": bm25_time
|
||||
}
|
||||
|
||||
return results
|
||||
|
||||
def test_markdown_generation():
|
||||
"""Test markdown generation with different configurations."""
|
||||
results = []
|
||||
|
||||
# Initialize generators with different configurations
|
||||
generators = {
|
||||
"no_filter": DefaultMarkdownGenerator(),
|
||||
"pruning": DefaultMarkdownGenerator(
|
||||
content_filter=PruningContentFilter(threshold=0.48)
|
||||
),
|
||||
"bm25": DefaultMarkdownGenerator(
|
||||
content_filter=BM25ContentFilter(
|
||||
user_query="test article content important"
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
# Test each generator with each HTML sample
|
||||
for test_name, html in TEST_HTML_SAMPLES.items():
|
||||
for gen_name, generator in generators.items():
|
||||
start_time = time.time()
|
||||
result = generator.generate_markdown(
|
||||
html,
|
||||
base_url="http://example.com",
|
||||
citations=True
|
||||
)
|
||||
|
||||
results.append({
|
||||
"test_case": test_name,
|
||||
"generator": gen_name,
|
||||
"time": time.time() - start_time,
|
||||
"raw_length": len(result.raw_markdown),
|
||||
"fit_length": len(result.fit_markdown) if result.fit_markdown else 0,
|
||||
"citations": len(result.references_markdown)
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
def main():
|
||||
"""Run all tests and print results."""
|
||||
print("Starting content filter tests...")
|
||||
filter_results = test_content_filters()
|
||||
|
||||
print("\nContent Filter Results:")
|
||||
print("-" * 50)
|
||||
for test_name, metrics in filter_results.items():
|
||||
print(f"\nTest case: {test_name}")
|
||||
print(f"Original length: {metrics['original_length']}")
|
||||
print(f"Pruned length: {metrics['pruned_length']} ({metrics['pruning_time']:.3f}s)")
|
||||
print(f"BM25 length: {metrics['bm25_length']} ({metrics['bm25_time']:.3f}s)")
|
||||
|
||||
print("\nStarting markdown generation tests...")
|
||||
markdown_results = test_markdown_generation()
|
||||
|
||||
print("\nMarkdown Generation Results:")
|
||||
print("-" * 50)
|
||||
for result in markdown_results:
|
||||
print(f"\nTest: {result['test_case']} - Generator: {result['generator']}")
|
||||
print(f"Time: {result['time']:.3f}s")
|
||||
print(f"Raw length: {result['raw_length']}")
|
||||
print(f"Fit length: {result['fit_length']}")
|
||||
print(f"Citations: {result['citations']}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,711 @@
|
||||
"""
|
||||
Comprehensive test cases for AsyncUrlSeeder with BM25 scoring functionality.
|
||||
Tests cover all features including query-based scoring, metadata extraction,
|
||||
edge cases, and integration scenarios.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import pytest
|
||||
from typing import List, Dict, Any
|
||||
from crawl4ai import AsyncUrlSeeder, SeedingConfig, AsyncLogger
|
||||
import json
|
||||
from datetime import datetime
|
||||
|
||||
# Test domain - using docs.crawl4ai.com as it has the actual documentation
|
||||
TEST_DOMAIN = "kidocode.com"
|
||||
TEST_DOMAIN = "docs.crawl4ai.com"
|
||||
TEST_DOMAIN = "www.bbc.com/sport"
|
||||
|
||||
|
||||
class TestAsyncUrlSeederBM25:
|
||||
"""Comprehensive test suite for AsyncUrlSeeder with BM25 scoring."""
|
||||
|
||||
async def create_seeder(self):
|
||||
"""Create an AsyncUrlSeeder instance for testing."""
|
||||
logger = AsyncLogger()
|
||||
return AsyncUrlSeeder(logger=logger)
|
||||
|
||||
# ============================================
|
||||
# Basic BM25 Scoring Tests
|
||||
# ============================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_basic_bm25_scoring(self, seeder):
|
||||
"""Test basic BM25 scoring with a simple query."""
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query="premier league highlights",
|
||||
scoring_method="bm25",
|
||||
max_urls=200,
|
||||
verbose=True,
|
||||
force=True # Force fresh fetch
|
||||
)
|
||||
|
||||
results = await seeder.urls(TEST_DOMAIN, config)
|
||||
|
||||
# Verify results have relevance scores
|
||||
assert all("relevance_score" in r for r in results)
|
||||
|
||||
# Verify scores are normalized between 0 and 1
|
||||
scores = [r["relevance_score"] for r in results]
|
||||
assert all(0.0 <= s <= 1.0 for s in scores)
|
||||
|
||||
# Verify results are sorted by relevance (descending)
|
||||
assert scores == sorted(scores, reverse=True)
|
||||
|
||||
# Print top 5 results for manual verification
|
||||
print("\nTop 5 results for 'web crawling tutorial':")
|
||||
for i, r in enumerate(results[:5]):
|
||||
print(f"{i+1}. Score: {r['relevance_score']:.3f} - {r['url']}")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_variations(self, seeder):
|
||||
"""Test BM25 scoring with different query variations."""
|
||||
queries = [
|
||||
"VAR controversy",
|
||||
"player ratings",
|
||||
"live score update",
|
||||
"transfer rumours",
|
||||
"post match analysis",
|
||||
"injury news"
|
||||
]
|
||||
|
||||
for query in queries:
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query=query,
|
||||
scoring_method="bm25",
|
||||
max_urls=100,
|
||||
# force=True
|
||||
)
|
||||
|
||||
results = await seeder.urls(TEST_DOMAIN, config)
|
||||
|
||||
# Verify each query produces scored results
|
||||
assert len(results) > 0
|
||||
assert all("relevance_score" in r for r in results)
|
||||
|
||||
print(f"\nTop result for '{query}':")
|
||||
if results:
|
||||
top = results[0]
|
||||
print(f" Score: {top['relevance_score']:.3f} - {top['url']}")
|
||||
|
||||
# ============================================
|
||||
# Score Threshold Tests
|
||||
# ============================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_score_threshold_filtering(self, seeder):
|
||||
"""Test filtering results by minimum relevance score."""
|
||||
thresholds = [0.1, 0.3, 0.5, 0.7]
|
||||
|
||||
for threshold in thresholds:
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query="league standings",
|
||||
score_threshold=threshold,
|
||||
scoring_method="bm25",
|
||||
max_urls=50
|
||||
)
|
||||
|
||||
results = await seeder.urls(TEST_DOMAIN, config)
|
||||
|
||||
# Verify all results meet threshold
|
||||
if results:
|
||||
assert all(r["relevance_score"] >= threshold for r in results)
|
||||
|
||||
print(f"\nThreshold {threshold}: {len(results)} URLs passed")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extreme_thresholds(self, seeder):
|
||||
"""Test edge cases with extreme threshold values."""
|
||||
# Very low threshold - should return many results
|
||||
config_low = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query="match",
|
||||
score_threshold=0.001,
|
||||
scoring_method="bm25"
|
||||
)
|
||||
results_low = await seeder.urls(TEST_DOMAIN, config_low)
|
||||
|
||||
# Very high threshold - might return few or no results
|
||||
config_high = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query="match",
|
||||
score_threshold=0.99,
|
||||
scoring_method="bm25"
|
||||
)
|
||||
results_high = await seeder.urls(TEST_DOMAIN, config_high)
|
||||
|
||||
# Low threshold should return more results than high
|
||||
assert len(results_low) >= len(results_high)
|
||||
print(f"\nLow threshold (0.001): {len(results_low)} results")
|
||||
print(f"High threshold (0.99): {len(results_high)} results")
|
||||
|
||||
# ============================================
|
||||
# Metadata Extraction Tests
|
||||
# ============================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_comprehensive_metadata_extraction(self, seeder):
|
||||
"""Test extraction of all metadata types including JSON-LD."""
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query="match report",
|
||||
scoring_method="bm25",
|
||||
max_urls=5,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
results = await seeder.urls(TEST_DOMAIN, config)
|
||||
|
||||
for result in results:
|
||||
head_data = result.get("head_data", {})
|
||||
|
||||
# Check for various metadata fields
|
||||
print(f"\nMetadata for {result['url']}:")
|
||||
print(f" Title: {head_data.get('title', 'N/A')}")
|
||||
print(f" Charset: {head_data.get('charset', 'N/A')}")
|
||||
print(f" Lang: {head_data.get('lang', 'N/A')}")
|
||||
|
||||
# Check meta tags
|
||||
meta = head_data.get("meta", {})
|
||||
if meta:
|
||||
print(" Meta tags found:")
|
||||
for key in ["description", "keywords", "author", "viewport"]:
|
||||
if key in meta:
|
||||
print(f" {key}: {meta[key][:50]}...")
|
||||
|
||||
# Check for Open Graph tags
|
||||
og_tags = {k: v for k, v in meta.items() if k.startswith("og:")}
|
||||
if og_tags:
|
||||
print(" Open Graph tags found:")
|
||||
for k, v in list(og_tags.items())[:3]:
|
||||
print(f" {k}: {v[:50]}...")
|
||||
|
||||
# Check JSON-LD
|
||||
if head_data.get("jsonld"):
|
||||
print(f" JSON-LD schemas found: {len(head_data['jsonld'])}")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_jsonld_extraction_scoring(self, seeder):
|
||||
"""Test that JSON-LD data contributes to BM25 scoring."""
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query="Premier League match report highlights",
|
||||
scoring_method="bm25",
|
||||
max_urls=20
|
||||
)
|
||||
|
||||
results = await seeder.urls(TEST_DOMAIN, config)
|
||||
|
||||
# Find results with JSON-LD data
|
||||
jsonld_results = [r for r in results if r.get("head_data", {}).get("jsonld")]
|
||||
|
||||
if jsonld_results:
|
||||
print(f"\nFound {len(jsonld_results)} URLs with JSON-LD data")
|
||||
for r in jsonld_results[:3]:
|
||||
print(f" Score: {r['relevance_score']:.3f} - {r['url']}")
|
||||
jsonld_data = r["head_data"]["jsonld"]
|
||||
print(f" JSON-LD types: {[item.get('@type', 'Unknown') for item in jsonld_data if isinstance(item, dict)]}")
|
||||
|
||||
# ============================================
|
||||
# Edge Cases and Error Handling
|
||||
# ============================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_query(self, seeder):
|
||||
"""Test behavior with empty query string."""
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query="",
|
||||
scoring_method="bm25",
|
||||
max_urls=10
|
||||
)
|
||||
|
||||
results = await seeder.urls(TEST_DOMAIN, config)
|
||||
|
||||
# Should return results but all with zero scores
|
||||
assert len(results) > 0
|
||||
assert all(r.get("relevance_score", 0) == 0 for r in results)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_query_without_extract_head(self, seeder):
|
||||
"""Test query scoring when extract_head is False."""
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=False, # This should trigger a warning
|
||||
query="Premier League match report highlights",
|
||||
scoring_method="bm25",
|
||||
max_urls=10
|
||||
)
|
||||
|
||||
results = await seeder.urls(TEST_DOMAIN, config)
|
||||
|
||||
# Results should not have relevance scores
|
||||
assert all("relevance_score" not in r for r in results)
|
||||
print("\nVerified: No scores added when extract_head=False")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_special_characters_in_query(self, seeder):
|
||||
"""Test queries with special characters and symbols."""
|
||||
special_queries = [
|
||||
"premier league + analytics",
|
||||
"injury/rehab routines",
|
||||
"AI-powered scouting",
|
||||
"match stats & xG",
|
||||
"tactical@breakdown",
|
||||
"transfer-window.yml"
|
||||
]
|
||||
|
||||
for query in special_queries:
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query=query,
|
||||
scoring_method="bm25",
|
||||
max_urls=5
|
||||
)
|
||||
|
||||
try:
|
||||
results = await seeder.urls(TEST_DOMAIN, config)
|
||||
assert isinstance(results, list)
|
||||
print(f"\n✓ Query '{query}' processed successfully")
|
||||
except Exception as e:
|
||||
pytest.fail(f"Failed on query '{query}': {str(e)}")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unicode_query(self, seeder):
|
||||
"""Test queries with Unicode characters."""
|
||||
unicode_queries = [
|
||||
"网页爬虫", # Chinese
|
||||
"веб-краулер", # Russian
|
||||
"🚀 crawl4ai", # Emoji
|
||||
"naïve implementation", # Accented characters
|
||||
]
|
||||
|
||||
for query in unicode_queries:
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query=query,
|
||||
scoring_method="bm25",
|
||||
max_urls=5
|
||||
)
|
||||
|
||||
try:
|
||||
results = await seeder.urls(TEST_DOMAIN, config)
|
||||
assert isinstance(results, list)
|
||||
print(f"\n✓ Unicode query '{query}' processed successfully")
|
||||
except Exception as e:
|
||||
print(f"\n✗ Unicode query '{query}' failed: {str(e)}")
|
||||
|
||||
# ============================================
|
||||
# Performance and Scalability Tests
|
||||
# ============================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_large_scale_scoring(self, seeder):
|
||||
"""Test BM25 scoring with many URLs."""
|
||||
config = SeedingConfig(
|
||||
source="cc+sitemap", # Use both sources for more URLs
|
||||
extract_head=True,
|
||||
query="world cup group standings",
|
||||
scoring_method="bm25",
|
||||
max_urls=100,
|
||||
concurrency=20,
|
||||
hits_per_sec=10
|
||||
)
|
||||
|
||||
start_time = asyncio.get_event_loop().time()
|
||||
results = await seeder.urls(TEST_DOMAIN, config)
|
||||
elapsed = asyncio.get_event_loop().time() - start_time
|
||||
|
||||
print(f"\nProcessed {len(results)} URLs in {elapsed:.2f} seconds")
|
||||
print(f"Average time per URL: {elapsed/len(results)*1000:.1f}ms")
|
||||
|
||||
# Verify scoring worked at scale
|
||||
assert all("relevance_score" in r for r in results)
|
||||
|
||||
# Check score distribution
|
||||
scores = [r["relevance_score"] for r in results]
|
||||
print(f"Score distribution:")
|
||||
print(f" Min: {min(scores):.3f}")
|
||||
print(f" Max: {max(scores):.3f}")
|
||||
print(f" Avg: {sum(scores)/len(scores):.3f}")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_scoring_consistency(self, seeder):
|
||||
"""Test that concurrent requests produce consistent scores."""
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query="live score update",
|
||||
scoring_method="bm25",
|
||||
max_urls=20,
|
||||
concurrency=10
|
||||
)
|
||||
|
||||
# Run the same query multiple times
|
||||
results_list = []
|
||||
for _ in range(3):
|
||||
results = await seeder.urls(TEST_DOMAIN, config)
|
||||
results_list.append(results)
|
||||
|
||||
# Compare scores across runs (they should be identical for same URLs)
|
||||
url_scores = {}
|
||||
for results in results_list:
|
||||
for r in results:
|
||||
url = r["url"]
|
||||
score = r["relevance_score"]
|
||||
if url in url_scores:
|
||||
# Scores should be very close (allowing for tiny float differences)
|
||||
assert abs(url_scores[url] - score) < 0.001
|
||||
else:
|
||||
url_scores[url] = score
|
||||
|
||||
print(f"\n✓ Consistent scores across {len(results_list)} runs")
|
||||
|
||||
# ============================================
|
||||
# Multi-Domain Tests
|
||||
# ============================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_many_urls_with_scoring(self, seeder):
|
||||
"""Test many_urls method with BM25 scoring."""
|
||||
domains = [TEST_DOMAIN, "docs.crawl4ai.com", "example.com"]
|
||||
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
# live_check=True,
|
||||
query="fixture list",
|
||||
scoring_method="bm25",
|
||||
score_threshold=0.2,
|
||||
max_urls=10,
|
||||
force=True, # Force fresh fetch
|
||||
)
|
||||
|
||||
results_dict = await seeder.many_urls(domains, config)
|
||||
|
||||
for domain, results in results_dict.items():
|
||||
print(f"\nDomain: {domain}")
|
||||
print(f" Found {len(results)} URLs above threshold")
|
||||
if results:
|
||||
top = results[0]
|
||||
print(f" Top result: {top['relevance_score']:.3f} - {top['url']}")
|
||||
|
||||
# ============================================
|
||||
# Complex Query Tests
|
||||
# ============================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_word_complex_queries(self, seeder):
|
||||
"""Test complex multi-word queries."""
|
||||
complex_queries = [
|
||||
"how to follow live match commentary",
|
||||
"extract expected goals stats from match data",
|
||||
"premier league match report analysis",
|
||||
"transfer rumours and confirmed signings tracker",
|
||||
"tactical breakdown of high press strategy"
|
||||
]
|
||||
|
||||
for query in complex_queries:
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query=query,
|
||||
scoring_method="bm25",
|
||||
max_urls=5
|
||||
)
|
||||
|
||||
results = await seeder.urls(TEST_DOMAIN, config)
|
||||
|
||||
if results:
|
||||
print(f"\nQuery: '{query}'")
|
||||
print(f"Top match: {results[0]['relevance_score']:.3f} - {results[0]['url']}")
|
||||
|
||||
# Extract matched terms from metadata
|
||||
head_data = results[0].get("head_data", {})
|
||||
title = head_data.get("title", "")
|
||||
description = head_data.get("meta", {}).get("description", "")
|
||||
|
||||
# Simple term matching for verification
|
||||
query_terms = set(query.lower().split())
|
||||
title_terms = set(title.lower().split())
|
||||
desc_terms = set(description.lower().split())
|
||||
|
||||
matched_terms = query_terms & (title_terms | desc_terms)
|
||||
if matched_terms:
|
||||
print(f"Matched terms: {', '.join(matched_terms)}")
|
||||
|
||||
# ============================================
|
||||
# Cache and Force Tests
|
||||
# ============================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scoring_with_cache(self, seeder):
|
||||
"""Test that scoring works correctly with cached results."""
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query="injury update timeline",
|
||||
scoring_method="bm25",
|
||||
max_urls=10,
|
||||
force=False # Use cache
|
||||
)
|
||||
|
||||
# First run - populate cache
|
||||
results1 = await seeder.urls(TEST_DOMAIN, config)
|
||||
|
||||
# Second run - should use cache
|
||||
results2 = await seeder.urls(TEST_DOMAIN, config)
|
||||
|
||||
# Results should be identical
|
||||
assert len(results1) == len(results2)
|
||||
for r1, r2 in zip(results1, results2):
|
||||
assert r1["url"] == r2["url"]
|
||||
assert abs(r1["relevance_score"] - r2["relevance_score"]) < 0.001
|
||||
|
||||
print("\n✓ Cache produces consistent scores")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_force_refresh_scoring(self, seeder):
|
||||
"""Test force=True bypasses cache for fresh scoring."""
|
||||
config_cached = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query="transfer window",
|
||||
scoring_method="bm25",
|
||||
max_urls=5,
|
||||
force=False
|
||||
)
|
||||
|
||||
config_forced = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query="transfer window",
|
||||
scoring_method="bm25",
|
||||
max_urls=5,
|
||||
force=True
|
||||
)
|
||||
|
||||
# Run with cache
|
||||
start1 = asyncio.get_event_loop().time()
|
||||
results1 = await seeder.urls(TEST_DOMAIN, config_cached)
|
||||
time1 = asyncio.get_event_loop().time() - start1
|
||||
|
||||
# Run with force (should be slower due to fresh fetch)
|
||||
start2 = asyncio.get_event_loop().time()
|
||||
results2 = await seeder.urls(TEST_DOMAIN, config_forced)
|
||||
time2 = asyncio.get_event_loop().time() - start2
|
||||
|
||||
print(f"\nCached run: {time1:.2f}s")
|
||||
print(f"Forced run: {time2:.2f}s")
|
||||
|
||||
# Both should produce scored results
|
||||
assert all("relevance_score" in r for r in results1)
|
||||
assert all("relevance_score" in r for r in results2)
|
||||
|
||||
# ============================================
|
||||
# Source Combination Tests
|
||||
# ============================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scoring_with_multiple_sources(self, seeder):
|
||||
"""Test BM25 scoring with combined sources (cc+sitemap)."""
|
||||
config = SeedingConfig(
|
||||
source="cc+sitemap",
|
||||
extract_head=True,
|
||||
query="match highlights video",
|
||||
scoring_method="bm25",
|
||||
score_threshold=0.3,
|
||||
max_urls=30,
|
||||
concurrency=15
|
||||
)
|
||||
|
||||
results = await seeder.urls(TEST_DOMAIN, config)
|
||||
|
||||
# Verify we got results from both sources
|
||||
print(f"\nCombined sources returned {len(results)} URLs above threshold")
|
||||
|
||||
# Check URL diversity
|
||||
unique_paths = set()
|
||||
for r in results:
|
||||
path = r["url"].replace("https://", "").replace("http://", "").split("/", 1)[-1]
|
||||
unique_paths.add(path.split("?")[0]) # Remove query params
|
||||
|
||||
print(f"Unique paths found: {len(unique_paths)}")
|
||||
|
||||
# All should be scored and above threshold
|
||||
assert all(r["relevance_score"] >= 0.3 for r in results)
|
||||
|
||||
# ============================================
|
||||
# Integration Tests
|
||||
# ============================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_full_workflow_integration(self, seeder):
|
||||
"""Test complete workflow: discover -> score -> filter -> use."""
|
||||
# Step 1: Discover and score URLs
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query="premier league opening fixtures",
|
||||
scoring_method="bm25",
|
||||
score_threshold=0.4,
|
||||
max_urls=10,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
results = await seeder.urls(TEST_DOMAIN, config)
|
||||
|
||||
print(f"\nStep 1: Found {len(results)} relevant URLs")
|
||||
|
||||
# Step 2: Analyze top results
|
||||
if results:
|
||||
top_urls = results[:3]
|
||||
print("\nStep 2: Top 3 URLs for crawling:")
|
||||
for i, r in enumerate(top_urls):
|
||||
print(f"{i+1}. Score: {r['relevance_score']:.3f}")
|
||||
print(f" URL: {r['url']}")
|
||||
print(f" Title: {r['head_data'].get('title', 'N/A')}")
|
||||
|
||||
# Check metadata quality
|
||||
meta = r['head_data'].get('meta', {})
|
||||
if 'description' in meta:
|
||||
print(f" Description: {meta['description'][:80]}...")
|
||||
|
||||
# Step 3: Verify these URLs would be good for actual crawling
|
||||
assert all(r["status"] == "valid" for r in results[:3])
|
||||
print("\nStep 3: All top URLs are valid for crawling ✓")
|
||||
|
||||
# ============================================
|
||||
# Report Generation
|
||||
# ============================================
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_generate_scoring_report(self, seeder):
|
||||
"""Generate a comprehensive report of BM25 scoring effectiveness."""
|
||||
queries = {
|
||||
"beginner": "match schedule",
|
||||
"advanced": "tactical analysis pressing",
|
||||
"api": "VAR decision explanation",
|
||||
"deployment": "fixture changes due to weather",
|
||||
"extraction": "expected goals statistics"
|
||||
}
|
||||
|
||||
report = {
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"domain": TEST_DOMAIN,
|
||||
"results": {}
|
||||
}
|
||||
|
||||
for category, query in queries.items():
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query=query,
|
||||
scoring_method="bm25",
|
||||
max_urls=10
|
||||
)
|
||||
|
||||
results = await seeder.urls(TEST_DOMAIN, config)
|
||||
|
||||
report["results"][category] = {
|
||||
"query": query,
|
||||
"total_results": len(results),
|
||||
"top_results": [
|
||||
{
|
||||
"url": r["url"],
|
||||
"score": r["relevance_score"],
|
||||
"title": r["head_data"].get("title", "")
|
||||
}
|
||||
for r in results[:3]
|
||||
],
|
||||
"score_distribution": {
|
||||
"min": min(r["relevance_score"] for r in results) if results else 0,
|
||||
"max": max(r["relevance_score"] for r in results) if results else 0,
|
||||
"avg": sum(r["relevance_score"] for r in results) / len(results) if results else 0
|
||||
}
|
||||
}
|
||||
|
||||
# Print report
|
||||
print("\n" + "="*60)
|
||||
print("BM25 SCORING EFFECTIVENESS REPORT")
|
||||
print("="*60)
|
||||
print(f"Domain: {report['domain']}")
|
||||
print(f"Timestamp: {report['timestamp']}")
|
||||
print("\nResults by Category:")
|
||||
|
||||
for category, data in report["results"].items():
|
||||
print(f"\n{category.upper()}: '{data['query']}'")
|
||||
print(f" Total results: {data['total_results']}")
|
||||
print(f" Score range: {data['score_distribution']['min']:.3f} - {data['score_distribution']['max']:.3f}")
|
||||
print(f" Average score: {data['score_distribution']['avg']:.3f}")
|
||||
print(" Top matches:")
|
||||
for i, result in enumerate(data['top_results']):
|
||||
print(f" {i+1}. [{result['score']:.3f}] {result['title']}")
|
||||
|
||||
|
||||
# ============================================
|
||||
# Standalone test runner
|
||||
# ============================================
|
||||
|
||||
async def run_all_tests():
|
||||
"""Run all tests standalone (without pytest)."""
|
||||
print("Running AsyncUrlSeeder BM25 Tests...")
|
||||
print("="*60)
|
||||
|
||||
test_instance = TestAsyncUrlSeederBM25()
|
||||
seeder = await test_instance.create_seeder()
|
||||
|
||||
# Run each test method
|
||||
test_methods = [
|
||||
# test_instance.test_basic_bm25_scoring,
|
||||
# test_instance.test_query_variations,
|
||||
# test_instance.test_score_threshold_filtering,
|
||||
# test_instance.test_extreme_thresholds,
|
||||
# test_instance.test_comprehensive_metadata_extraction,
|
||||
# test_instance.test_jsonld_extraction_scoring,
|
||||
# test_instance.test_empty_query,
|
||||
# test_instance.test_query_without_extract_head,
|
||||
# test_instance.test_special_characters_in_query,
|
||||
# test_instance.test_unicode_query,
|
||||
# test_instance.test_large_scale_scoring,
|
||||
# test_instance.test_concurrent_scoring_consistency,
|
||||
# test_instance.test_many_urls_with_scoring,
|
||||
test_instance.test_multi_word_complex_queries,
|
||||
test_instance.test_scoring_with_cache,
|
||||
test_instance.test_force_refresh_scoring,
|
||||
test_instance.test_scoring_with_multiple_sources,
|
||||
test_instance.test_full_workflow_integration,
|
||||
test_instance.test_generate_scoring_report
|
||||
]
|
||||
|
||||
for test_method in test_methods:
|
||||
try:
|
||||
print(f"\nRunning {test_method.__name__}...")
|
||||
await test_method(seeder)
|
||||
print(f"✓ {test_method.__name__} passed")
|
||||
except Exception as e:
|
||||
import traceback
|
||||
print(f"✗ {test_method.__name__} failed: {str(e)}")
|
||||
print(f" Error type: {type(e).__name__}")
|
||||
traceback.print_exc()
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("Test suite completed!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Run tests directly
|
||||
asyncio.run(run_all_tests())
|
||||
@@ -0,0 +1,228 @@
|
||||
import asyncio
|
||||
import pytest
|
||||
from typing import List
|
||||
from crawl4ai import (
|
||||
AsyncWebCrawler,
|
||||
BrowserConfig,
|
||||
CrawlerRunConfig,
|
||||
MemoryAdaptiveDispatcher,
|
||||
RateLimiter,
|
||||
CacheMode
|
||||
)
|
||||
from crawl4ai.extraction_strategy import ExtractionStrategy
|
||||
|
||||
class MockExtractionStrategy(ExtractionStrategy):
|
||||
"""Mock extraction strategy for testing URL parameter handling"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.run_calls = []
|
||||
|
||||
def extract(self, url: str, html: str, *args, **kwargs):
|
||||
return [{"test": "data"}]
|
||||
|
||||
def run(self, url: str, sections: List[str], *args, **kwargs):
|
||||
self.run_calls.append(url)
|
||||
return super().run(url, sections, *args, **kwargs)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("viewport", [
|
||||
(800, 600),
|
||||
(1024, 768),
|
||||
(1920, 1080)
|
||||
])
|
||||
async def test_viewport_config(viewport):
|
||||
"""Test different viewport configurations"""
|
||||
width, height = viewport
|
||||
browser_config = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True,
|
||||
viewport_width=width,
|
||||
viewport_height=height
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://example.com",
|
||||
config=CrawlerRunConfig(
|
||||
# cache_mode=CacheMode.BYPASS,
|
||||
page_timeout=30000 # 30 seconds
|
||||
)
|
||||
)
|
||||
assert result.success
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_memory_management():
|
||||
"""Test memory-adaptive dispatching"""
|
||||
browser_config = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True,
|
||||
viewport_width=1024,
|
||||
viewport_height=768
|
||||
)
|
||||
|
||||
dispatcher = MemoryAdaptiveDispatcher(
|
||||
memory_threshold_percent=70.0,
|
||||
check_interval=1.0,
|
||||
max_session_permit=5
|
||||
)
|
||||
|
||||
urls = ["https://example.com"] * 3 # Test with multiple identical URLs
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
results = await crawler.arun_many(
|
||||
urls=urls,
|
||||
config=CrawlerRunConfig(page_timeout=30000),
|
||||
dispatcher=dispatcher
|
||||
)
|
||||
assert len(results) == len(urls)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rate_limiting():
|
||||
"""Test rate limiting functionality"""
|
||||
browser_config = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True
|
||||
)
|
||||
|
||||
dispatcher = MemoryAdaptiveDispatcher(
|
||||
rate_limiter=RateLimiter(
|
||||
base_delay=(1.0, 2.0),
|
||||
max_delay=5.0,
|
||||
max_retries=2
|
||||
),
|
||||
memory_threshold_percent=70.0
|
||||
)
|
||||
|
||||
urls = [
|
||||
"https://example.com",
|
||||
"https://example.org",
|
||||
"https://example.net"
|
||||
]
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
results = await crawler.arun_many(
|
||||
urls=urls,
|
||||
config=CrawlerRunConfig(page_timeout=30000),
|
||||
dispatcher=dispatcher
|
||||
)
|
||||
assert len(results) == len(urls)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_javascript_execution():
|
||||
"""Test JavaScript execution capabilities"""
|
||||
browser_config = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True,
|
||||
java_script_enabled=True
|
||||
)
|
||||
|
||||
js_code = """
|
||||
document.body.style.backgroundColor = 'red';
|
||||
return document.body.style.backgroundColor;
|
||||
"""
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://example.com",
|
||||
config=CrawlerRunConfig(
|
||||
js_code=js_code,
|
||||
page_timeout=30000
|
||||
)
|
||||
)
|
||||
assert result.success
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("error_url", [
|
||||
"https://invalid.domain.test",
|
||||
"https://httpbin.org/status/404",
|
||||
"https://httpbin.org/status/503",
|
||||
"https://httpbin.org/status/403"
|
||||
])
|
||||
async def test_error_handling(error_url):
|
||||
"""Test error handling for various failure scenarios"""
|
||||
browser_config = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(
|
||||
url=error_url,
|
||||
config=CrawlerRunConfig(
|
||||
page_timeout=10000, # Short timeout for error cases
|
||||
cache_mode=CacheMode.BYPASS
|
||||
)
|
||||
)
|
||||
assert not result.success
|
||||
assert result.error_message is not None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extraction_strategy_run_with_regular_url():
|
||||
"""
|
||||
Regression test for extraction_strategy.run URL parameter handling with regular URLs.
|
||||
|
||||
This test verifies that when is_raw_html=False (regular URL),
|
||||
extraction_strategy.run is called with the actual URL.
|
||||
"""
|
||||
browser_config = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
mock_strategy = MockExtractionStrategy()
|
||||
|
||||
# Test regular URL (is_raw_html=False)
|
||||
regular_url = "https://example.com"
|
||||
result = await crawler.arun(
|
||||
url=regular_url,
|
||||
config=CrawlerRunConfig(
|
||||
page_timeout=30000,
|
||||
extraction_strategy=mock_strategy,
|
||||
cache_mode=CacheMode.BYPASS
|
||||
)
|
||||
)
|
||||
|
||||
assert result.success
|
||||
assert len(mock_strategy.run_calls) == 1
|
||||
assert mock_strategy.run_calls[0] == regular_url, f"Expected '{regular_url}', got '{mock_strategy.run_calls[0]}'"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_extraction_strategy_run_with_raw_html():
|
||||
"""
|
||||
Regression test for extraction_strategy.run URL parameter handling with raw HTML.
|
||||
|
||||
This test verifies that when is_raw_html=True (URL starts with "raw:"),
|
||||
extraction_strategy.run is called with "Raw HTML" instead of the actual URL.
|
||||
"""
|
||||
browser_config = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
mock_strategy = MockExtractionStrategy()
|
||||
|
||||
# Test raw HTML URL (is_raw_html=True automatically set)
|
||||
raw_html_url = "raw:<html><body><h1>Test HTML</h1><p>This is a test.</p></body></html>"
|
||||
result = await crawler.arun(
|
||||
url=raw_html_url,
|
||||
config=CrawlerRunConfig(
|
||||
page_timeout=30000,
|
||||
extraction_strategy=mock_strategy,
|
||||
cache_mode=CacheMode.BYPASS
|
||||
)
|
||||
)
|
||||
|
||||
assert result.success
|
||||
assert len(mock_strategy.run_calls) == 1
|
||||
assert mock_strategy.run_calls[0] == "Raw HTML", f"Expected 'Raw HTML', got '{mock_strategy.run_calls[0]}'"
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_viewport_config((1024, 768)))
|
||||
asyncio.run(test_memory_management())
|
||||
asyncio.run(test_rate_limiting())
|
||||
asyncio.run(test_javascript_execution())
|
||||
asyncio.run(test_extraction_strategy_run_with_regular_url())
|
||||
asyncio.run(test_extraction_strategy_run_with_raw_html())
|
||||
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Simple test to verify BestFirstCrawlingStrategy fixes.
|
||||
This test crawls a real website and shows that:
|
||||
1. Higher-scoring pages are crawled first (priority queue fix)
|
||||
2. Links are scored before truncation (link discovery fix)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
|
||||
from crawl4ai.deep_crawling import BestFirstCrawlingStrategy
|
||||
from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer
|
||||
|
||||
async def test_best_first_strategy():
|
||||
"""Test BestFirstCrawlingStrategy with keyword scoring"""
|
||||
|
||||
print("=" * 70)
|
||||
print("Testing BestFirstCrawlingStrategy with Real URL")
|
||||
print("=" * 70)
|
||||
print("\nThis test will:")
|
||||
print("1. Crawl Python.org documentation")
|
||||
print("2. Score pages based on keywords: 'tutorial', 'guide', 'reference'")
|
||||
print("3. Show that higher-scoring pages are crawled first")
|
||||
print("-" * 70)
|
||||
|
||||
# Create a keyword scorer that prioritizes tutorial/guide pages
|
||||
scorer = KeywordRelevanceScorer(
|
||||
keywords=["tutorial", "guide", "reference", "documentation"],
|
||||
weight=1.0,
|
||||
case_sensitive=False
|
||||
)
|
||||
|
||||
# Create the strategy with scoring
|
||||
strategy = BestFirstCrawlingStrategy(
|
||||
max_depth=2, # Crawl 2 levels deep
|
||||
max_pages=10, # Limit to 10 pages total
|
||||
url_scorer=scorer, # Use keyword scoring
|
||||
include_external=False # Only internal links
|
||||
)
|
||||
|
||||
# Configure browser and crawler
|
||||
browser_config = BrowserConfig(
|
||||
headless=True, # Run in background
|
||||
verbose=False # Reduce output noise
|
||||
)
|
||||
|
||||
crawler_config = CrawlerRunConfig(
|
||||
deep_crawl_strategy=strategy,
|
||||
verbose=False
|
||||
)
|
||||
|
||||
print("\nStarting crawl of https://docs.python.org/3/")
|
||||
print("Looking for pages with keywords: tutorial, guide, reference, documentation")
|
||||
print("-" * 70)
|
||||
|
||||
crawled_urls = []
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
# Crawl and collect results
|
||||
results = await crawler.arun(
|
||||
url="https://docs.python.org/3/",
|
||||
config=crawler_config
|
||||
)
|
||||
|
||||
# Process results
|
||||
if isinstance(results, list):
|
||||
for result in results:
|
||||
score = result.metadata.get('score', 0) if result.metadata else 0
|
||||
depth = result.metadata.get('depth', 0) if result.metadata else 0
|
||||
crawled_urls.append({
|
||||
'url': result.url,
|
||||
'score': score,
|
||||
'depth': depth,
|
||||
'success': result.success
|
||||
})
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("CRAWL RESULTS (in order of crawling)")
|
||||
print("=" * 70)
|
||||
|
||||
for i, item in enumerate(crawled_urls, 1):
|
||||
status = "✓" if item['success'] else "✗"
|
||||
# Highlight high-scoring pages
|
||||
if item['score'] > 0.5:
|
||||
print(f"{i:2}. [{status}] Score: {item['score']:.2f} | Depth: {item['depth']} | {item['url']}")
|
||||
print(f" ^ HIGH SCORE - Contains keywords!")
|
||||
else:
|
||||
print(f"{i:2}. [{status}] Score: {item['score']:.2f} | Depth: {item['depth']} | {item['url']}")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("ANALYSIS")
|
||||
print("=" * 70)
|
||||
|
||||
# Check if higher scores appear early in the crawl
|
||||
scores = [item['score'] for item in crawled_urls[1:]] # Skip initial URL
|
||||
high_score_indices = [i for i, s in enumerate(scores) if s > 0.3]
|
||||
|
||||
if high_score_indices and high_score_indices[0] < len(scores) / 2:
|
||||
print("✅ SUCCESS: Higher-scoring pages (with keywords) were crawled early!")
|
||||
print(" This confirms the priority queue fix is working.")
|
||||
else:
|
||||
print("⚠️ Check the crawl order above - higher scores should appear early")
|
||||
|
||||
# Show score distribution
|
||||
print(f"\nScore Statistics:")
|
||||
print(f" - Total pages crawled: {len(crawled_urls)}")
|
||||
print(f" - Average score: {sum(item['score'] for item in crawled_urls) / len(crawled_urls):.2f}")
|
||||
print(f" - Max score: {max(item['score'] for item in crawled_urls):.2f}")
|
||||
print(f" - Pages with keywords: {sum(1 for item in crawled_urls if item['score'] > 0.3)}")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("TEST COMPLETE")
|
||||
print("=" * 70)
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n🔍 BestFirstCrawlingStrategy Simple Test\n")
|
||||
asyncio.run(test_best_first_strategy())
|
||||
@@ -0,0 +1,85 @@
|
||||
import asyncio
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
|
||||
from playwright.async_api import Page, BrowserContext
|
||||
|
||||
async def test_reuse_context_by_config():
|
||||
# We will store each context ID in these maps to confirm reuse
|
||||
context_ids_for_A = []
|
||||
context_ids_for_B = []
|
||||
|
||||
# Create a small hook to track context creation
|
||||
async def on_page_context_created(page: Page, context: BrowserContext, config: CrawlerRunConfig, **kwargs):
|
||||
c_id = id(context)
|
||||
print(f"[HOOK] on_page_context_created - Context ID: {c_id}")
|
||||
# Distinguish which config we used by checking a custom hook param
|
||||
config_label = config.shared_data.get("config_label", "unknown")
|
||||
if config_label == "A":
|
||||
context_ids_for_A.append(c_id)
|
||||
elif config_label == "B":
|
||||
context_ids_for_B.append(c_id)
|
||||
return page
|
||||
|
||||
# Browser config - Headless, verbose so we see logs
|
||||
browser_config = BrowserConfig(headless=True, verbose=True)
|
||||
|
||||
# Two crawler run configs that differ (for example, text_mode):
|
||||
configA = CrawlerRunConfig(
|
||||
only_text=True,
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
wait_until="domcontentloaded",
|
||||
shared_data = {
|
||||
"config_label" : "A"
|
||||
}
|
||||
)
|
||||
configB = CrawlerRunConfig(
|
||||
only_text=False,
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
wait_until="domcontentloaded",
|
||||
shared_data = {
|
||||
"config_label" : "B"
|
||||
}
|
||||
)
|
||||
|
||||
# Create the crawler
|
||||
crawler = AsyncWebCrawler(config=browser_config)
|
||||
|
||||
# Attach our custom hook
|
||||
# Note: "on_page_context_created" will be called each time a new context+page is generated
|
||||
crawler.crawler_strategy.set_hook("on_page_context_created", on_page_context_created)
|
||||
|
||||
# Start the crawler (launches the browser)
|
||||
await crawler.start()
|
||||
|
||||
# For demonstration, we’ll crawl a benign site multiple times with each config
|
||||
test_url = "https://example.com"
|
||||
print("\n--- Crawling with config A (text_mode=True) ---")
|
||||
for _ in range(2):
|
||||
# Pass an extra kwarg to the hook so we know which config is being used
|
||||
await crawler.arun(test_url, config=configA)
|
||||
|
||||
print("\n--- Crawling with config B (text_mode=False) ---")
|
||||
for _ in range(2):
|
||||
await crawler.arun(test_url, config=configB)
|
||||
|
||||
# Close the crawler (shuts down the browser, closes contexts)
|
||||
await crawler.close()
|
||||
|
||||
# Validate and show the results
|
||||
print("\n=== RESULTS ===")
|
||||
print(f"Config A context IDs: {context_ids_for_A}")
|
||||
print(f"Config B context IDs: {context_ids_for_B}")
|
||||
if len(set(context_ids_for_A)) == 1:
|
||||
print("✅ All config A crawls used the SAME BrowserContext.")
|
||||
else:
|
||||
print("❌ Config A crawls created multiple contexts unexpectedly.")
|
||||
if len(set(context_ids_for_B)) == 1:
|
||||
print("✅ All config B crawls used the SAME BrowserContext.")
|
||||
else:
|
||||
print("❌ Config B crawls created multiple contexts unexpectedly.")
|
||||
if set(context_ids_for_A).isdisjoint(context_ids_for_B):
|
||||
print("✅ Config A context is different from Config B context.")
|
||||
else:
|
||||
print("❌ A and B ended up sharing the same context somehow!")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_reuse_context_by_config())
|
||||
@@ -0,0 +1,106 @@
|
||||
"""
|
||||
Tests for the content_source parameter in markdown generation.
|
||||
"""
|
||||
import unittest
|
||||
import asyncio
|
||||
from unittest.mock import patch, MagicMock
|
||||
|
||||
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator, MarkdownGenerationStrategy
|
||||
from crawl4ai.async_webcrawler import AsyncWebCrawler
|
||||
from crawl4ai.async_configs import CrawlerRunConfig
|
||||
from crawl4ai.models import MarkdownGenerationResult
|
||||
|
||||
HTML_SAMPLE = """
|
||||
<html>
|
||||
<head><title>Test Page</title></head>
|
||||
<body>
|
||||
<h1>Test Content</h1>
|
||||
<p>This is a test paragraph.</p>
|
||||
<div class="container">
|
||||
<p>This is content within a container.</p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
class TestContentSourceParameter(unittest.TestCase):
|
||||
"""Test cases for the content_source parameter in markdown generation."""
|
||||
|
||||
def setUp(self):
|
||||
"""Set up test fixtures."""
|
||||
self.loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(self.loop)
|
||||
|
||||
def tearDown(self):
|
||||
"""Tear down test fixtures."""
|
||||
self.loop.close()
|
||||
|
||||
def test_default_content_source(self):
|
||||
"""Test that the default content_source is 'cleaned_html'."""
|
||||
# Can't directly instantiate abstract class, so just test DefaultMarkdownGenerator
|
||||
generator = DefaultMarkdownGenerator()
|
||||
self.assertEqual(generator.content_source, "cleaned_html")
|
||||
|
||||
def test_custom_content_source(self):
|
||||
"""Test that content_source can be customized."""
|
||||
generator = DefaultMarkdownGenerator(content_source="fit_html")
|
||||
self.assertEqual(generator.content_source, "fit_html")
|
||||
|
||||
@patch('crawl4ai.markdown_generation_strategy.CustomHTML2Text')
|
||||
def test_html_processing_using_input_html(self, mock_html2text):
|
||||
"""Test that generate_markdown uses input_html parameter."""
|
||||
# Setup mock
|
||||
mock_instance = MagicMock()
|
||||
mock_instance.handle.return_value = "# Test Content\n\nThis is a test paragraph."
|
||||
mock_html2text.return_value = mock_instance
|
||||
|
||||
# Create generator and call generate_markdown
|
||||
generator = DefaultMarkdownGenerator()
|
||||
result = generator.generate_markdown(input_html="<h1>Test Content</h1><p>This is a test paragraph.</p>")
|
||||
|
||||
# Verify input_html was passed to HTML2Text handler
|
||||
mock_instance.handle.assert_called_once()
|
||||
# Get the first positional argument
|
||||
args, _ = mock_instance.handle.call_args
|
||||
self.assertEqual(args[0], "<h1>Test Content</h1><p>This is a test paragraph.</p>")
|
||||
|
||||
# Check result
|
||||
self.assertIsInstance(result, MarkdownGenerationResult)
|
||||
self.assertEqual(result.raw_markdown, "# Test Content\n\nThis is a test paragraph.")
|
||||
|
||||
def test_html_source_selection_logic(self):
|
||||
"""Test that the HTML source selection logic works correctly."""
|
||||
# We'll test the dispatch pattern directly to avoid async complexities
|
||||
|
||||
# Create test data
|
||||
raw_html = "<html><body><h1>Raw HTML</h1></body></html>"
|
||||
cleaned_html = "<html><body><h1>Cleaned HTML</h1></body></html>"
|
||||
fit_html = "<html><body><h1>Preprocessed HTML</h1></body></html>"
|
||||
|
||||
# Test the dispatch pattern
|
||||
html_source_selector = {
|
||||
"raw_html": lambda: raw_html,
|
||||
"cleaned_html": lambda: cleaned_html,
|
||||
"fit_html": lambda: fit_html,
|
||||
}
|
||||
|
||||
# Test Case 1: content_source="cleaned_html"
|
||||
source_lambda = html_source_selector.get("cleaned_html")
|
||||
self.assertEqual(source_lambda(), cleaned_html)
|
||||
|
||||
# Test Case 2: content_source="raw_html"
|
||||
source_lambda = html_source_selector.get("raw_html")
|
||||
self.assertEqual(source_lambda(), raw_html)
|
||||
|
||||
# Test Case 3: content_source="fit_html"
|
||||
source_lambda = html_source_selector.get("fit_html")
|
||||
self.assertEqual(source_lambda(), fit_html)
|
||||
|
||||
# Test Case 4: Invalid content_source falls back to cleaned_html
|
||||
source_lambda = html_source_selector.get("invalid_source", lambda: cleaned_html)
|
||||
self.assertEqual(source_lambda(), cleaned_html)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,17 @@
|
||||
|
||||
# example_usageexample_usageexample_usage# example_usage.py
|
||||
import asyncio
|
||||
from crawl4ai.crawlers import get_crawler
|
||||
|
||||
async def main():
|
||||
# Get the registered crawler
|
||||
example_crawler = get_crawler("example_site.content")
|
||||
|
||||
# Crawl example.com
|
||||
result = await example_crawler(url="https://example.com")
|
||||
|
||||
print(result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,46 @@
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
|
||||
from crawl4ai import CrawlerRunConfig, AsyncWebCrawler, CacheMode
|
||||
from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
|
||||
from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
|
||||
# from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, BestFirstCrawlingStrategy
|
||||
|
||||
|
||||
async def main():
|
||||
"""Example deep crawl of documentation site."""
|
||||
config = CrawlerRunConfig(
|
||||
deep_crawl_strategy = BFSDeepCrawlStrategy(
|
||||
max_depth=2,
|
||||
include_external=False
|
||||
),
|
||||
stream=False,
|
||||
verbose=True,
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
scraping_strategy=LXMLWebScrapingStrategy()
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
start_time = time.perf_counter()
|
||||
print("\nStarting deep crawl in batch mode:")
|
||||
results = await crawler.arun(
|
||||
url="https://docs.crawl4ai.com",
|
||||
config=config
|
||||
)
|
||||
print(f"Crawled {len(results)} pages")
|
||||
print(f"Example page: {results[0].url}")
|
||||
print(f"Duration: {time.perf_counter() - start_time:.2f} seconds\n")
|
||||
|
||||
print("Starting deep crawl in streaming mode:")
|
||||
config.stream = True
|
||||
start_time = time.perf_counter()
|
||||
async for result in await crawler.arun(
|
||||
url="https://docs.crawl4ai.com",
|
||||
config=config
|
||||
):
|
||||
print(f"→ {result.url} (Depth: {result.metadata.get('depth', 0)})")
|
||||
print(f"Duration: {time.perf_counter() - start_time:.2f} seconds")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,279 @@
|
||||
from crawl4ai.deep_crawling.filters import ContentRelevanceFilter, URLPatternFilter, DomainFilter, ContentTypeFilter, SEOFilter
|
||||
async def test_pattern_filter():
|
||||
# Test cases as list of tuples instead of dict for multiple patterns
|
||||
test_cases = [
|
||||
# Simple suffix patterns (*.html)
|
||||
("*.html", {
|
||||
"https://example.com/page.html": True,
|
||||
"https://example.com/path/doc.html": True,
|
||||
"https://example.com/page.htm": False,
|
||||
"https://example.com/page.html?param=1": True,
|
||||
}),
|
||||
|
||||
# Path prefix patterns (/foo/*)
|
||||
("*/article/*", {
|
||||
"https://example.com/article/123": True,
|
||||
"https://example.com/blog/article/456": True,
|
||||
"https://example.com/articles/789": False,
|
||||
"https://example.com/article": False,
|
||||
}),
|
||||
|
||||
# Complex patterns
|
||||
("blog-*-[0-9]", {
|
||||
"https://example.com/blog-post-1": True,
|
||||
"https://example.com/blog-test-9": True,
|
||||
"https://example.com/blog-post": False,
|
||||
"https://example.com/blog-post-x": False,
|
||||
}),
|
||||
|
||||
# Multiple patterns case
|
||||
(["*.pdf", "*/download/*"], {
|
||||
"https://example.com/doc.pdf": True,
|
||||
"https://example.com/download/file.txt": True,
|
||||
"https://example.com/path/download/doc": True,
|
||||
"https://example.com/uploads/file.txt": False,
|
||||
}),
|
||||
|
||||
# Edge cases
|
||||
("*", {
|
||||
"https://example.com": True,
|
||||
"": True,
|
||||
"http://test.com/path": True,
|
||||
}),
|
||||
|
||||
# Complex regex
|
||||
(r"^https?://.*\.example\.com/\d+", {
|
||||
"https://sub.example.com/123": True,
|
||||
"http://test.example.com/456": True,
|
||||
"https://example.com/789": False,
|
||||
"https://sub.example.com/abc": False,
|
||||
})
|
||||
]
|
||||
|
||||
def run_accuracy_test():
|
||||
print("\nAccuracy Tests:")
|
||||
print("-" * 50)
|
||||
|
||||
all_passed = True
|
||||
for patterns, test_urls in test_cases:
|
||||
filter_obj = URLPatternFilter(patterns)
|
||||
|
||||
for url, expected in test_urls.items():
|
||||
result = filter_obj.apply(url)
|
||||
if result != expected:
|
||||
print(f"❌ Failed: Pattern '{patterns}' with URL '{url}'")
|
||||
print(f" Expected: {expected}, Got: {result}")
|
||||
all_passed = False
|
||||
else:
|
||||
print(f"✅ Passed: Pattern '{patterns}' with URL '{url}'")
|
||||
|
||||
return all_passed
|
||||
|
||||
# Run tests
|
||||
print("Running Pattern Filter Tests...")
|
||||
accuracy_passed = run_accuracy_test()
|
||||
|
||||
if accuracy_passed:
|
||||
print("\n✨ All accuracy tests passed!")
|
||||
|
||||
else:
|
||||
print("\n❌ Some accuracy tests failed!")
|
||||
|
||||
async def test_domain_filter():
|
||||
from itertools import chain
|
||||
|
||||
# Test cases
|
||||
test_cases = [
|
||||
# Allowed domains
|
||||
({"allowed": "example.com"}, {
|
||||
"https://example.com/page": True,
|
||||
"http://example.com": True,
|
||||
"https://sub.example.com": False,
|
||||
"https://other.com": False,
|
||||
}),
|
||||
|
||||
({"allowed": ["example.com", "test.com"]}, {
|
||||
"https://example.com/page": True,
|
||||
"https://test.com/home": True,
|
||||
"https://other.com": False,
|
||||
}),
|
||||
|
||||
# Blocked domains
|
||||
({"blocked": "malicious.com"}, {
|
||||
"https://malicious.com": False,
|
||||
"https://safe.com": True,
|
||||
"http://malicious.com/login": False,
|
||||
}),
|
||||
|
||||
({"blocked": ["spam.com", "ads.com"]}, {
|
||||
"https://spam.com": False,
|
||||
"https://ads.com/banner": False,
|
||||
"https://example.com": True,
|
||||
}),
|
||||
|
||||
# Allowed and Blocked combination
|
||||
({"allowed": "example.com", "blocked": "sub.example.com"}, {
|
||||
"https://example.com": True,
|
||||
"https://sub.example.com": False,
|
||||
"https://other.com": False,
|
||||
}),
|
||||
]
|
||||
|
||||
def run_accuracy_test():
|
||||
print("\nAccuracy Tests:")
|
||||
print("-" * 50)
|
||||
|
||||
all_passed = True
|
||||
for params, test_urls in test_cases:
|
||||
filter_obj = DomainFilter(
|
||||
allowed_domains=params.get("allowed"),
|
||||
blocked_domains=params.get("blocked"),
|
||||
)
|
||||
|
||||
for url, expected in test_urls.items():
|
||||
result = filter_obj.apply(url)
|
||||
if result != expected:
|
||||
print(f"\u274C Failed: Params {params} with URL '{url}'")
|
||||
print(f" Expected: {expected}, Got: {result}")
|
||||
all_passed = False
|
||||
else:
|
||||
print(f"\u2705 Passed: Params {params} with URL '{url}'")
|
||||
|
||||
return all_passed
|
||||
|
||||
# Run tests
|
||||
print("Running Domain Filter Tests...")
|
||||
accuracy_passed = run_accuracy_test()
|
||||
|
||||
if accuracy_passed:
|
||||
print("\n\u2728 All accuracy tests passed!")
|
||||
else:
|
||||
print("\n\u274C Some accuracy tests failed!")
|
||||
|
||||
async def test_content_relevance_filter():
|
||||
relevance_filter = ContentRelevanceFilter(
|
||||
query="What was the cause of american civil war?",
|
||||
threshold=1
|
||||
)
|
||||
|
||||
test_cases = {
|
||||
"https://en.wikipedia.org/wiki/Cricket": False,
|
||||
"https://en.wikipedia.org/wiki/American_Civil_War": True,
|
||||
}
|
||||
|
||||
print("\nRunning Content Relevance Filter Tests...")
|
||||
print("-" * 50)
|
||||
|
||||
all_passed = True
|
||||
for url, expected in test_cases.items():
|
||||
result = await relevance_filter.apply(url)
|
||||
if result != expected:
|
||||
print(f"\u274C Failed: URL '{url}'")
|
||||
print(f" Expected: {expected}, Got: {result}")
|
||||
all_passed = False
|
||||
else:
|
||||
print(f"\u2705 Passed: URL '{url}'")
|
||||
|
||||
if all_passed:
|
||||
print("\n\u2728 All content relevance tests passed!")
|
||||
else:
|
||||
print("\n\u274C Some content relevance tests failed!")
|
||||
|
||||
async def test_content_type_filter():
|
||||
from itertools import chain
|
||||
|
||||
# Test cases
|
||||
test_cases = [
|
||||
# Allowed single type
|
||||
({"allowed": "image/png"}, {
|
||||
"https://example.com/image.png": True,
|
||||
"https://example.com/photo.jpg": False,
|
||||
"https://example.com/document.pdf": False,
|
||||
}),
|
||||
|
||||
# Multiple allowed types
|
||||
({"allowed": ["image/jpeg", "application/pdf"]}, {
|
||||
"https://example.com/photo.jpg": True,
|
||||
"https://example.com/document.pdf": True,
|
||||
"https://example.com/script.js": False,
|
||||
}),
|
||||
|
||||
# No extension should be allowed
|
||||
({"allowed": "application/json"}, {
|
||||
"https://example.com/api/data": True,
|
||||
"https://example.com/data.json": True,
|
||||
"https://example.com/page.html": False,
|
||||
}),
|
||||
|
||||
# Unknown extensions should not be allowed
|
||||
({"allowed": "application/octet-stream"}, {
|
||||
"https://example.com/file.unknown": True,
|
||||
"https://example.com/archive.zip": False,
|
||||
"https://example.com/software.exe": False,
|
||||
}),
|
||||
]
|
||||
|
||||
def run_accuracy_test():
|
||||
print("\nAccuracy Tests:")
|
||||
print("-" * 50)
|
||||
|
||||
all_passed = True
|
||||
for params, test_urls in test_cases:
|
||||
filter_obj = ContentTypeFilter(
|
||||
allowed_types=params.get("allowed"),
|
||||
)
|
||||
|
||||
for url, expected in test_urls.items():
|
||||
result = filter_obj.apply(url)
|
||||
if result != expected:
|
||||
print(f"\u274C Failed: Params {params} with URL '{url}'")
|
||||
print(f" Expected: {expected}, Got: {result}")
|
||||
all_passed = False
|
||||
else:
|
||||
print(f"\u2705 Passed: Params {params} with URL '{url}'")
|
||||
|
||||
return all_passed
|
||||
|
||||
# Run tests
|
||||
print("Running Content Type Filter Tests...")
|
||||
accuracy_passed = run_accuracy_test()
|
||||
|
||||
if accuracy_passed:
|
||||
print("\n\u2728 All accuracy tests passed!")
|
||||
else:
|
||||
print("\n\u274C Some accuracy tests failed!")
|
||||
|
||||
async def test_seo_filter():
|
||||
seo_filter = SEOFilter(threshold=0.5, keywords=["SEO", "search engines", "Optimization"])
|
||||
|
||||
test_cases = {
|
||||
"https://en.wikipedia.org/wiki/Search_engine_optimization": True,
|
||||
"https://en.wikipedia.org/wiki/Randomness": False,
|
||||
}
|
||||
|
||||
print("\nRunning SEO Filter Tests...")
|
||||
print("-" * 50)
|
||||
|
||||
all_passed = True
|
||||
for url, expected in test_cases.items():
|
||||
result = await seo_filter.apply(url)
|
||||
if result != expected:
|
||||
print(f"\u274C Failed: URL '{url}'")
|
||||
print(f" Expected: {expected}, Got: {result}")
|
||||
all_passed = False
|
||||
else:
|
||||
print(f"\u2705 Passed: URL '{url}'")
|
||||
|
||||
if all_passed:
|
||||
print("\n\u2728 All SEO filter tests passed!")
|
||||
else:
|
||||
print("\n\u274C Some SEO filter tests failed!")
|
||||
|
||||
import asyncio
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_pattern_filter())
|
||||
asyncio.run(test_domain_filter())
|
||||
asyncio.run(test_content_type_filter())
|
||||
asyncio.run(test_content_relevance_filter())
|
||||
asyncio.run(test_seo_filter())
|
||||
@@ -0,0 +1,179 @@
|
||||
from crawl4ai.deep_crawling.scorers import CompositeScorer, ContentTypeScorer, DomainAuthorityScorer, FreshnessScorer, KeywordRelevanceScorer, PathDepthScorer
|
||||
|
||||
|
||||
def test_scorers():
|
||||
test_cases = [
|
||||
# Keyword Scorer Tests
|
||||
{
|
||||
"scorer_type": "keyword",
|
||||
"config": {
|
||||
"keywords": ["python", "blog"],
|
||||
"weight": 1.0,
|
||||
"case_sensitive": False
|
||||
},
|
||||
"urls": {
|
||||
"https://example.com/python-blog": 1.0,
|
||||
"https://example.com/PYTHON-BLOG": 1.0,
|
||||
"https://example.com/python-only": 0.5,
|
||||
"https://example.com/other": 0.0
|
||||
}
|
||||
},
|
||||
|
||||
# Path Depth Scorer Tests
|
||||
{
|
||||
"scorer_type": "path_depth",
|
||||
"config": {
|
||||
"optimal_depth": 2,
|
||||
"weight": 1.0
|
||||
},
|
||||
"urls": {
|
||||
"https://example.com/a/b": 1.0,
|
||||
"https://example.com/a": 0.5,
|
||||
"https://example.com/a/b/c": 0.5,
|
||||
"https://example.com": 0.33333333
|
||||
}
|
||||
},
|
||||
|
||||
# Content Type Scorer Tests
|
||||
{
|
||||
"scorer_type": "content_type",
|
||||
"config": {
|
||||
"type_weights": {
|
||||
".html$": 1.0,
|
||||
".pdf$": 0.8,
|
||||
".jpg$": 0.6
|
||||
},
|
||||
"weight": 1.0
|
||||
},
|
||||
"urls": {
|
||||
"https://example.com/doc.html": 1.0,
|
||||
"https://example.com/doc.pdf": 0.8,
|
||||
"https://example.com/img.jpg": 0.6,
|
||||
"https://example.com/other.txt": 0.0
|
||||
}
|
||||
},
|
||||
|
||||
# Freshness Scorer Tests
|
||||
{
|
||||
"scorer_type": "freshness",
|
||||
"config": {
|
||||
"weight": 1.0, # Remove current_year since original doesn't support it
|
||||
},
|
||||
"urls": {
|
||||
"https://example.com/2024/01/post": 1.0,
|
||||
"https://example.com/2023/12/post": 0.9,
|
||||
"https://example.com/2022/post": 0.8,
|
||||
"https://example.com/no-date": 0.5
|
||||
}
|
||||
},
|
||||
|
||||
# Domain Authority Scorer Tests
|
||||
{
|
||||
"scorer_type": "domain",
|
||||
"config": {
|
||||
"domain_weights": {
|
||||
"python.org": 1.0,
|
||||
"github.com": 0.8,
|
||||
"medium.com": 0.6
|
||||
},
|
||||
"default_weight": 0.3,
|
||||
"weight": 1.0
|
||||
},
|
||||
"urls": {
|
||||
"https://python.org/about": 1.0,
|
||||
"https://github.com/repo": 0.8,
|
||||
"https://medium.com/post": 0.6,
|
||||
"https://unknown.com": 0.3
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
def create_scorer(scorer_type, config):
|
||||
if scorer_type == "keyword":
|
||||
return KeywordRelevanceScorer(**config)
|
||||
elif scorer_type == "path_depth":
|
||||
return PathDepthScorer(**config)
|
||||
elif scorer_type == "content_type":
|
||||
return ContentTypeScorer(**config)
|
||||
elif scorer_type == "freshness":
|
||||
return FreshnessScorer(**config,current_year=2024)
|
||||
elif scorer_type == "domain":
|
||||
return DomainAuthorityScorer(**config)
|
||||
|
||||
def run_accuracy_test():
|
||||
print("\nAccuracy Tests:")
|
||||
print("-" * 50)
|
||||
|
||||
all_passed = True
|
||||
for test_case in test_cases:
|
||||
print(f"\nTesting {test_case['scorer_type']} scorer:")
|
||||
scorer = create_scorer(
|
||||
test_case['scorer_type'],
|
||||
test_case['config']
|
||||
)
|
||||
|
||||
for url, expected in test_case['urls'].items():
|
||||
score = round(scorer.score(url), 8)
|
||||
expected = round(expected, 8)
|
||||
|
||||
if abs(score - expected) > 0.00001:
|
||||
print(f"❌ Scorer Failed: URL '{url}'")
|
||||
print(f" Expected: {expected}, Got: {score}")
|
||||
all_passed = False
|
||||
else:
|
||||
print(f"✅ Scorer Passed: URL '{url}'")
|
||||
|
||||
|
||||
return all_passed
|
||||
|
||||
def run_composite_test():
|
||||
print("\nTesting Composite Scorer:")
|
||||
print("-" * 50)
|
||||
|
||||
# Create test data
|
||||
test_urls = {
|
||||
"https://python.org/blog/2024/01/new-release.html":0.86666667,
|
||||
"https://github.com/repo/old-code.pdf": 0.62,
|
||||
"https://unknown.com/random": 0.26
|
||||
}
|
||||
|
||||
# Create composite scorers with all types
|
||||
scorers = []
|
||||
|
||||
for test_case in test_cases:
|
||||
scorer = create_scorer(
|
||||
test_case['scorer_type'],
|
||||
test_case['config']
|
||||
)
|
||||
scorers.append(scorer)
|
||||
|
||||
composite = CompositeScorer(scorers, normalize=True)
|
||||
|
||||
all_passed = True
|
||||
for url, expected in test_urls.items():
|
||||
score = round(composite.score(url), 8)
|
||||
|
||||
if abs(score - expected) > 0.00001:
|
||||
print(f"❌ Composite Failed: URL '{url}'")
|
||||
print(f" Expected: {expected}, Got: {score}")
|
||||
all_passed = False
|
||||
else:
|
||||
print(f"✅ Composite Passed: URL '{url}'")
|
||||
|
||||
return all_passed
|
||||
|
||||
# Run tests
|
||||
print("Running Scorer Tests...")
|
||||
accuracy_passed = run_accuracy_test()
|
||||
composite_passed = run_composite_test()
|
||||
|
||||
if accuracy_passed and composite_passed:
|
||||
print("\n✨ All tests passed!")
|
||||
# Note: Already have performance tests in run_scorer_performance_test()
|
||||
else:
|
||||
print("\n❌ Some tests failed!")
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_scorers()
|
||||
@@ -0,0 +1,34 @@
|
||||
import asyncio
|
||||
from crawl4ai import CrawlerRunConfig, AsyncWebCrawler, BrowserConfig
|
||||
from pathlib import Path
|
||||
import os
|
||||
|
||||
async def test_basic_download():
|
||||
|
||||
# Custom folder (otherwise defaults to ~/.crawl4ai/downloads)
|
||||
downloads_path = os.path.join(Path.home(), ".crawl4ai", "downloads")
|
||||
os.makedirs(downloads_path, exist_ok=True)
|
||||
browser_config = BrowserConfig(
|
||||
accept_downloads=True,
|
||||
downloads_path=downloads_path
|
||||
)
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
run_config = CrawlerRunConfig(
|
||||
js_code="""
|
||||
const link = document.querySelector('a[href$=".exe"]');
|
||||
if (link) { link.click(); }
|
||||
""",
|
||||
delay_before_return_html=5
|
||||
)
|
||||
result = await crawler.arun("https://www.python.org/downloads/", config=run_config)
|
||||
|
||||
if result.downloaded_files:
|
||||
print("Downloaded files:")
|
||||
for file_path in result.downloaded_files:
|
||||
print("•", file_path)
|
||||
else:
|
||||
print("No files downloaded.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_basic_download())
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Test flatten_shadow_dom feature — full comparison."""
|
||||
import asyncio
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
|
||||
|
||||
URL = "https://store.boschrexroth.com/en/us/p/hydraulic-cylinder-r900999011"
|
||||
|
||||
|
||||
async def run_test(label, bc, rc):
|
||||
print(f"\n{'='*70}")
|
||||
print(f"TEST: {label}")
|
||||
print(f"{'='*70}")
|
||||
async with AsyncWebCrawler(config=bc) as crawler:
|
||||
result = await crawler.arun(URL, config=rc)
|
||||
|
||||
html = result.html or ""
|
||||
cleaned = result.cleaned_html or ""
|
||||
md = ""
|
||||
if result.markdown and hasattr(result.markdown, "raw_markdown"):
|
||||
md = result.markdown.raw_markdown or ""
|
||||
|
||||
print(f" Success: {result.success}")
|
||||
print(f" Raw HTML: {len(html):>8} chars")
|
||||
print(f" Cleaned HTML: {len(cleaned):>8} chars")
|
||||
print(f" Markdown: {len(md):>8} chars")
|
||||
|
||||
checks = {
|
||||
"Product title": "HYDRAULIC CYLINDER" in md,
|
||||
"Part number (R900999011)": "R900999011" in md,
|
||||
"Product description": "mill type design" in md.lower(),
|
||||
"Feature: 6 types of mounting":"6 types of mounting" in md,
|
||||
"Feature: safety vent": "safety vent" in md.lower(),
|
||||
"Product Description heading": "Product Description" in md,
|
||||
"Technical Specs heading": "Technical Specs" in md,
|
||||
"Downloads heading": "Downloads" in md,
|
||||
"Specs table: CDH1": "CDH1" in md,
|
||||
"Specs table: 250 bar": "250" in md,
|
||||
}
|
||||
print(f"\n Content checks:")
|
||||
passes = sum(1 for v in checks.values() if v)
|
||||
for k, v in checks.items():
|
||||
print(f" {'PASS' if v else 'FAIL'} {k}")
|
||||
print(f"\n Result: {passes}/{len(checks)} checks passed")
|
||||
|
||||
# Show product content section
|
||||
for term in ["Product Description"]:
|
||||
idx = md.find(term)
|
||||
if idx >= 0:
|
||||
print(f"\n --- Product content section ---")
|
||||
print(md[idx:idx+1500])
|
||||
return result
|
||||
|
||||
|
||||
async def main():
|
||||
bc = BrowserConfig(headless=True)
|
||||
|
||||
r1 = await run_test(
|
||||
"BASELINE (no shadow flattening)",
|
||||
bc,
|
||||
CrawlerRunConfig(wait_until="load", delay_before_return_html=3.0),
|
||||
)
|
||||
|
||||
r2 = await run_test(
|
||||
"WITH flatten_shadow_dom=True",
|
||||
bc,
|
||||
CrawlerRunConfig(
|
||||
wait_until="load",
|
||||
delay_before_return_html=3.0,
|
||||
flatten_shadow_dom=True,
|
||||
),
|
||||
)
|
||||
|
||||
# Summary
|
||||
md1 = r1.markdown.raw_markdown if r1.markdown else ""
|
||||
md2 = r2.markdown.raw_markdown if r2.markdown else ""
|
||||
print(f"\n{'='*70}")
|
||||
print(f"SUMMARY")
|
||||
print(f"{'='*70}")
|
||||
print(f" Baseline markdown: {len(md1):>6} chars")
|
||||
print(f" Flattened markdown: {len(md2):>6} chars")
|
||||
print(f" Improvement: {len(md2)/max(len(md1),1):.1f}x more content")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,654 @@
|
||||
"""Tests for TokenUsage accumulation in generate_schema / agenerate_schema.
|
||||
|
||||
Covers:
|
||||
- Backward compatibility (usage=None, the default)
|
||||
- Single-shot schema generation accumulates usage
|
||||
- Validation retry loop accumulates across all LLM calls
|
||||
- _infer_target_json accumulates its own LLM call
|
||||
- Sync wrapper forwards usage correctly
|
||||
- JSON parse failure retry also accumulates usage
|
||||
- usage object receives correct cumulative totals
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import pytest
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, patch, MagicMock
|
||||
|
||||
from crawl4ai.extraction_strategy import JsonElementExtractionStrategy, JsonCssExtractionStrategy
|
||||
from crawl4ai.models import TokenUsage
|
||||
|
||||
# The functions are imported lazily inside method bodies via `from .utils import ...`
|
||||
# so we must patch at the source module.
|
||||
PATCH_TARGET = "crawl4ai.utils.aperform_completion_with_backoff"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers: fake LLM response builder
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _make_llm_response(content: str, prompt_tokens: int = 100, completion_tokens: int = 50):
|
||||
"""Build a fake litellm-style response with .usage and .choices."""
|
||||
return SimpleNamespace(
|
||||
usage=SimpleNamespace(
|
||||
prompt_tokens=prompt_tokens,
|
||||
completion_tokens=completion_tokens,
|
||||
total_tokens=prompt_tokens + completion_tokens,
|
||||
completion_tokens_details=None,
|
||||
prompt_tokens_details=None,
|
||||
),
|
||||
choices=[
|
||||
SimpleNamespace(
|
||||
message=SimpleNamespace(content=content)
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
# A valid CSS schema that will pass validation against SAMPLE_HTML
|
||||
VALID_SCHEMA = {
|
||||
"name": "products",
|
||||
"baseSelector": ".product",
|
||||
"fields": [
|
||||
{"name": "title", "selector": ".title", "type": "text"},
|
||||
{"name": "price", "selector": ".price", "type": "text"},
|
||||
],
|
||||
}
|
||||
|
||||
SAMPLE_HTML = """
|
||||
<div class="products">
|
||||
<div class="product">
|
||||
<span class="title">Widget</span>
|
||||
<span class="price">$10</span>
|
||||
</div>
|
||||
<div class="product">
|
||||
<span class="title">Gadget</span>
|
||||
<span class="price">$20</span>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
# A schema with a bad baseSelector — will fail validation and trigger retry
|
||||
BAD_SCHEMA = {
|
||||
"name": "products",
|
||||
"baseSelector": ".nonexistent-selector",
|
||||
"fields": [
|
||||
{"name": "title", "selector": ".title", "type": "text"},
|
||||
{"name": "price", "selector": ".price", "type": "text"},
|
||||
],
|
||||
}
|
||||
|
||||
# Fake LLMConfig
|
||||
@dataclass
|
||||
class FakeLLMConfig:
|
||||
provider: str = "fake/model"
|
||||
api_token: str = "fake-token"
|
||||
base_url: str = None
|
||||
backoff_base_delay: float = 0
|
||||
backoff_max_attempts: int = 1
|
||||
backoff_exponential_factor: int = 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestGenerateSchemaUsage:
|
||||
"""Test suite for usage tracking in generate_schema / agenerate_schema."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backward_compat_usage_none(self):
|
||||
"""When usage is not passed (default None), everything works as before."""
|
||||
mock_response = _make_llm_response(json.dumps(VALID_SCHEMA))
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
result = await JsonElementExtractionStrategy.agenerate_schema(
|
||||
html=SAMPLE_HTML,
|
||||
llm_config=FakeLLMConfig(),
|
||||
validate=False,
|
||||
)
|
||||
|
||||
assert isinstance(result, dict)
|
||||
assert result["name"] == "products"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_single_shot_no_validate(self):
|
||||
"""Single LLM call with validate=False populates usage correctly."""
|
||||
usage = TokenUsage()
|
||||
mock_response = _make_llm_response(
|
||||
json.dumps(VALID_SCHEMA), prompt_tokens=200, completion_tokens=80
|
||||
)
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
result = await JsonElementExtractionStrategy.agenerate_schema(
|
||||
html=SAMPLE_HTML,
|
||||
llm_config=FakeLLMConfig(),
|
||||
validate=False,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
assert result["name"] == "products"
|
||||
assert usage.prompt_tokens == 200
|
||||
assert usage.completion_tokens == 80
|
||||
assert usage.total_tokens == 280
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validation_success_first_try(self):
|
||||
"""With validate=True and schema passes validation on first try, usage reflects 1 call."""
|
||||
usage = TokenUsage()
|
||||
mock_response = _make_llm_response(
|
||||
json.dumps(VALID_SCHEMA), prompt_tokens=300, completion_tokens=120
|
||||
)
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
result = await JsonElementExtractionStrategy.agenerate_schema(
|
||||
html=SAMPLE_HTML,
|
||||
llm_config=FakeLLMConfig(),
|
||||
validate=True,
|
||||
max_refinements=3,
|
||||
usage=usage,
|
||||
# Provide target_json_example to skip _infer_target_json
|
||||
target_json_example='{"title": "x", "price": "y"}',
|
||||
)
|
||||
|
||||
assert result["name"] == "products"
|
||||
# Only 1 LLM call since validation passed
|
||||
assert usage.prompt_tokens == 300
|
||||
assert usage.completion_tokens == 120
|
||||
assert usage.total_tokens == 420
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validation_retries_accumulate_usage(self):
|
||||
"""When validation fails, retry calls accumulate into the same usage object."""
|
||||
usage = TokenUsage()
|
||||
|
||||
# First call returns bad schema (fails validation), second returns good schema
|
||||
responses = [
|
||||
_make_llm_response(json.dumps(BAD_SCHEMA), prompt_tokens=300, completion_tokens=100),
|
||||
_make_llm_response(json.dumps(VALID_SCHEMA), prompt_tokens=350, completion_tokens=120),
|
||||
]
|
||||
call_count = 0
|
||||
|
||||
async def mock_completion(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
idx = min(call_count, len(responses) - 1)
|
||||
call_count += 1
|
||||
return responses[idx]
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
side_effect=mock_completion,
|
||||
):
|
||||
result = await JsonElementExtractionStrategy.agenerate_schema(
|
||||
html=SAMPLE_HTML,
|
||||
llm_config=FakeLLMConfig(),
|
||||
validate=True,
|
||||
max_refinements=3,
|
||||
usage=usage,
|
||||
target_json_example='{"title": "x", "price": "y"}',
|
||||
)
|
||||
|
||||
assert result["name"] == "products"
|
||||
# Two LLM calls: 300+350=650 prompt, 100+120=220 completion
|
||||
assert usage.prompt_tokens == 650
|
||||
assert usage.completion_tokens == 220
|
||||
assert usage.total_tokens == 870
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_infer_target_json_accumulates_usage(self):
|
||||
"""When validate=True and no target_json_example, _infer_target_json makes an extra LLM call."""
|
||||
usage = TokenUsage()
|
||||
|
||||
infer_response = _make_llm_response(
|
||||
'{"title": "Widget", "price": "$10"}',
|
||||
prompt_tokens=50,
|
||||
completion_tokens=30,
|
||||
)
|
||||
schema_response = _make_llm_response(
|
||||
json.dumps(VALID_SCHEMA),
|
||||
prompt_tokens=300,
|
||||
completion_tokens=120,
|
||||
)
|
||||
|
||||
call_count = 0
|
||||
|
||||
async def mock_completion(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
# First call is _infer_target_json, second is schema generation
|
||||
if call_count == 1:
|
||||
return infer_response
|
||||
return schema_response
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
side_effect=mock_completion,
|
||||
):
|
||||
result = await JsonElementExtractionStrategy.agenerate_schema(
|
||||
html=SAMPLE_HTML,
|
||||
query="extract product title and price",
|
||||
llm_config=FakeLLMConfig(),
|
||||
validate=True,
|
||||
max_refinements=3,
|
||||
usage=usage,
|
||||
# No target_json_example — triggers _infer_target_json
|
||||
)
|
||||
|
||||
assert result["name"] == "products"
|
||||
# _infer_target_json: 50+30 = 80
|
||||
# schema generation: 300+120 = 420
|
||||
# Total: 350 prompt, 150 completion, 500 total
|
||||
assert usage.prompt_tokens == 350
|
||||
assert usage.completion_tokens == 150
|
||||
assert usage.total_tokens == 500
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_infer_plus_retries_accumulate(self):
|
||||
"""Full pipeline: infer + bad schema + good schema = 3 calls accumulated."""
|
||||
usage = TokenUsage()
|
||||
|
||||
infer_resp = _make_llm_response(
|
||||
'{"title": "x", "price": "y"}',
|
||||
prompt_tokens=50, completion_tokens=20,
|
||||
)
|
||||
bad_resp = _make_llm_response(
|
||||
json.dumps(BAD_SCHEMA),
|
||||
prompt_tokens=300, completion_tokens=100,
|
||||
)
|
||||
good_resp = _make_llm_response(
|
||||
json.dumps(VALID_SCHEMA),
|
||||
prompt_tokens=400, completion_tokens=150,
|
||||
)
|
||||
|
||||
call_count = 0
|
||||
|
||||
async def mock_completion(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
return infer_resp
|
||||
elif call_count == 2:
|
||||
return bad_resp
|
||||
else:
|
||||
return good_resp
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
side_effect=mock_completion,
|
||||
):
|
||||
result = await JsonElementExtractionStrategy.agenerate_schema(
|
||||
html=SAMPLE_HTML,
|
||||
query="extract products",
|
||||
llm_config=FakeLLMConfig(),
|
||||
validate=True,
|
||||
max_refinements=3,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
# 3 calls total
|
||||
assert call_count == 3
|
||||
assert usage.prompt_tokens == 750 # 50 + 300 + 400
|
||||
assert usage.completion_tokens == 270 # 20 + 100 + 150
|
||||
assert usage.total_tokens == 1020 # 70 + 400 + 550
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_json_parse_failure_retry_accumulates(self):
|
||||
"""When LLM returns invalid JSON, the retry also accumulates usage."""
|
||||
usage = TokenUsage()
|
||||
|
||||
# First response is not valid JSON, second is valid
|
||||
bad_json_resp = _make_llm_response(
|
||||
"this is not json {{{",
|
||||
prompt_tokens=200, completion_tokens=60,
|
||||
)
|
||||
good_resp = _make_llm_response(
|
||||
json.dumps(VALID_SCHEMA),
|
||||
prompt_tokens=250, completion_tokens=80,
|
||||
)
|
||||
|
||||
call_count = 0
|
||||
|
||||
async def mock_completion(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
return bad_json_resp
|
||||
return good_resp
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
side_effect=mock_completion,
|
||||
):
|
||||
result = await JsonElementExtractionStrategy.agenerate_schema(
|
||||
html=SAMPLE_HTML,
|
||||
llm_config=FakeLLMConfig(),
|
||||
validate=True,
|
||||
max_refinements=3,
|
||||
usage=usage,
|
||||
target_json_example='{"title": "x", "price": "y"}',
|
||||
)
|
||||
|
||||
assert result["name"] == "products"
|
||||
# Both calls tracked: even the one that returned bad JSON
|
||||
assert usage.prompt_tokens == 450 # 200 + 250
|
||||
assert usage.completion_tokens == 140 # 60 + 80
|
||||
assert usage.total_tokens == 590
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_usage_none_does_not_crash(self):
|
||||
"""Explicitly passing usage=None should not raise any errors."""
|
||||
mock_response = _make_llm_response(json.dumps(VALID_SCHEMA))
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
result = await JsonElementExtractionStrategy.agenerate_schema(
|
||||
html=SAMPLE_HTML,
|
||||
llm_config=FakeLLMConfig(),
|
||||
validate=False,
|
||||
usage=None,
|
||||
)
|
||||
|
||||
assert isinstance(result, dict)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_preexisting_usage_values_are_added_to(self):
|
||||
"""If usage already has values, new tokens are ADDED, not replaced."""
|
||||
usage = TokenUsage(prompt_tokens=1000, completion_tokens=500, total_tokens=1500)
|
||||
|
||||
mock_response = _make_llm_response(
|
||||
json.dumps(VALID_SCHEMA), prompt_tokens=200, completion_tokens=80
|
||||
)
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
await JsonElementExtractionStrategy.agenerate_schema(
|
||||
html=SAMPLE_HTML,
|
||||
llm_config=FakeLLMConfig(),
|
||||
validate=False,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
assert usage.prompt_tokens == 1200 # 1000 + 200
|
||||
assert usage.completion_tokens == 580 # 500 + 80
|
||||
assert usage.total_tokens == 1780 # 1500 + 280
|
||||
|
||||
def test_sync_wrapper_passes_usage(self):
|
||||
"""The sync generate_schema forwards usage to agenerate_schema."""
|
||||
usage = TokenUsage()
|
||||
mock_response = _make_llm_response(
|
||||
json.dumps(VALID_SCHEMA), prompt_tokens=200, completion_tokens=80
|
||||
)
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
result = JsonElementExtractionStrategy.generate_schema(
|
||||
html=SAMPLE_HTML,
|
||||
llm_config=FakeLLMConfig(),
|
||||
validate=False,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
assert result["name"] == "products"
|
||||
assert usage.prompt_tokens == 200
|
||||
assert usage.completion_tokens == 80
|
||||
assert usage.total_tokens == 280
|
||||
|
||||
def test_sync_wrapper_usage_none_backward_compat(self):
|
||||
"""Sync wrapper with no usage arg (default) still works."""
|
||||
mock_response = _make_llm_response(json.dumps(VALID_SCHEMA))
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
result = JsonElementExtractionStrategy.generate_schema(
|
||||
html=SAMPLE_HTML,
|
||||
llm_config=FakeLLMConfig(),
|
||||
validate=False,
|
||||
)
|
||||
|
||||
assert isinstance(result, dict)
|
||||
assert result["name"] == "products"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_max_refinements_zero_single_call(self):
|
||||
"""max_refinements=0 with validate=True means exactly 1 attempt, 1 usage entry."""
|
||||
usage = TokenUsage()
|
||||
mock_response = _make_llm_response(
|
||||
json.dumps(BAD_SCHEMA), prompt_tokens=300, completion_tokens=100
|
||||
)
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
result = await JsonElementExtractionStrategy.agenerate_schema(
|
||||
html=SAMPLE_HTML,
|
||||
llm_config=FakeLLMConfig(),
|
||||
validate=True,
|
||||
max_refinements=0,
|
||||
usage=usage,
|
||||
target_json_example='{"title": "x", "price": "y"}',
|
||||
)
|
||||
|
||||
# Even though validation fails, only 1 attempt (0 refinements)
|
||||
assert usage.prompt_tokens == 300
|
||||
assert usage.completion_tokens == 100
|
||||
assert usage.total_tokens == 400
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_css_subclass_inherits_usage(self):
|
||||
"""JsonCssExtractionStrategy.agenerate_schema also supports usage."""
|
||||
usage = TokenUsage()
|
||||
mock_response = _make_llm_response(
|
||||
json.dumps(VALID_SCHEMA), prompt_tokens=150, completion_tokens=60
|
||||
)
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
result = await JsonCssExtractionStrategy.agenerate_schema(
|
||||
html=SAMPLE_HTML,
|
||||
llm_config=FakeLLMConfig(),
|
||||
validate=False,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
assert result["name"] == "products"
|
||||
assert usage.total_tokens == 210
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_infer_target_json_failure_still_tracks_nothing(self):
|
||||
"""If _infer_target_json raises (and catches), usage should not break.
|
||||
|
||||
When the inference LLM call itself throws an exception before we get
|
||||
response.usage, no tokens should be added (graceful degradation).
|
||||
"""
|
||||
usage = TokenUsage()
|
||||
|
||||
call_count = 0
|
||||
|
||||
async def mock_completion(*args, **kwargs):
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count == 1:
|
||||
# _infer_target_json call — simulate exception
|
||||
raise ConnectionError("LLM is down")
|
||||
# Schema generation call
|
||||
return _make_llm_response(
|
||||
json.dumps(VALID_SCHEMA),
|
||||
prompt_tokens=300,
|
||||
completion_tokens=100,
|
||||
)
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
side_effect=mock_completion,
|
||||
):
|
||||
result = await JsonElementExtractionStrategy.agenerate_schema(
|
||||
html=SAMPLE_HTML,
|
||||
query="extract products",
|
||||
llm_config=FakeLLMConfig(),
|
||||
validate=True,
|
||||
max_refinements=0,
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
# Only the schema call counted; infer call failed before tracking
|
||||
assert usage.prompt_tokens == 300
|
||||
assert usage.completion_tokens == 100
|
||||
assert usage.total_tokens == 400
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multiple_bad_retries_then_best_effort(self):
|
||||
"""All retries fail validation, usage still accumulates for every attempt."""
|
||||
usage = TokenUsage()
|
||||
|
||||
# Every call returns bad schema — validation will always fail
|
||||
mock_response = _make_llm_response(
|
||||
json.dumps(BAD_SCHEMA), prompt_tokens=200, completion_tokens=80
|
||||
)
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
result = await JsonElementExtractionStrategy.agenerate_schema(
|
||||
html=SAMPLE_HTML,
|
||||
llm_config=FakeLLMConfig(),
|
||||
validate=True,
|
||||
max_refinements=2, # 1 initial + 2 retries = 3 calls
|
||||
usage=usage,
|
||||
target_json_example='{"title": "x", "price": "y"}',
|
||||
)
|
||||
|
||||
# Returns best-effort (last schema), but all 3 calls tracked
|
||||
assert usage.prompt_tokens == 600 # 200 * 3
|
||||
assert usage.completion_tokens == 240 # 80 * 3
|
||||
assert usage.total_tokens == 840 # 280 * 3
|
||||
|
||||
|
||||
class TestInferTargetJsonUsage:
|
||||
"""Isolated tests for _infer_target_json usage tracking."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_infer_tracks_usage(self):
|
||||
"""Direct call to _infer_target_json with usage accumulator."""
|
||||
usage = TokenUsage()
|
||||
mock_response = _make_llm_response(
|
||||
'{"name": "test", "value": "123"}',
|
||||
prompt_tokens=80,
|
||||
completion_tokens=25,
|
||||
)
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
result = await JsonElementExtractionStrategy._infer_target_json(
|
||||
query="extract names and values",
|
||||
html_snippet="<div>test</div>",
|
||||
llm_config=FakeLLMConfig(),
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
assert result == {"name": "test", "value": "123"}
|
||||
assert usage.prompt_tokens == 80
|
||||
assert usage.completion_tokens == 25
|
||||
assert usage.total_tokens == 105
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_infer_usage_none_backward_compat(self):
|
||||
"""_infer_target_json with usage=None (default) still works."""
|
||||
mock_response = _make_llm_response('{"name": "test"}')
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
result = await JsonElementExtractionStrategy._infer_target_json(
|
||||
query="extract names",
|
||||
html_snippet="<div>test</div>",
|
||||
llm_config=FakeLLMConfig(),
|
||||
)
|
||||
|
||||
assert result == {"name": "test"}
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_infer_exception_no_usage_side_effect(self):
|
||||
"""When _infer_target_json fails, usage is untouched (exception before tracking)."""
|
||||
usage = TokenUsage(prompt_tokens=100, completion_tokens=50, total_tokens=150)
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
new_callable=AsyncMock,
|
||||
side_effect=RuntimeError("API down"),
|
||||
):
|
||||
result = await JsonElementExtractionStrategy._infer_target_json(
|
||||
query="extract names",
|
||||
html_snippet="<div>test</div>",
|
||||
llm_config=FakeLLMConfig(),
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
# Returns None on failure
|
||||
assert result is None
|
||||
# Usage unchanged — exception happened before tracking
|
||||
assert usage.prompt_tokens == 100
|
||||
assert usage.completion_tokens == 50
|
||||
assert usage.total_tokens == 150
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_infer_empty_response_still_tracks(self):
|
||||
"""When LLM returns empty content, usage is still tracked (response was received)."""
|
||||
usage = TokenUsage()
|
||||
mock_response = _make_llm_response("", prompt_tokens=80, completion_tokens=5)
|
||||
|
||||
with patch(
|
||||
PATCH_TARGET,
|
||||
new_callable=AsyncMock,
|
||||
return_value=mock_response,
|
||||
):
|
||||
result = await JsonElementExtractionStrategy._infer_target_json(
|
||||
query="extract names",
|
||||
html_snippet="<div>test</div>",
|
||||
llm_config=FakeLLMConfig(),
|
||||
usage=usage,
|
||||
)
|
||||
|
||||
# Returns None because content is empty
|
||||
assert result is None
|
||||
# But usage was tracked because we got a response
|
||||
assert usage.prompt_tokens == 80
|
||||
assert usage.completion_tokens == 5
|
||||
assert usage.total_tokens == 85
|
||||
@@ -0,0 +1,116 @@
|
||||
from tkinter import N
|
||||
from crawl4ai.async_crawler_strategy import AsyncHTTPCrawlerStrategy
|
||||
from crawl4ai.async_logger import AsyncLogger
|
||||
from crawl4ai import CrawlerRunConfig, HTTPCrawlerConfig
|
||||
from crawl4ai.async_crawler_strategy import ConnectionTimeoutError
|
||||
import asyncio
|
||||
import os
|
||||
|
||||
async def main():
|
||||
"""Test the AsyncHTTPCrawlerStrategy with various scenarios"""
|
||||
logger = AsyncLogger(verbose=True)
|
||||
|
||||
# Initialize the strategy with default HTTPCrawlerConfig
|
||||
crawler = AsyncHTTPCrawlerStrategy(
|
||||
browser_config=HTTPCrawlerConfig(),
|
||||
logger=logger
|
||||
)
|
||||
# Test 1: Basic HTTP GET
|
||||
print("\n=== Test 1: Basic HTTP GET ===")
|
||||
result = await crawler.crawl("https://example.com")
|
||||
print(f"Status: {result.status_code}")
|
||||
print(f"Content length: {len(result.html)}")
|
||||
print(f"Headers: {dict(result.response_headers)}")
|
||||
|
||||
# Test 2: POST request with JSON
|
||||
print("\n=== Test 2: POST with JSON ===")
|
||||
crawler.browser_config = crawler.browser_config.clone(
|
||||
method="POST",
|
||||
json={"test": "data"},
|
||||
headers={"Content-Type": "application/json"}
|
||||
)
|
||||
try:
|
||||
result = await crawler.crawl(
|
||||
"https://httpbin.org/post",
|
||||
)
|
||||
print(f"Status: {result.status_code}")
|
||||
print(f"Response: {result.html[:200]}...")
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
|
||||
# Test 3: File handling
|
||||
crawler.browser_config = HTTPCrawlerConfig()
|
||||
print("\n=== Test 3: Local file handling ===")
|
||||
# Create a tmp file with test content
|
||||
from tempfile import NamedTemporaryFile
|
||||
with NamedTemporaryFile(delete=False) as f:
|
||||
f.write(b"<html><body>Test content</body></html>")
|
||||
f.close()
|
||||
result = await crawler.crawl(f"file://{f.name}")
|
||||
print(f"File content: {result.html}")
|
||||
|
||||
# Test 4: Raw content
|
||||
print("\n=== Test 4: Raw content handling ===")
|
||||
raw_html = "raw://<html><body>Raw test content</body></html>"
|
||||
result = await crawler.crawl(raw_html)
|
||||
print(f"Raw content: {result.html}")
|
||||
|
||||
# Test 5: Custom hooks
|
||||
print("\n=== Test 5: Custom hooks ===")
|
||||
async def before_request(url, kwargs):
|
||||
print(f"Before request to {url}")
|
||||
kwargs['headers']['X-Custom'] = 'test'
|
||||
|
||||
async def after_request(response):
|
||||
print(f"After request, status: {response.status_code}")
|
||||
|
||||
crawler.set_hook('before_request', before_request)
|
||||
crawler.set_hook('after_request', after_request)
|
||||
result = await crawler.crawl("https://example.com")
|
||||
|
||||
# Test 6: Error handling
|
||||
print("\n=== Test 6: Error handling ===")
|
||||
try:
|
||||
await crawler.crawl("https://nonexistent.domain.test")
|
||||
except Exception as e:
|
||||
print(f"Expected error: {e}")
|
||||
|
||||
# Test 7: Redirects
|
||||
print("\n=== Test 7: Redirect handling ===")
|
||||
crawler.browser_config = HTTPCrawlerConfig(follow_redirects=True)
|
||||
result = await crawler.crawl("http://httpbin.org/redirect/1")
|
||||
print(f"Final URL: {result.redirected_url}")
|
||||
|
||||
# Test 8: Custom timeout
|
||||
print("\n=== Test 8: Custom timeout ===")
|
||||
try:
|
||||
await crawler.crawl(
|
||||
"https://httpbin.org/delay/5",
|
||||
config=CrawlerRunConfig(page_timeout=2)
|
||||
)
|
||||
except ConnectionTimeoutError as e:
|
||||
print(f"Expected timeout: {e}")
|
||||
|
||||
# Test 9: SSL verification
|
||||
print("\n=== Test 9: SSL verification ===")
|
||||
crawler.browser_config = HTTPCrawlerConfig(verify_ssl=False)
|
||||
try:
|
||||
await crawler.crawl("https://expired.badssl.com/")
|
||||
print("Connected to invalid SSL site with verification disabled")
|
||||
except Exception as e:
|
||||
print(f"SSL error: {e}")
|
||||
|
||||
# Test 10: Large file streaming
|
||||
print("\n=== Test 10: Large file streaming ===")
|
||||
from tempfile import NamedTemporaryFile
|
||||
with NamedTemporaryFile(delete=False) as f:
|
||||
f.write(b"<html><body>" + b"X" * 1024 * 1024 * 10 + b"</body></html>")
|
||||
f.close()
|
||||
result = await crawler.crawl("file://" + f.name)
|
||||
print(f"Large file content length: {len(result.html)}")
|
||||
os.remove(f.name)
|
||||
|
||||
crawler.close()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,86 @@
|
||||
import os
|
||||
import asyncio
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
|
||||
from crawl4ai import LLMConfig
|
||||
from crawl4ai.content_filter_strategy import LLMContentFilter
|
||||
|
||||
async def test_llm_filter():
|
||||
# Create an HTML source that needs intelligent filtering
|
||||
url = "https://docs.python.org/3/tutorial/classes.html"
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
headless=True,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
|
||||
run_config = CrawlerRunConfig(cache_mode=CacheMode.ENABLED)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
# First get the raw HTML
|
||||
result = await crawler.arun(url, config=run_config)
|
||||
html = result.cleaned_html
|
||||
|
||||
# Initialize LLM filter with focused instruction
|
||||
filter = LLMContentFilter(
|
||||
llm_config=LLMConfig(provider="openai/gpt-4o",api_token=os.getenv('OPENAI_API_KEY')),
|
||||
instruction="""
|
||||
Focus on extracting the core educational content about Python classes.
|
||||
Include:
|
||||
- Key concepts and their explanations
|
||||
- Important code examples
|
||||
- Essential technical details
|
||||
Exclude:
|
||||
- Navigation elements
|
||||
- Sidebars
|
||||
- Footer content
|
||||
- Version information
|
||||
- Any non-essential UI elements
|
||||
|
||||
Format the output as clean markdown with proper code blocks and headers.
|
||||
""",
|
||||
verbose=True
|
||||
)
|
||||
|
||||
filter = LLMContentFilter(
|
||||
llm_config = LLMConfig(provider="openai/gpt-4o",api_token=os.getenv('OPENAI_API_KEY')),
|
||||
chunk_token_threshold=2 ** 12 * 2, # 2048 * 2
|
||||
instruction="""
|
||||
Extract the main educational content while preserving its original wording and substance completely. Your task is to:
|
||||
|
||||
1. Maintain the exact language and terminology used in the main content
|
||||
2. Keep all technical explanations, examples, and educational content intact
|
||||
3. Preserve the original flow and structure of the core content
|
||||
4. Remove only clearly irrelevant elements like:
|
||||
- Navigation menus
|
||||
- Advertisement sections
|
||||
- Cookie notices
|
||||
- Footers with site information
|
||||
- Sidebars with external links
|
||||
- Any UI elements that don't contribute to learning
|
||||
|
||||
The goal is to create a clean markdown version that reads exactly like the original article,
|
||||
keeping all valuable content but free from distracting elements. Imagine you're creating
|
||||
a perfect reading experience where nothing valuable is lost, but all noise is removed.
|
||||
""",
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# Apply filtering
|
||||
filtered_content = filter.filter_content(html, ignore_cache = True)
|
||||
|
||||
# Show results
|
||||
print("\nFiltered Content Length:", len(filtered_content))
|
||||
print("\nFirst 500 chars of filtered content:")
|
||||
if filtered_content:
|
||||
print(filtered_content[0][:500])
|
||||
|
||||
# Save on disc the markdown version
|
||||
with open("filtered_content.md", "w", encoding="utf-8") as f:
|
||||
f.write("\n".join(filtered_content))
|
||||
|
||||
# Show token usage
|
||||
filter.show_usage()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_llm_filter())
|
||||
@@ -0,0 +1,115 @@
|
||||
"""
|
||||
Sample script to test the max_scroll_steps parameter implementation
|
||||
"""
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Get the grandparent directory
|
||||
grandparent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.path.append(grandparent_dir)
|
||||
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
|
||||
|
||||
|
||||
|
||||
from crawl4ai import AsyncWebCrawler
|
||||
from crawl4ai.async_configs import CrawlerRunConfig
|
||||
|
||||
async def test_max_scroll_steps():
|
||||
"""
|
||||
Test the max_scroll_steps parameter with different configurations
|
||||
"""
|
||||
print("🚀 Testing max_scroll_steps parameter implementation")
|
||||
print("=" * 60)
|
||||
|
||||
async with AsyncWebCrawler(verbose=True) as crawler:
|
||||
|
||||
# Test 1: Without max_scroll_steps (unlimited scrolling)
|
||||
print("\\n📋 Test 1: Unlimited scrolling (max_scroll_steps=None)")
|
||||
config1 = CrawlerRunConfig(
|
||||
scan_full_page=True,
|
||||
scroll_delay=0.1,
|
||||
max_scroll_steps=None, # Default behavior
|
||||
verbose=True
|
||||
)
|
||||
|
||||
print(f"Config: scan_full_page={config1.scan_full_page}, max_scroll_steps={config1.max_scroll_steps}")
|
||||
|
||||
try:
|
||||
result1 = await crawler.arun(
|
||||
url="https://example.com", # Simple page for testing
|
||||
config=config1
|
||||
)
|
||||
print(f"✅ Test 1 Success: Crawled {len(result1.markdown)} characters")
|
||||
except Exception as e:
|
||||
print(f"❌ Test 1 Failed: {e}")
|
||||
|
||||
# Test 2: With limited scroll steps
|
||||
print("\\n📋 Test 2: Limited scrolling (max_scroll_steps=3)")
|
||||
config2 = CrawlerRunConfig(
|
||||
scan_full_page=True,
|
||||
scroll_delay=0.1,
|
||||
max_scroll_steps=3, # Limit to 3 scroll steps
|
||||
verbose=True
|
||||
)
|
||||
|
||||
print(f"Config: scan_full_page={config2.scan_full_page}, max_scroll_steps={config2.max_scroll_steps}")
|
||||
|
||||
try:
|
||||
result2 = await crawler.arun(
|
||||
url="https://techcrunch.com/", # Another test page
|
||||
config=config2
|
||||
)
|
||||
print(f"✅ Test 2 Success: Crawled {len(result2.markdown)} characters")
|
||||
except Exception as e:
|
||||
print(f"❌ Test 2 Failed: {e}")
|
||||
|
||||
# Test 3: Test serialization/deserialization
|
||||
print("\\n📋 Test 3: Configuration serialization test")
|
||||
config3 = CrawlerRunConfig(
|
||||
scan_full_page=True,
|
||||
max_scroll_steps=5,
|
||||
scroll_delay=0.2
|
||||
)
|
||||
|
||||
# Test to_dict
|
||||
config_dict = config3.to_dict()
|
||||
print(f"Serialized max_scroll_steps: {config_dict.get('max_scroll_steps')}")
|
||||
|
||||
# Test from_kwargs
|
||||
config4 = CrawlerRunConfig.from_kwargs({
|
||||
'scan_full_page': True,
|
||||
'max_scroll_steps': 7,
|
||||
'scroll_delay': 0.3
|
||||
})
|
||||
print(f"Deserialized max_scroll_steps: {config4.max_scroll_steps}")
|
||||
print("✅ Test 3 Success: Serialization works correctly")
|
||||
|
||||
# Test 4: Edge case - max_scroll_steps = 0
|
||||
print("\\n📋 Test 4: Edge case (max_scroll_steps=0)")
|
||||
config5 = CrawlerRunConfig(
|
||||
scan_full_page=True,
|
||||
max_scroll_steps=0, # Should not scroll at all
|
||||
verbose=True
|
||||
)
|
||||
|
||||
try:
|
||||
result5 = await crawler.arun(
|
||||
url="https://techcrunch.com/",
|
||||
config=config5
|
||||
)
|
||||
print(f"✅ Test 4 Success: No scrolling performed, crawled {len(result5.markdown)} characters")
|
||||
except Exception as e:
|
||||
print(f"❌ Test 4 Failed: {e}")
|
||||
|
||||
print("\\n" + "=" * 60)
|
||||
print("🎉 All tests completed!")
|
||||
print("\\nThe max_scroll_steps parameter is working correctly:")
|
||||
print("- None: Unlimited scrolling (default behavior)")
|
||||
print("- Positive integer: Limits scroll steps to that number")
|
||||
print("- 0: No scrolling performed")
|
||||
print("- Properly serializes/deserializes in config")
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("Starting max_scroll_steps test...")
|
||||
asyncio.run(test_max_scroll_steps())
|
||||
@@ -0,0 +1,213 @@
|
||||
# test_mhtml_capture.py
|
||||
|
||||
import pytest
|
||||
import asyncio
|
||||
import re # For more robust MHTML checks
|
||||
|
||||
# Assuming these can be imported directly from the crawl4ai library
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CrawlResult
|
||||
|
||||
# A reliable, simple static HTML page for testing
|
||||
# Using httpbin as it's designed for testing clients
|
||||
TEST_URL_SIMPLE = "https://httpbin.org/html"
|
||||
EXPECTED_CONTENT_SIMPLE = "Herman Melville - Moby-Dick"
|
||||
|
||||
# A slightly more complex page that might involve JS (good secondary test)
|
||||
TEST_URL_JS = "https://quotes.toscrape.com/js/"
|
||||
EXPECTED_CONTENT_JS = "Quotes to Scrape" # Title of the page, which should be present in MHTML
|
||||
|
||||
# Removed the custom event_loop fixture as pytest-asyncio provides a default one.
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mhtml_capture_when_enabled():
|
||||
"""
|
||||
Verify that when CrawlerRunConfig has capture_mhtml=True,
|
||||
the CrawlResult contains valid MHTML content.
|
||||
"""
|
||||
# Create a fresh browser config and crawler instance for this test
|
||||
browser_config = BrowserConfig(headless=True) # Use headless for testing CI/CD
|
||||
# --- Key: Enable MHTML capture in the run config ---
|
||||
run_config = CrawlerRunConfig(capture_mhtml=True)
|
||||
|
||||
# Create a fresh crawler instance
|
||||
crawler = AsyncWebCrawler(config=browser_config)
|
||||
|
||||
try:
|
||||
# Start the browser
|
||||
await crawler.start()
|
||||
|
||||
# Perform the crawl with the MHTML-enabled config
|
||||
result: CrawlResult = await crawler.arun(TEST_URL_SIMPLE, config=run_config)
|
||||
|
||||
# --- Assertions ---
|
||||
assert result is not None, "Crawler should return a result object"
|
||||
assert result.success is True, f"Crawling {TEST_URL_SIMPLE} should succeed. Error: {result.error_message}"
|
||||
|
||||
# 1. Check if the mhtml attribute exists (will fail if CrawlResult not updated)
|
||||
assert hasattr(result, 'mhtml'), "CrawlResult object must have an 'mhtml' attribute"
|
||||
|
||||
# 2. Check if mhtml is populated
|
||||
assert result.mhtml is not None, "MHTML content should be captured when enabled"
|
||||
assert isinstance(result.mhtml, str), "MHTML content should be a string"
|
||||
assert len(result.mhtml) > 500, "MHTML content seems too short, likely invalid" # Basic sanity check
|
||||
|
||||
# 3. Check for MHTML structure indicators (more robust than simple string contains)
|
||||
# MHTML files are multipart MIME messages
|
||||
assert re.search(r"Content-Type: multipart/related;", result.mhtml, re.IGNORECASE), \
|
||||
"MHTML should contain 'Content-Type: multipart/related;'"
|
||||
# Should contain a boundary definition
|
||||
assert re.search(r"boundary=\"----MultipartBoundary", result.mhtml), \
|
||||
"MHTML should contain a multipart boundary"
|
||||
# Should contain the main HTML part
|
||||
assert re.search(r"Content-Type: text/html", result.mhtml, re.IGNORECASE), \
|
||||
"MHTML should contain a 'Content-Type: text/html' part"
|
||||
|
||||
# 4. Check if the *actual page content* is within the MHTML string
|
||||
# This confirms the snapshot captured the rendered page
|
||||
assert EXPECTED_CONTENT_SIMPLE in result.mhtml, \
|
||||
f"Expected content '{EXPECTED_CONTENT_SIMPLE}' not found within the captured MHTML"
|
||||
|
||||
# 5. Ensure standard HTML is still present and correct
|
||||
assert result.html is not None, "Standard HTML should still be present"
|
||||
assert isinstance(result.html, str), "Standard HTML should be a string"
|
||||
assert EXPECTED_CONTENT_SIMPLE in result.html, \
|
||||
f"Expected content '{EXPECTED_CONTENT_SIMPLE}' not found within the standard HTML"
|
||||
|
||||
finally:
|
||||
# Important: Ensure browser is completely closed even if assertions fail
|
||||
await crawler.close()
|
||||
# Help the garbage collector clean up
|
||||
crawler = None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mhtml_capture_when_disabled_explicitly():
|
||||
"""
|
||||
Verify that when CrawlerRunConfig explicitly has capture_mhtml=False,
|
||||
the CrawlResult.mhtml attribute is None.
|
||||
"""
|
||||
# Create a fresh browser config and crawler instance for this test
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
# --- Key: Explicitly disable MHTML capture ---
|
||||
run_config = CrawlerRunConfig(capture_mhtml=False)
|
||||
|
||||
# Create a fresh crawler instance
|
||||
crawler = AsyncWebCrawler(config=browser_config)
|
||||
|
||||
try:
|
||||
# Start the browser
|
||||
await crawler.start()
|
||||
result: CrawlResult = await crawler.arun(TEST_URL_SIMPLE, config=run_config)
|
||||
|
||||
assert result is not None
|
||||
assert result.success is True, f"Crawling {TEST_URL_SIMPLE} should succeed. Error: {result.error_message}"
|
||||
|
||||
# 1. Check attribute existence (important for TDD start)
|
||||
assert hasattr(result, 'mhtml'), "CrawlResult object must have an 'mhtml' attribute"
|
||||
|
||||
# 2. Check mhtml is None
|
||||
assert result.mhtml is None, "MHTML content should be None when explicitly disabled"
|
||||
|
||||
# 3. Ensure standard HTML is still present
|
||||
assert result.html is not None
|
||||
assert EXPECTED_CONTENT_SIMPLE in result.html
|
||||
|
||||
finally:
|
||||
# Important: Ensure browser is completely closed even if assertions fail
|
||||
await crawler.close()
|
||||
# Help the garbage collector clean up
|
||||
crawler = None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mhtml_capture_when_disabled_by_default():
|
||||
"""
|
||||
Verify that if capture_mhtml is not specified (using its default),
|
||||
the CrawlResult.mhtml attribute is None.
|
||||
(This assumes the default value for capture_mhtml in CrawlerRunConfig is False)
|
||||
"""
|
||||
# Create a fresh browser config and crawler instance for this test
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
# --- Key: Use default run config ---
|
||||
run_config = CrawlerRunConfig() # Do not specify capture_mhtml
|
||||
|
||||
# Create a fresh crawler instance
|
||||
crawler = AsyncWebCrawler(config=browser_config)
|
||||
|
||||
try:
|
||||
# Start the browser
|
||||
await crawler.start()
|
||||
result: CrawlResult = await crawler.arun(TEST_URL_SIMPLE, config=run_config)
|
||||
|
||||
assert result is not None
|
||||
assert result.success is True, f"Crawling {TEST_URL_SIMPLE} should succeed. Error: {result.error_message}"
|
||||
|
||||
# 1. Check attribute existence
|
||||
assert hasattr(result, 'mhtml'), "CrawlResult object must have an 'mhtml' attribute"
|
||||
|
||||
# 2. Check mhtml is None (assuming default is False)
|
||||
assert result.mhtml is None, "MHTML content should be None when using default config (assuming default=False)"
|
||||
|
||||
# 3. Ensure standard HTML is still present
|
||||
assert result.html is not None
|
||||
assert EXPECTED_CONTENT_SIMPLE in result.html
|
||||
|
||||
finally:
|
||||
# Important: Ensure browser is completely closed even if assertions fail
|
||||
await crawler.close()
|
||||
# Help the garbage collector clean up
|
||||
crawler = None
|
||||
|
||||
# Optional: Add a test for a JS-heavy page if needed
|
||||
@pytest.mark.asyncio
|
||||
async def test_mhtml_capture_on_js_page_when_enabled():
|
||||
"""
|
||||
Verify MHTML capture works on a page requiring JavaScript execution.
|
||||
"""
|
||||
# Create a fresh browser config and crawler instance for this test
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
run_config = CrawlerRunConfig(
|
||||
capture_mhtml=True,
|
||||
# Add a small wait or JS execution if needed for the JS page to fully render
|
||||
# For quotes.toscrape.com/js/, it renders quickly, but a wait might be safer
|
||||
# wait_for_timeout=2000 # Example: wait up to 2 seconds
|
||||
js_code="await new Promise(r => setTimeout(r, 500));" # Small delay after potential load
|
||||
)
|
||||
|
||||
# Create a fresh crawler instance
|
||||
crawler = AsyncWebCrawler(config=browser_config)
|
||||
|
||||
try:
|
||||
# Start the browser
|
||||
await crawler.start()
|
||||
result: CrawlResult = await crawler.arun(TEST_URL_JS, config=run_config)
|
||||
|
||||
assert result is not None
|
||||
assert result.success is True, f"Crawling {TEST_URL_JS} should succeed. Error: {result.error_message}"
|
||||
assert hasattr(result, 'mhtml'), "CrawlResult object must have an 'mhtml' attribute"
|
||||
assert result.mhtml is not None, "MHTML content should be captured on JS page when enabled"
|
||||
assert isinstance(result.mhtml, str), "MHTML content should be a string"
|
||||
assert len(result.mhtml) > 500, "MHTML content from JS page seems too short"
|
||||
|
||||
# Check for MHTML structure
|
||||
assert re.search(r"Content-Type: multipart/related;", result.mhtml, re.IGNORECASE)
|
||||
assert re.search(r"Content-Type: text/html", result.mhtml, re.IGNORECASE)
|
||||
|
||||
# Check for content rendered by JS within the MHTML
|
||||
assert EXPECTED_CONTENT_JS in result.mhtml, \
|
||||
f"Expected JS-rendered content '{EXPECTED_CONTENT_JS}' not found within the captured MHTML"
|
||||
|
||||
# Check standard HTML too
|
||||
assert result.html is not None
|
||||
assert EXPECTED_CONTENT_JS in result.html, \
|
||||
f"Expected JS-rendered content '{EXPECTED_CONTENT_JS}' not found within the standard HTML"
|
||||
|
||||
finally:
|
||||
# Important: Ensure browser is completely closed even if assertions fail
|
||||
await crawler.close()
|
||||
# Help the garbage collector clean up
|
||||
crawler = None
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Use pytest for async tests
|
||||
pytest.main(["-xvs", __file__])
|
||||
@@ -0,0 +1,185 @@
|
||||
from crawl4ai.async_webcrawler import AsyncWebCrawler
|
||||
from crawl4ai.async_configs import CrawlerRunConfig, BrowserConfig
|
||||
import asyncio
|
||||
import aiohttp
|
||||
from aiohttp import web
|
||||
import tempfile
|
||||
import shutil
|
||||
import os, sys, time, json
|
||||
|
||||
|
||||
async def start_test_server():
|
||||
app = web.Application()
|
||||
|
||||
async def basic_page(request):
|
||||
return web.Response(text="""
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Network Request Test</title>
|
||||
</head>
|
||||
<body>
|
||||
<h1>Test Page for Network Capture</h1>
|
||||
<p>This page performs network requests and console logging.</p>
|
||||
<img src="/image.png" alt="Test Image">
|
||||
<script>
|
||||
console.log("Basic console log");
|
||||
console.error("Error message");
|
||||
console.warn("Warning message");
|
||||
|
||||
// Make some XHR requests
|
||||
const xhr = new XMLHttpRequest();
|
||||
xhr.open('GET', '/api/data', true);
|
||||
xhr.send();
|
||||
|
||||
// Make a fetch request
|
||||
fetch('/api/json')
|
||||
.then(response => response.json())
|
||||
.catch(error => console.error('Fetch error:', error));
|
||||
|
||||
// Trigger an error
|
||||
setTimeout(() => {
|
||||
try {
|
||||
nonExistentFunction();
|
||||
} catch (e) {
|
||||
console.error("Caught error:", e);
|
||||
}
|
||||
}, 100);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
""", content_type="text/html")
|
||||
|
||||
async def image(request):
|
||||
# Return a small 1x1 transparent PNG
|
||||
return web.Response(body=bytes.fromhex('89504E470D0A1A0A0000000D49484452000000010000000108060000001F15C4890000000D4944415478DA63FAFFFF3F030079DB00018D959DE70000000049454E44AE426082'), content_type="image/png")
|
||||
|
||||
async def api_data(request):
|
||||
return web.Response(text="sample data")
|
||||
|
||||
async def api_json(request):
|
||||
return web.json_response({"status": "success", "message": "JSON data"})
|
||||
|
||||
# Register routes
|
||||
app.router.add_get('/', basic_page)
|
||||
app.router.add_get('/image.png', image)
|
||||
app.router.add_get('/api/data', api_data)
|
||||
app.router.add_get('/api/json', api_json)
|
||||
|
||||
runner = web.AppRunner(app)
|
||||
await runner.setup()
|
||||
site = web.TCPSite(runner, 'localhost', 8080)
|
||||
await site.start()
|
||||
|
||||
return runner
|
||||
|
||||
|
||||
async def test_network_console_capture():
|
||||
print("\n=== Testing Network and Console Capture ===\n")
|
||||
|
||||
# Start test server
|
||||
runner = await start_test_server()
|
||||
try:
|
||||
browser_config = BrowserConfig(headless=True)
|
||||
|
||||
# Test with capture disabled (default)
|
||||
print("\n1. Testing with capture disabled (default)...")
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
config = CrawlerRunConfig(
|
||||
wait_until="networkidle", # Wait for network to be idle
|
||||
)
|
||||
result = await crawler.arun(url="http://localhost:8080/", config=config)
|
||||
|
||||
assert result.network_requests is None, "Network requests should be None when capture is disabled"
|
||||
assert result.console_messages is None, "Console messages should be None when capture is disabled"
|
||||
print("✓ Default config correctly returns None for network_requests and console_messages")
|
||||
|
||||
# Test with network capture enabled
|
||||
print("\n2. Testing with network capture enabled...")
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
config = CrawlerRunConfig(
|
||||
wait_until="networkidle", # Wait for network to be idle
|
||||
capture_network_requests=True
|
||||
)
|
||||
result = await crawler.arun(url="http://localhost:8080/", config=config)
|
||||
|
||||
assert result.network_requests is not None, "Network requests should be captured"
|
||||
print(f"✓ Captured {len(result.network_requests)} network requests")
|
||||
|
||||
# Check if we have both requests and responses
|
||||
request_count = len([r for r in result.network_requests if r.get("event_type") == "request"])
|
||||
response_count = len([r for r in result.network_requests if r.get("event_type") == "response"])
|
||||
print(f" - {request_count} requests, {response_count} responses")
|
||||
|
||||
# Check if we captured specific resources
|
||||
urls = [r.get("url") for r in result.network_requests]
|
||||
has_image = any("/image.png" in url for url in urls)
|
||||
has_api_data = any("/api/data" in url for url in urls)
|
||||
has_api_json = any("/api/json" in url for url in urls)
|
||||
|
||||
assert has_image, "Should have captured image request"
|
||||
assert has_api_data, "Should have captured API data request"
|
||||
assert has_api_json, "Should have captured API JSON request"
|
||||
print("✓ Captured expected network requests (image, API endpoints)")
|
||||
|
||||
# Test with console capture enabled
|
||||
print("\n3. Testing with console capture enabled...")
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
config = CrawlerRunConfig(
|
||||
wait_until="networkidle", # Wait for network to be idle
|
||||
capture_console_messages=True
|
||||
)
|
||||
result = await crawler.arun(url="http://localhost:8080/", config=config)
|
||||
|
||||
assert result.console_messages is not None, "Console messages should be captured"
|
||||
print(f"✓ Captured {len(result.console_messages)} console messages")
|
||||
|
||||
# Check if we have different types of console messages
|
||||
message_types = set(msg.get("type") for msg in result.console_messages if "type" in msg)
|
||||
print(f" - Message types: {', '.join(message_types)}")
|
||||
|
||||
# Print all captured messages for debugging
|
||||
print(" - Captured messages:")
|
||||
for msg in result.console_messages:
|
||||
print(f" * Type: {msg.get('type', 'N/A')}, Text: {msg.get('text', 'N/A')}")
|
||||
|
||||
# Look for specific messages
|
||||
messages = [msg.get("text") for msg in result.console_messages if "text" in msg]
|
||||
has_basic_log = any("Basic console log" in msg for msg in messages)
|
||||
has_error_msg = any("Error message" in msg for msg in messages)
|
||||
has_warning_msg = any("Warning message" in msg for msg in messages)
|
||||
|
||||
assert has_basic_log, "Should have captured basic console.log message"
|
||||
assert has_error_msg, "Should have captured console.error message"
|
||||
assert has_warning_msg, "Should have captured console.warn message"
|
||||
print("✓ Captured expected console messages (log, error, warning)")
|
||||
|
||||
# Test with both captures enabled
|
||||
print("\n4. Testing with both network and console capture enabled...")
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
config = CrawlerRunConfig(
|
||||
wait_until="networkidle", # Wait for network to be idle
|
||||
capture_network_requests=True,
|
||||
capture_console_messages=True
|
||||
)
|
||||
result = await crawler.arun(url="http://localhost:8080/", config=config)
|
||||
|
||||
assert result.network_requests is not None, "Network requests should be captured"
|
||||
assert result.console_messages is not None, "Console messages should be captured"
|
||||
print(f"✓ Successfully captured both {len(result.network_requests)} network requests and {len(result.console_messages)} console messages")
|
||||
|
||||
finally:
|
||||
await runner.cleanup()
|
||||
print("\nTest server shutdown")
|
||||
|
||||
|
||||
async def main():
|
||||
try:
|
||||
await test_network_console_capture()
|
||||
print("\n✅ All tests passed successfully!")
|
||||
except Exception as e:
|
||||
print(f"\n❌ Test failed: {str(e)}")
|
||||
raise
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,43 @@
|
||||
import asyncio
|
||||
import os
|
||||
from crawl4ai.async_webcrawler import AsyncWebCrawler
|
||||
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig, CacheMode
|
||||
|
||||
# Simple concurrency test for persistent context page creation
|
||||
# Usage: python scripts/test_persistent_context.py
|
||||
|
||||
URLS = [
|
||||
# "https://example.com",
|
||||
"https://httpbin.org/html",
|
||||
"https://www.python.org/",
|
||||
"https://www.rust-lang.org/",
|
||||
]
|
||||
|
||||
async def main():
|
||||
profile_dir = os.path.join(os.path.expanduser("~"), ".crawl4ai", "profiles", "test-persistent-profile")
|
||||
os.makedirs(profile_dir, exist_ok=True)
|
||||
|
||||
browser_config = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True,
|
||||
use_persistent_context=True,
|
||||
user_data_dir=profile_dir,
|
||||
use_managed_browser=True,
|
||||
verbose=True,
|
||||
)
|
||||
|
||||
run_cfg = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
stream=False,
|
||||
verbose=True,
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
results = await crawler.arun_many(URLS, config=run_cfg)
|
||||
for r in results:
|
||||
print(r.url, r.success, len(r.markdown.raw_markdown) if r.markdown else 0)
|
||||
# r = await crawler.arun(url=URLS[0], config=run_cfg)
|
||||
# print(r.url, r.success, len(r.markdown.raw_markdown) if r.markdown else 0)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,159 @@
|
||||
from crawl4ai.utils import RobotsParser
|
||||
|
||||
import asyncio
|
||||
import aiohttp
|
||||
from aiohttp import web
|
||||
import tempfile
|
||||
import shutil
|
||||
import os, sys, time, json
|
||||
|
||||
|
||||
async def test_robots_parser():
|
||||
print("\n=== Testing RobotsParser ===\n")
|
||||
|
||||
# Setup temporary directory for testing
|
||||
temp_dir = tempfile.mkdtemp()
|
||||
try:
|
||||
# 1. Basic setup test
|
||||
print("1. Testing basic initialization...")
|
||||
parser = RobotsParser(cache_dir=temp_dir)
|
||||
assert os.path.exists(parser.db_path), "Database file not created"
|
||||
print("✓ Basic initialization passed")
|
||||
|
||||
# 2. Test common cases
|
||||
print("\n2. Testing common cases...")
|
||||
allowed = await parser.can_fetch("https://www.example.com", "MyBot/1.0")
|
||||
print(f"✓ Regular website fetch: {'allowed' if allowed else 'denied'}")
|
||||
|
||||
# Test caching
|
||||
print("Testing cache...")
|
||||
start = time.time()
|
||||
await parser.can_fetch("https://www.example.com", "MyBot/1.0")
|
||||
duration = time.time() - start
|
||||
print(f"✓ Cached lookup took: {duration*1000:.2f}ms")
|
||||
assert duration < 0.03, "Cache lookup too slow"
|
||||
|
||||
# 3. Edge cases
|
||||
print("\n3. Testing edge cases...")
|
||||
|
||||
# Empty URL
|
||||
result = await parser.can_fetch("", "MyBot/1.0")
|
||||
print(f"✓ Empty URL handled: {'allowed' if result else 'denied'}")
|
||||
|
||||
# Invalid URL
|
||||
result = await parser.can_fetch("not_a_url", "MyBot/1.0")
|
||||
print(f"✓ Invalid URL handled: {'allowed' if result else 'denied'}")
|
||||
|
||||
# URL without scheme
|
||||
result = await parser.can_fetch("example.com/page", "MyBot/1.0")
|
||||
print(f"✓ URL without scheme handled: {'allowed' if result else 'denied'}")
|
||||
|
||||
# 4. Test with local server
|
||||
async def start_test_server():
|
||||
app = web.Application()
|
||||
|
||||
async def robots_txt(request):
|
||||
return web.Response(text="""User-agent: *
|
||||
Disallow: /private/
|
||||
Allow: /public/
|
||||
""")
|
||||
|
||||
async def malformed_robots(request):
|
||||
return web.Response(text="<<<malformed>>>")
|
||||
|
||||
async def timeout_robots(request):
|
||||
await asyncio.sleep(5)
|
||||
return web.Response(text="Should timeout")
|
||||
|
||||
async def empty_robots(request):
|
||||
return web.Response(text="")
|
||||
|
||||
async def giant_robots(request):
|
||||
return web.Response(text="User-agent: *\nDisallow: /\n" * 10000)
|
||||
|
||||
# Mount all handlers at root level
|
||||
app.router.add_get('/robots.txt', robots_txt)
|
||||
app.router.add_get('/malformed/robots.txt', malformed_robots)
|
||||
app.router.add_get('/timeout/robots.txt', timeout_robots)
|
||||
app.router.add_get('/empty/robots.txt', empty_robots)
|
||||
app.router.add_get('/giant/robots.txt', giant_robots)
|
||||
|
||||
runner = web.AppRunner(app)
|
||||
await runner.setup()
|
||||
site = web.TCPSite(runner, 'localhost', 8080)
|
||||
await site.start()
|
||||
return runner
|
||||
|
||||
runner = await start_test_server()
|
||||
try:
|
||||
print("\n4. Testing robots.txt rules...")
|
||||
base_url = "http://localhost:8080"
|
||||
|
||||
# Test public access
|
||||
result = await parser.can_fetch(f"{base_url}/public/page", "bot")
|
||||
print(f"Public access (/public/page): {'allowed' if result else 'denied'}")
|
||||
assert result, "Public path should be allowed"
|
||||
|
||||
# Test private access
|
||||
result = await parser.can_fetch(f"{base_url}/private/secret", "bot")
|
||||
print(f"Private access (/private/secret): {'allowed' if result else 'denied'}")
|
||||
assert not result, "Private path should be denied"
|
||||
|
||||
# Test malformed
|
||||
result = await parser.can_fetch("http://localhost:8080/malformed/page", "bot")
|
||||
print(f"✓ Malformed robots.txt handled: {'allowed' if result else 'denied'}")
|
||||
|
||||
# Test timeout
|
||||
start = time.time()
|
||||
result = await parser.can_fetch("http://localhost:8080/timeout/page", "bot")
|
||||
duration = time.time() - start
|
||||
print(f"✓ Timeout handled (took {duration:.2f}s): {'allowed' if result else 'denied'}")
|
||||
assert duration < 3, "Timeout not working"
|
||||
|
||||
# Test empty
|
||||
result = await parser.can_fetch("http://localhost:8080/empty/page", "bot")
|
||||
print(f"✓ Empty robots.txt handled: {'allowed' if result else 'denied'}")
|
||||
|
||||
# Test giant file
|
||||
start = time.time()
|
||||
result = await parser.can_fetch("http://localhost:8080/giant/page", "bot")
|
||||
duration = time.time() - start
|
||||
print(f"✓ Giant robots.txt handled (took {duration:.2f}s): {'allowed' if result else 'denied'}")
|
||||
|
||||
finally:
|
||||
await runner.cleanup()
|
||||
|
||||
# 5. Cache manipulation
|
||||
print("\n5. Testing cache manipulation...")
|
||||
|
||||
# Clear expired
|
||||
parser.clear_expired()
|
||||
print("✓ Clear expired entries completed")
|
||||
|
||||
# Clear all
|
||||
parser.clear_cache()
|
||||
print("✓ Clear all cache completed")
|
||||
|
||||
# Test with custom TTL
|
||||
custom_parser = RobotsParser(cache_dir=temp_dir, cache_ttl=1) # 1 second TTL
|
||||
await custom_parser.can_fetch("https://www.example.com", "bot")
|
||||
print("✓ Custom TTL fetch completed")
|
||||
await asyncio.sleep(1.1)
|
||||
start = time.time()
|
||||
await custom_parser.can_fetch("https://www.example.com", "bot")
|
||||
print(f"✓ TTL expiry working (refetched after {time.time() - start:.2f}s)")
|
||||
|
||||
finally:
|
||||
# Cleanup
|
||||
shutil.rmtree(temp_dir)
|
||||
print("\nTest cleanup completed")
|
||||
|
||||
async def main():
|
||||
try:
|
||||
await test_robots_parser()
|
||||
except Exception as e:
|
||||
print(f"Test failed: {str(e)}")
|
||||
raise
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,112 @@
|
||||
# https://claude.ai/chat/c4bbe93d-fb54-44ce-92af-76b4c8086c6b
|
||||
# https://claude.ai/chat/c24a768c-d8b2-478a-acc7-d76d42a308da
|
||||
import os, sys
|
||||
|
||||
parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.append(parent_dir)
|
||||
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
|
||||
|
||||
import asyncio
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
|
||||
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
|
||||
from crawl4ai import JsonCssExtractionStrategy, JsonXPathExtractionStrategy
|
||||
from crawl4ai.utils import preprocess_html_for_schema, JsonXPathExtractionStrategy
|
||||
import json
|
||||
|
||||
# Test HTML - A complex job board with companies, departments, and positions
|
||||
test_html = """
|
||||
<div class="company-listings">
|
||||
<div class="company" data-company-id="123">
|
||||
<div class="company-header">
|
||||
<img class="company-logo" src="google.png" alt="Google">
|
||||
<h1 class="company-name">Google</h1>
|
||||
<div class="company-meta">
|
||||
<span class="company-size">10,000+ employees</span>
|
||||
<span class="company-industry">Technology</span>
|
||||
<a href="https://google.careers" class="careers-link">Careers Page</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="departments">
|
||||
<div class="department">
|
||||
<h2 class="department-name">Engineering</h2>
|
||||
<div class="positions">
|
||||
<div class="position-card" data-position-id="eng-1">
|
||||
<h3 class="position-title">Senior Software Engineer</h3>
|
||||
<span class="salary-range">$150,000 - $250,000</span>
|
||||
<div class="position-meta">
|
||||
<span class="location">Mountain View, CA</span>
|
||||
<span class="job-type">Full-time</span>
|
||||
<span class="experience">5+ years</span>
|
||||
</div>
|
||||
<div class="skills-required">
|
||||
<span class="skill">Python</span>
|
||||
<span class="skill">Kubernetes</span>
|
||||
<span class="skill">Machine Learning</span>
|
||||
</div>
|
||||
<p class="position-description">Join our core engineering team...</p>
|
||||
<div class="application-info">
|
||||
<span class="posting-date">Posted: 2024-03-15</span>
|
||||
<button class="apply-btn" data-req-id="REQ12345">Apply Now</button>
|
||||
</div>
|
||||
</div>
|
||||
<!-- More positions -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="department">
|
||||
<h2 class="department-name">Marketing</h2>
|
||||
<div class="positions">
|
||||
<div class="position-card" data-position-id="mkt-1">
|
||||
<h3 class="position-title">Growth Marketing Manager</h3>
|
||||
<span class="salary-range">$120,000 - $180,000</span>
|
||||
<div class="position-meta">
|
||||
<span class="location">New York, NY</span>
|
||||
<span class="job-type">Full-time</span>
|
||||
<span class="experience">3+ years</span>
|
||||
</div>
|
||||
<div class="skills-required">
|
||||
<span class="skill">SEO</span>
|
||||
<span class="skill">Analytics</span>
|
||||
<span class="skill">Content Strategy</span>
|
||||
</div>
|
||||
<p class="position-description">Drive our growth initiatives...</p>
|
||||
<div class="application-info">
|
||||
<span class="posting-date">Posted: 2024-03-14</span>
|
||||
<button class="apply-btn" data-req-id="REQ12346">Apply Now</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
# Test cases
|
||||
def test_schema_generation():
|
||||
# Test 1: No query (should extract everything)
|
||||
print("\nTest 1: No Query (Full Schema)")
|
||||
schema1 = JsonCssExtractionStrategy.generate_schema(test_html)
|
||||
print(json.dumps(schema1, indent=2))
|
||||
|
||||
# Test 2: Query for just basic job info
|
||||
print("\nTest 2: Basic Job Info Query")
|
||||
query2 = "I only need job titles, salaries, and locations"
|
||||
schema2 = JsonCssExtractionStrategy.generate_schema(test_html, query2)
|
||||
print(json.dumps(schema2, indent=2))
|
||||
|
||||
# Test 3: Query for company and department structure
|
||||
print("\nTest 3: Organizational Structure Query")
|
||||
query3 = "Extract company details and department names, without position details"
|
||||
schema3 = JsonCssExtractionStrategy.generate_schema(test_html, query3)
|
||||
print(json.dumps(schema3, indent=2))
|
||||
|
||||
# Test 4: Query for specific skills tracking
|
||||
print("\nTest 4: Skills Analysis Query")
|
||||
query4 = "I want to analyze required skills across all positions"
|
||||
schema4 = JsonCssExtractionStrategy.generate_schema(test_html, query4)
|
||||
print(json.dumps(schema4, indent=2))
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_schema_generation()
|
||||
@@ -0,0 +1,50 @@
|
||||
import os, sys
|
||||
# append 2 parent directories to sys.path to import crawl4ai
|
||||
parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.append(parent_dir)
|
||||
parent_parent_dir = os.path.dirname(parent_dir)
|
||||
sys.path.append(parent_parent_dir)
|
||||
|
||||
import asyncio
|
||||
from crawl4ai import *
|
||||
|
||||
async def test_crawler():
|
||||
# Setup configurations
|
||||
browser_config = BrowserConfig(headless=True, verbose=False)
|
||||
crawler_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
markdown_generator=DefaultMarkdownGenerator(
|
||||
content_filter=PruningContentFilter(
|
||||
threshold=0.48,
|
||||
threshold_type="fixed",
|
||||
min_word_threshold=0
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
# Test URLs - mix of different sites
|
||||
urls = [
|
||||
"http://example.com",
|
||||
"http://example.org",
|
||||
"http://example.net",
|
||||
] * 10 # 15 total URLs
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
print("\n=== Testing Streaming Mode ===")
|
||||
async for result in await crawler.arun_many(
|
||||
urls=urls,
|
||||
config=crawler_config.clone(stream=True),
|
||||
):
|
||||
print(f"Received result for: {result.url} - Success: {result.success}")
|
||||
|
||||
print("\n=== Testing Batch Mode ===")
|
||||
results = await crawler.arun_many(
|
||||
urls=urls,
|
||||
config=crawler_config,
|
||||
)
|
||||
print(f"Received all {len(results)} results at once")
|
||||
for result in results:
|
||||
print(f"Batch result for: {result.url} - Success: {result.success}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_crawler())
|
||||
@@ -0,0 +1,39 @@
|
||||
import os, sys
|
||||
# append 2 parent directories to sys.path to import crawl4ai
|
||||
parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.append(parent_dir)
|
||||
parent_parent_dir = os.path.dirname(parent_dir)
|
||||
sys.path.append(parent_parent_dir)
|
||||
|
||||
|
||||
import asyncio
|
||||
from typing import List
|
||||
from crawl4ai import *
|
||||
from crawl4ai.async_dispatcher import MemoryAdaptiveDispatcher
|
||||
|
||||
async def test_streaming():
|
||||
browser_config = BrowserConfig(headless=True, verbose=True)
|
||||
crawler_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
markdown_generator=DefaultMarkdownGenerator(
|
||||
# content_filter=PruningContentFilter(
|
||||
# threshold=0.48,
|
||||
# threshold_type="fixed",
|
||||
# min_word_threshold=0
|
||||
# )
|
||||
),
|
||||
)
|
||||
|
||||
urls = ["http://example.com"] * 10
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
dispatcher = MemoryAdaptiveDispatcher(
|
||||
max_session_permit=5,
|
||||
check_interval=0.5
|
||||
)
|
||||
|
||||
async for result in dispatcher.run_urls_stream(urls, crawler, crawler_config):
|
||||
print(f"Got result for {result.url} - Success: {result.result.success}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_streaming())
|
||||
@@ -0,0 +1,321 @@
|
||||
"""
|
||||
Tests for _strip_markdown_fences helper and agenerate_schema() JSON parsing fix.
|
||||
|
||||
Covers:
|
||||
- Unit tests for _strip_markdown_fences (pure logic, no API calls)
|
||||
- Real integration tests calling Anthropic/OpenAI/Groq against quotes.toscrape.com
|
||||
- Regression tests ensuring clean JSON is never corrupted
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import pytest
|
||||
|
||||
from crawl4ai.extraction_strategy import (
|
||||
_strip_markdown_fences,
|
||||
JsonCssExtractionStrategy,
|
||||
JsonXPathExtractionStrategy,
|
||||
)
|
||||
from crawl4ai.async_configs import LLMConfig
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sample schemas for unit tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
SIMPLE_SCHEMA = {
|
||||
"name": "Quotes",
|
||||
"baseSelector": ".quote",
|
||||
"fields": [
|
||||
{"name": "text", "selector": ".text", "type": "text"},
|
||||
{"name": "author", "selector": ".author", "type": "text"},
|
||||
],
|
||||
}
|
||||
|
||||
NESTED_SCHEMA = {
|
||||
"name": "Products",
|
||||
"baseSelector": ".product-card",
|
||||
"baseFields": [{"name": "id", "selector": "", "type": "attribute", "attribute": "data-id"}],
|
||||
"fields": [
|
||||
{"name": "title", "selector": "h2.title", "type": "text"},
|
||||
{"name": "price", "selector": ".price", "type": "text"},
|
||||
{"name": "description", "selector": ".desc", "type": "text"},
|
||||
{"name": "image", "selector": "img.product-img", "type": "attribute", "attribute": "src"},
|
||||
],
|
||||
}
|
||||
|
||||
TEST_URL = "https://quotes.toscrape.com/"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Unit tests for _strip_markdown_fences
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestStripMarkdownFences:
|
||||
"""Direct unit tests for the _strip_markdown_fences helper."""
|
||||
|
||||
def test_clean_json_passthrough(self):
|
||||
"""Clean JSON (no fences) must pass through unchanged."""
|
||||
raw = json.dumps(SIMPLE_SCHEMA)
|
||||
assert _strip_markdown_fences(raw) == raw
|
||||
|
||||
def test_json_fence(self):
|
||||
"""```json ... ``` wrapping is stripped correctly."""
|
||||
raw = '```json\n{"key": "value"}\n```'
|
||||
assert json.loads(_strip_markdown_fences(raw)) == {"key": "value"}
|
||||
|
||||
def test_bare_fence(self):
|
||||
"""``` ... ``` (no language tag) is stripped correctly."""
|
||||
raw = '```\n{"key": "value"}\n```'
|
||||
assert json.loads(_strip_markdown_fences(raw)) == {"key": "value"}
|
||||
|
||||
def test_fence_with_language_variants(self):
|
||||
"""Various language tags after ``` are stripped."""
|
||||
for lang in ["json", "JSON", "javascript", "js", "text", "jsonc"]:
|
||||
raw = f"```{lang}\n{{\"a\": 1}}\n```"
|
||||
result = _strip_markdown_fences(raw)
|
||||
assert json.loads(result) == {"a": 1}, f"Failed for language tag: {lang}"
|
||||
|
||||
def test_leading_trailing_whitespace(self):
|
||||
"""Whitespace around fenced content is stripped."""
|
||||
raw = ' \n ```json\n{"key": "value"}\n``` \n '
|
||||
assert json.loads(_strip_markdown_fences(raw)) == {"key": "value"}
|
||||
|
||||
def test_no_fences_with_whitespace(self):
|
||||
"""Plain JSON with surrounding whitespace is handled."""
|
||||
raw = ' \n {"key": "value"} \n '
|
||||
assert json.loads(_strip_markdown_fences(raw)) == {"key": "value"}
|
||||
|
||||
def test_nested_code_block_in_value(self):
|
||||
"""JSON with a string value containing ``` is not corrupted."""
|
||||
inner = {"code": "Use ```python\\nprint()\\n``` for code blocks"}
|
||||
raw = f'```json\n{json.dumps(inner)}\n```'
|
||||
result = _strip_markdown_fences(raw)
|
||||
parsed = json.loads(result)
|
||||
assert "```python" in parsed["code"]
|
||||
|
||||
def test_complex_schema(self):
|
||||
"""A real-world multi-field schema wrapped in fences parses correctly."""
|
||||
raw = f"```json\n{json.dumps(NESTED_SCHEMA, indent=2)}\n```"
|
||||
result = _strip_markdown_fences(raw)
|
||||
assert json.loads(result) == NESTED_SCHEMA
|
||||
|
||||
def test_empty_string(self):
|
||||
"""Empty string returns empty string."""
|
||||
assert _strip_markdown_fences("") == ""
|
||||
|
||||
def test_only_whitespace(self):
|
||||
"""Whitespace-only string returns empty string."""
|
||||
assert _strip_markdown_fences(" \n\n ") == ""
|
||||
|
||||
def test_only_fences(self):
|
||||
"""Bare fences with nothing inside return empty string."""
|
||||
assert _strip_markdown_fences("```json\n```") == ""
|
||||
|
||||
def test_multiline_json(self):
|
||||
"""Multiline pretty-printed JSON inside fences."""
|
||||
pretty = json.dumps(SIMPLE_SCHEMA, indent=4)
|
||||
raw = f"```json\n{pretty}\n```"
|
||||
assert json.loads(_strip_markdown_fences(raw)) == SIMPLE_SCHEMA
|
||||
|
||||
def test_already_clean_does_not_mutate(self):
|
||||
"""Passing already-clean JSON multiple times is idempotent."""
|
||||
raw = json.dumps(SIMPLE_SCHEMA)
|
||||
once = _strip_markdown_fences(raw)
|
||||
twice = _strip_markdown_fences(once)
|
||||
assert once == twice == raw
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Real integration tests — actual LLM API calls against quotes.toscrape.com
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
def _validate_schema(schema: dict):
|
||||
"""Validate that a generated schema has the expected structure."""
|
||||
assert isinstance(schema, dict), f"Schema must be a dict, got {type(schema)}"
|
||||
assert "name" in schema, "Schema must have a 'name' field"
|
||||
assert "baseSelector" in schema, "Schema must have a 'baseSelector' field"
|
||||
assert "fields" in schema, "Schema must have a 'fields' field"
|
||||
assert isinstance(schema["fields"], list), "'fields' must be a list"
|
||||
assert len(schema["fields"]) > 0, "'fields' must not be empty"
|
||||
for field in schema["fields"]:
|
||||
assert "name" in field, f"Each field must have a 'name': {field}"
|
||||
assert "selector" in field, f"Each field must have a 'selector': {field}"
|
||||
assert "type" in field, f"Each field must have a 'type': {field}"
|
||||
|
||||
|
||||
class TestRealAnthropicSchemaGeneration:
|
||||
"""Real API calls to Anthropic models — the exact scenario from the bug report."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
not os.getenv("CRAWL4AI_ANTHROPIC_KEY"),
|
||||
reason="CRAWL4AI_ANTHROPIC_KEY not set",
|
||||
)
|
||||
async def test_anthropic_haiku_css_schema(self):
|
||||
"""Reproduce the original bug: anthropic/claude-haiku-4-5 + CSS schema."""
|
||||
schema = await JsonCssExtractionStrategy.agenerate_schema(
|
||||
url=TEST_URL,
|
||||
schema_type="CSS",
|
||||
query="Extract all quotes with their text, author, and tags",
|
||||
llm_config=LLMConfig(
|
||||
provider="anthropic/claude-haiku-4-5",
|
||||
api_token=os.getenv("CRAWL4AI_ANTHROPIC_KEY"),
|
||||
),
|
||||
)
|
||||
_validate_schema(schema)
|
||||
print(f"\n[Anthropic Haiku CSS] Generated schema: {json.dumps(schema, indent=2)}")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
not os.getenv("CRAWL4AI_ANTHROPIC_KEY"),
|
||||
reason="CRAWL4AI_ANTHROPIC_KEY not set",
|
||||
)
|
||||
async def test_anthropic_haiku_xpath_schema(self):
|
||||
"""Anthropic haiku with XPath schema type."""
|
||||
schema = await JsonXPathExtractionStrategy.agenerate_schema(
|
||||
url=TEST_URL,
|
||||
schema_type="XPATH",
|
||||
query="Extract all quotes with their text, author, and tags",
|
||||
llm_config=LLMConfig(
|
||||
provider="anthropic/claude-haiku-4-5",
|
||||
api_token=os.getenv("CRAWL4AI_ANTHROPIC_KEY"),
|
||||
),
|
||||
)
|
||||
_validate_schema(schema)
|
||||
print(f"\n[Anthropic Haiku XPath] Generated schema: {json.dumps(schema, indent=2)}")
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
not os.getenv("CRAWL4AI_ANTHROPIC_KEY"),
|
||||
reason="CRAWL4AI_ANTHROPIC_KEY not set",
|
||||
)
|
||||
async def test_anthropic_no_query(self):
|
||||
"""Anthropic with no query — should auto-detect schema from page structure."""
|
||||
schema = await JsonCssExtractionStrategy.agenerate_schema(
|
||||
url=TEST_URL,
|
||||
schema_type="CSS",
|
||||
llm_config=LLMConfig(
|
||||
provider="anthropic/claude-haiku-4-5",
|
||||
api_token=os.getenv("CRAWL4AI_ANTHROPIC_KEY"),
|
||||
),
|
||||
)
|
||||
_validate_schema(schema)
|
||||
print(f"\n[Anthropic Haiku no-query] Generated schema: {json.dumps(schema, indent=2)}")
|
||||
|
||||
|
||||
class TestRealOpenAISchemaGeneration:
|
||||
"""OpenAI models — should still work as before (regression check)."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
not os.getenv("CRAWL4AI_OPENAI_KEY"),
|
||||
reason="CRAWL4AI_OPENAI_KEY not set",
|
||||
)
|
||||
async def test_openai_gpt4o_mini_css_schema(self):
|
||||
"""OpenAI gpt-4o-mini with CSS — this already worked, must not regress."""
|
||||
schema = await JsonCssExtractionStrategy.agenerate_schema(
|
||||
url=TEST_URL,
|
||||
schema_type="CSS",
|
||||
query="Extract all quotes with their text, author, and tags",
|
||||
llm_config=LLMConfig(
|
||||
provider="openai/gpt-4o-mini",
|
||||
api_token=os.getenv("CRAWL4AI_OPENAI_KEY"),
|
||||
),
|
||||
)
|
||||
_validate_schema(schema)
|
||||
print(f"\n[OpenAI gpt-4o-mini CSS] Generated schema: {json.dumps(schema, indent=2)}")
|
||||
|
||||
|
||||
class TestRealGroqSchemaGeneration:
|
||||
"""Groq with the updated model name."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.skipif(
|
||||
not os.getenv("CRAWL4AI_GROQ_KEY") and not os.getenv("GROQ_API_KEY"),
|
||||
reason="No Groq API key set",
|
||||
)
|
||||
async def test_groq_llama33_css_schema(self):
|
||||
"""Groq with llama-3.3-70b-versatile (replacement for decommissioned 3.1)."""
|
||||
api_key = os.getenv("CRAWL4AI_GROQ_KEY") or os.getenv("GROQ_API_KEY")
|
||||
schema = await JsonCssExtractionStrategy.agenerate_schema(
|
||||
url=TEST_URL,
|
||||
schema_type="CSS",
|
||||
query="Extract all quotes with their text, author, and tags",
|
||||
llm_config=LLMConfig(
|
||||
provider="groq/llama-3.3-70b-versatile",
|
||||
api_token=api_key,
|
||||
),
|
||||
)
|
||||
_validate_schema(schema)
|
||||
print(f"\n[Groq llama-3.3] Generated schema: {json.dumps(schema, indent=2)}")
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Regression: ensure _strip_markdown_fences doesn't break valid JSON
|
||||
# ===========================================================================
|
||||
|
||||
|
||||
class TestRegressionNoBreakage:
|
||||
"""Ensure the fix doesn't break any currently-working JSON formats."""
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw_json",
|
||||
[
|
||||
'{"simple": true}',
|
||||
'[]',
|
||||
'[{"a": 1}, {"a": 2}]',
|
||||
'{"nested": {"deep": {"value": 42}}}',
|
||||
'{"unicode": "\u3053\u3093\u306b\u3061\u306f\u4e16\u754c"}',
|
||||
'{"special": "line1\\nline2\\ttab"}',
|
||||
'{"url": "https://example.com/path?q=1&b=2"}',
|
||||
json.dumps(SIMPLE_SCHEMA),
|
||||
json.dumps(NESTED_SCHEMA),
|
||||
json.dumps(NESTED_SCHEMA, indent=2),
|
||||
json.dumps(NESTED_SCHEMA, indent=4),
|
||||
],
|
||||
ids=[
|
||||
"simple_object",
|
||||
"empty_array",
|
||||
"array_of_objects",
|
||||
"deeply_nested",
|
||||
"unicode_content",
|
||||
"escape_sequences",
|
||||
"url_in_value",
|
||||
"simple_schema_compact",
|
||||
"nested_schema_compact",
|
||||
"nested_schema_indent2",
|
||||
"nested_schema_indent4",
|
||||
],
|
||||
)
|
||||
def test_clean_json_unchanged(self, raw_json):
|
||||
"""Already-clean JSON must parse identically after stripping."""
|
||||
original = json.loads(raw_json)
|
||||
after_strip = json.loads(_strip_markdown_fences(raw_json))
|
||||
assert after_strip == original
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"raw_json",
|
||||
[
|
||||
'{"simple": true}',
|
||||
'[]',
|
||||
'[{"a": 1}, {"a": 2}]',
|
||||
json.dumps(SIMPLE_SCHEMA),
|
||||
json.dumps(NESTED_SCHEMA, indent=2),
|
||||
],
|
||||
ids=[
|
||||
"simple_object",
|
||||
"empty_array",
|
||||
"array_of_objects",
|
||||
"simple_schema",
|
||||
"nested_schema",
|
||||
],
|
||||
)
|
||||
def test_fenced_json_matches_clean(self, raw_json):
|
||||
"""Fenced version of any JSON must parse to the same value as clean."""
|
||||
original = json.loads(raw_json)
|
||||
fenced = f"```json\n{raw_json}\n```"
|
||||
after_strip = json.loads(_strip_markdown_fences(fenced))
|
||||
assert after_strip == original
|
||||
@@ -0,0 +1,85 @@
|
||||
import sys
|
||||
import os
|
||||
|
||||
# Get the grandparent directory
|
||||
grandparent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
sys.path.append(grandparent_dir)
|
||||
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
|
||||
|
||||
import asyncio
|
||||
from crawl4ai.deep_crawling.filters import URLPatternFilter
|
||||
|
||||
|
||||
def test_prefix_boundary_matching():
|
||||
"""Test that prefix patterns respect path boundaries"""
|
||||
print("=== Testing URLPatternFilter Prefix Boundary Fix ===")
|
||||
|
||||
filter_obj = URLPatternFilter(patterns=['https://langchain-ai.github.io/langgraph/*'])
|
||||
|
||||
test_cases = [
|
||||
('https://langchain-ai.github.io/langgraph/', True),
|
||||
('https://langchain-ai.github.io/langgraph/concepts/', True),
|
||||
('https://langchain-ai.github.io/langgraph/tutorials/', True),
|
||||
('https://langchain-ai.github.io/langgraph?param=1', True),
|
||||
('https://langchain-ai.github.io/langgraph#section', True),
|
||||
('https://langchain-ai.github.io/langgraphjs/', False),
|
||||
('https://langchain-ai.github.io/langgraphjs/concepts/', False),
|
||||
('https://other-site.com/langgraph/', False),
|
||||
]
|
||||
|
||||
all_passed = True
|
||||
for url, expected in test_cases:
|
||||
result = filter_obj.apply(url)
|
||||
status = "PASS" if result == expected else "FAIL"
|
||||
if result != expected:
|
||||
all_passed = False
|
||||
print(f"{status:4} | Expected: {expected:5} | Got: {result:5} | {url}")
|
||||
|
||||
return all_passed
|
||||
|
||||
|
||||
def test_edge_cases():
|
||||
"""Test edge cases for path boundary matching"""
|
||||
print("\n=== Testing Edge Cases ===")
|
||||
|
||||
test_patterns = [
|
||||
('/api/*', [
|
||||
('/api/', True),
|
||||
('/api/v1', True),
|
||||
('/api?param=1', True),
|
||||
('/apiv2/', False),
|
||||
('/api_old/', False),
|
||||
]),
|
||||
|
||||
('*/docs/*', [
|
||||
('example.com/docs/', True),
|
||||
('example.com/docs/guide', True),
|
||||
('example.com/documentation/', False),
|
||||
('example.com/docs_old/', False),
|
||||
]),
|
||||
]
|
||||
|
||||
all_passed = True
|
||||
for pattern, test_cases in test_patterns:
|
||||
print(f"\nPattern: {pattern}")
|
||||
filter_obj = URLPatternFilter(patterns=[pattern])
|
||||
|
||||
for url, expected in test_cases:
|
||||
result = filter_obj.apply(url)
|
||||
status = "PASS" if result == expected else "FAIL"
|
||||
if result != expected:
|
||||
all_passed = False
|
||||
print(f" {status:4} | Expected: {expected:5} | Got: {result:5} | {url}")
|
||||
|
||||
return all_passed
|
||||
|
||||
if __name__ == "__main__":
|
||||
test1_passed = test_prefix_boundary_matching()
|
||||
test2_passed = test_edge_cases()
|
||||
|
||||
if test1_passed and test2_passed:
|
||||
print("\n✅ All tests passed!")
|
||||
sys.exit(0)
|
||||
else:
|
||||
print("\n❌ Some tests failed!")
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,32 @@
|
||||
import asyncio
|
||||
import pytest
|
||||
from crawl4ai import AsyncLogger, AsyncUrlSeeder, SeedingConfig
|
||||
from pathlib import Path
|
||||
import httpx
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sitemap_source_does_not_hit_commoncrawl():
|
||||
config = SeedingConfig(
|
||||
source="sitemap",
|
||||
live_check=False,
|
||||
extract_head=False,
|
||||
max_urls=50,
|
||||
verbose=True,
|
||||
force=False
|
||||
)
|
||||
|
||||
async with AsyncUrlSeeder(logger=AsyncLogger(verbose=True)) as seeder:
|
||||
async def boom(*args, **kwargs):
|
||||
print("DEBUG: _latest_index called")
|
||||
raise httpx.ConnectTimeout("Simulated CommonCrawl outage")
|
||||
|
||||
seeder._latest_index = boom
|
||||
try:
|
||||
await seeder.urls("https://docs.crawl4ai.com/", config)
|
||||
print("PASS: _latest_index was NOT called (expected after fix).")
|
||||
except httpx.ConnectTimeout:
|
||||
print("FAIL: _latest_index WAS called even though source='sitemap'.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(test_sitemap_source_does_not_hit_commoncrawl())
|
||||
@@ -0,0 +1,62 @@
|
||||
import asyncio
|
||||
from crawl4ai import *
|
||||
|
||||
async def test_real_websites():
|
||||
print("\n=== Testing Real Website Robots.txt Compliance ===\n")
|
||||
|
||||
browser_config = BrowserConfig(headless=True, verbose=True)
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
|
||||
# Test cases with URLs
|
||||
test_cases = [
|
||||
# Public sites that should be allowed
|
||||
("https://example.com", True), # Simple public site
|
||||
("https://httpbin.org/get", True), # API endpoint
|
||||
|
||||
# Sites with known strict robots.txt
|
||||
("https://www.facebook.com/robots.txt", False), # Social media
|
||||
("https://www.google.com/search", False), # Search pages
|
||||
|
||||
# Edge cases
|
||||
("https://api.github.com", True), # API service
|
||||
("https://raw.githubusercontent.com", True), # Content delivery
|
||||
|
||||
# Non-existent/error cases
|
||||
("https://thisisnotarealwebsite.com", True), # Non-existent domain
|
||||
("https://localhost:12345", True), # Invalid port
|
||||
]
|
||||
|
||||
for url, expected in test_cases:
|
||||
print(f"\nTesting: {url}")
|
||||
try:
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
check_robots_txt=True, # Enable robots.txt checking
|
||||
verbose=True
|
||||
)
|
||||
|
||||
result = await crawler.arun(url=url, config=config)
|
||||
allowed = result.success and not result.error_message
|
||||
|
||||
print(f"Expected: {'allowed' if expected else 'denied'}")
|
||||
print(f"Actual: {'allowed' if allowed else 'denied'}")
|
||||
print(f"Status Code: {result.status_code}")
|
||||
if result.error_message:
|
||||
print(f"Error: {result.error_message}")
|
||||
|
||||
# Optional: Print robots.txt content if available
|
||||
if result.metadata and 'robots_txt' in result.metadata:
|
||||
print(f"Robots.txt rules:\n{result.metadata['robots_txt']}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"Test failed with error: {str(e)}")
|
||||
|
||||
async def main():
|
||||
try:
|
||||
await test_real_websites()
|
||||
except Exception as e:
|
||||
print(f"Test suite failed: {str(e)}")
|
||||
raise
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user