chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,235 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# 🚀 Crawl4AI v0.3.72 Release Announcement\n",
|
||||
"\n",
|
||||
"Welcome to the new release of **Crawl4AI v0.3.72**! This notebook highlights the latest features and demonstrates how they work in real-time. Follow along to see each feature in action!\n",
|
||||
"\n",
|
||||
"### What’s New?\n",
|
||||
"- ✨ `Fit Markdown`: Extracts only the main content from articles and blogs\n",
|
||||
"- 🛡️ **Magic Mode**: Comprehensive anti-bot detection bypass\n",
|
||||
"- 🌐 **Multi-browser support**: Switch between Chromium, Firefox, WebKit\n",
|
||||
"- 🔍 **Knowledge Graph Extraction**: Generate structured graphs of entities & relationships from any URL\n",
|
||||
"- 🤖 **Crawl4AI GPT Assistant**: Chat directly with our AI assistant for help, code generation, and faster learning (available [here](https://tinyurl.com/your-gpt-assistant-link))\n",
|
||||
"\n",
|
||||
"---\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 📥 Setup\n",
|
||||
"To start, we'll install `Crawl4AI` along with Playwright and `nest_asyncio` to ensure compatibility with Colab’s asynchronous environment."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Install Crawl4AI and dependencies\n",
|
||||
"!pip install crawl4ai\n",
|
||||
"!playwright install\n",
|
||||
"!pip install nest_asyncio"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Import nest_asyncio and apply it to allow asyncio in Colab\n",
|
||||
"import nest_asyncio\n",
|
||||
"nest_asyncio.apply()\n",
|
||||
"\n",
|
||||
"print('Setup complete!')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"---\n",
|
||||
"\n",
|
||||
"## ✨ Feature 1: `Fit Markdown`\n",
|
||||
"Extracts only the main content from articles and blog pages, removing sidebars, ads, and other distractions.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import asyncio\n",
|
||||
"from crawl4ai import AsyncWebCrawler\n",
|
||||
"\n",
|
||||
"async def fit_markdown_demo():\n",
|
||||
" async with AsyncWebCrawler() as crawler:\n",
|
||||
" result = await crawler.arun(url=\"https://janineintheworld.com/places-to-visit-in-central-mexico\")\n",
|
||||
" print(result.fit_markdown) # Shows main content in Markdown format\n",
|
||||
"\n",
|
||||
"# Run the demo\n",
|
||||
"await fit_markdown_demo()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"---\n",
|
||||
"\n",
|
||||
"## 🛡️ Feature 2: Magic Mode\n",
|
||||
"Magic Mode bypasses anti-bot detection to make crawling more reliable on protected websites.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"async def magic_mode_demo():\n",
|
||||
" async with AsyncWebCrawler() as crawler: # Enables anti-bot detection bypass\n",
|
||||
" result = await crawler.arun(\n",
|
||||
" url=\"https://www.reuters.com/markets/us/global-markets-view-usa-pix-2024-08-29/\",\n",
|
||||
" magic=True # Enables magic mode\n",
|
||||
" )\n",
|
||||
" print(result.markdown) # Shows the full content in Markdown format\n",
|
||||
"\n",
|
||||
"# Run the demo\n",
|
||||
"await magic_mode_demo()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"---\n",
|
||||
"\n",
|
||||
"## 🌐 Feature 3: Multi-Browser Support\n",
|
||||
"Crawl4AI now supports Chromium, Firefox, and WebKit. Here’s how to specify Firefox for a crawl.\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"async def multi_browser_demo():\n",
|
||||
" async with AsyncWebCrawler(browser_type=\"firefox\") as crawler: # Using Firefox instead of default Chromium\n",
|
||||
" result = await crawler.arun(url=\"https://crawl4i.com\")\n",
|
||||
" print(result.markdown) # Shows content extracted using Firefox\n",
|
||||
"\n",
|
||||
"# Run the demo\n",
|
||||
"await multi_browser_demo()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"---\n",
|
||||
"\n",
|
||||
"## ✨ Put them all together\n",
|
||||
"\n",
|
||||
"Let's combine all the features to extract the main content from a blog post, bypass anti-bot detection, and generate a knowledge graph from the extracted content."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from crawl4ai import LLMExtractionStrategy\n",
|
||||
"from pydantic import BaseModel\n",
|
||||
"import json, os\n",
|
||||
"from typing import List\n",
|
||||
"\n",
|
||||
"# Define classes for the knowledge graph structure\n",
|
||||
"class Landmark(BaseModel):\n",
|
||||
" name: str\n",
|
||||
" description: str\n",
|
||||
" activities: list[str] # E.g., visiting, sightseeing, relaxing\n",
|
||||
"\n",
|
||||
"class City(BaseModel):\n",
|
||||
" name: str\n",
|
||||
" description: str\n",
|
||||
" landmarks: list[Landmark]\n",
|
||||
" cultural_highlights: list[str] # E.g., food, music, traditional crafts\n",
|
||||
"\n",
|
||||
"class TravelKnowledgeGraph(BaseModel):\n",
|
||||
" cities: list[City] # Central Mexican cities to visit\n",
|
||||
"\n",
|
||||
"async def combined_demo():\n",
|
||||
" # Define the knowledge graph extraction strategy\n",
|
||||
" strategy = LLMExtractionStrategy(\n",
|
||||
" # provider=\"ollama/nemotron\",\n",
|
||||
" provider='openai/gpt-4o-mini', # Or any other provider, including Ollama and open source models\n",
|
||||
" pi_token=os.getenv('OPENAI_API_KEY'), # In case of Ollama just pass \"no-token\"\n",
|
||||
" schema=TravelKnowledgeGraph.schema(),\n",
|
||||
" instruction=(\n",
|
||||
" \"Extract cities, landmarks, and cultural highlights for places to visit in Central Mexico. \"\n",
|
||||
" \"For each city, list main landmarks with descriptions and activities, as well as cultural highlights.\"\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # Set up the AsyncWebCrawler with multi-browser support, Magic Mode, and Fit Markdown\n",
|
||||
" async with AsyncWebCrawler(browser_type=\"firefox\") as crawler:\n",
|
||||
" result = await crawler.arun(\n",
|
||||
" url=\"https://janineintheworld.com/places-to-visit-in-central-mexico\",\n",
|
||||
" extraction_strategy=strategy,\n",
|
||||
" bypass_cache=True,\n",
|
||||
" magic=True\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" # Display main article content in Fit Markdown format\n",
|
||||
" print(\"Extracted Main Content:\\n\", result.fit_markdown)\n",
|
||||
" \n",
|
||||
" # Display extracted knowledge graph of cities, landmarks, and cultural highlights\n",
|
||||
" if result.extracted_content:\n",
|
||||
" travel_graph = json.loads(result.extracted_content)\n",
|
||||
" print(\"\\nExtracted Knowledge Graph:\\n\", json.dumps(travel_graph, indent=2))\n",
|
||||
"\n",
|
||||
"# Run the combined demo\n",
|
||||
"await combined_demo()\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"---\n",
|
||||
"\n",
|
||||
"## 🤖 Crawl4AI GPT Assistant\n",
|
||||
"Chat with the Crawl4AI GPT Assistant for code generation, support, and learning Crawl4AI faster. Try it out [here](https://tinyurl.com/crawl4ai-gpt)!"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": []
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3",
|
||||
"language": "python",
|
||||
"name": "python3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python",
|
||||
"version": "3.9"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,408 @@
|
||||
"""
|
||||
🚀 Crawl4AI v0.7.0 Release Demo
|
||||
================================
|
||||
This demo showcases all major features introduced in v0.7.0 release.
|
||||
|
||||
Major Features:
|
||||
1. ✅ Adaptive Crawling - Intelligent crawling with confidence tracking
|
||||
2. ✅ Virtual Scroll Support - Handle infinite scroll pages
|
||||
3. ✅ Link Preview - Advanced link analysis with 3-layer scoring
|
||||
4. ✅ URL Seeder - Smart URL discovery and filtering
|
||||
5. ✅ C4A Script - Domain-specific language for web automation
|
||||
6. ✅ Chrome Extension Updates - Click2Crawl and instant schema extraction
|
||||
7. ✅ PDF Parsing Support - Extract content from PDF documents
|
||||
8. ✅ Nightly Builds - Automated nightly releases
|
||||
|
||||
Run this demo to see all features in action!
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from typing import List, Dict
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
from rich.panel import Panel
|
||||
from rich import box
|
||||
|
||||
from crawl4ai import (
|
||||
AsyncWebCrawler,
|
||||
CrawlerRunConfig,
|
||||
BrowserConfig,
|
||||
CacheMode,
|
||||
AdaptiveCrawler,
|
||||
AdaptiveConfig,
|
||||
AsyncUrlSeeder,
|
||||
SeedingConfig,
|
||||
c4a_compile,
|
||||
CompilationResult
|
||||
)
|
||||
from crawl4ai.async_configs import VirtualScrollConfig, LinkPreviewConfig
|
||||
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
|
||||
|
||||
console = Console()
|
||||
|
||||
def print_section(title: str, description: str = ""):
|
||||
"""Print a section header"""
|
||||
console.print(f"\n[bold cyan]{'=' * 60}[/bold cyan]")
|
||||
console.print(f"[bold yellow]{title}[/bold yellow]")
|
||||
if description:
|
||||
console.print(f"[dim]{description}[/dim]")
|
||||
console.print(f"[bold cyan]{'=' * 60}[/bold cyan]\n")
|
||||
|
||||
|
||||
async def demo_1_adaptive_crawling():
|
||||
"""Demo 1: Adaptive Crawling - Intelligent content extraction"""
|
||||
print_section(
|
||||
"Demo 1: Adaptive Crawling",
|
||||
"Intelligently learns and adapts to website patterns"
|
||||
)
|
||||
|
||||
# Create adaptive crawler with custom configuration
|
||||
config = AdaptiveConfig(
|
||||
strategy="statistical", # or "embedding"
|
||||
confidence_threshold=0.7,
|
||||
max_pages=10,
|
||||
top_k_links=3,
|
||||
min_gain_threshold=0.1
|
||||
)
|
||||
|
||||
# Example: Learn from a product page
|
||||
console.print("[cyan]Learning from product page patterns...[/cyan]")
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
adaptive = AdaptiveCrawler(crawler, config)
|
||||
|
||||
# Start adaptive crawl
|
||||
console.print("[cyan]Starting adaptive crawl...[/cyan]")
|
||||
result = await adaptive.digest(
|
||||
start_url="https://docs.python.org/3/",
|
||||
query="python decorators tutorial"
|
||||
)
|
||||
|
||||
console.print("[green]✓ Adaptive crawl completed[/green]")
|
||||
console.print(f" - Confidence Level: {adaptive.confidence:.0%}")
|
||||
console.print(f" - Pages Crawled: {len(result.crawled_urls)}")
|
||||
console.print(f" - Knowledge Base: {len(adaptive.state.knowledge_base)} documents")
|
||||
|
||||
# Get most relevant content
|
||||
relevant = adaptive.get_relevant_content(top_k=3)
|
||||
if relevant:
|
||||
console.print("\nMost relevant pages:")
|
||||
for i, page in enumerate(relevant, 1):
|
||||
console.print(f" {i}. {page['url']} (relevance: {page['score']:.2%})")
|
||||
|
||||
|
||||
async def demo_2_virtual_scroll():
|
||||
"""Demo 2: Virtual Scroll - Handle infinite scroll pages"""
|
||||
print_section(
|
||||
"Demo 2: Virtual Scroll Support",
|
||||
"Capture content from modern infinite scroll pages"
|
||||
)
|
||||
|
||||
# Configure virtual scroll - using body as container for example.com
|
||||
scroll_config = VirtualScrollConfig(
|
||||
container_selector="body", # Using body since example.com has simple structure
|
||||
scroll_count=3, # Just 3 scrolls for demo
|
||||
scroll_by="container_height", # or "page_height" or pixel value
|
||||
wait_after_scroll=0.5 # Wait 500ms after each scroll
|
||||
)
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
virtual_scroll_config=scroll_config,
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
wait_until="networkidle"
|
||||
)
|
||||
|
||||
console.print("[cyan]Virtual Scroll Configuration:[/cyan]")
|
||||
console.print(f" - Container: {scroll_config.container_selector}")
|
||||
console.print(f" - Scroll count: {scroll_config.scroll_count}")
|
||||
console.print(f" - Scroll by: {scroll_config.scroll_by}")
|
||||
console.print(f" - Wait after scroll: {scroll_config.wait_after_scroll}s")
|
||||
|
||||
console.print("\n[dim]Note: Using example.com for demo - in production, use this[/dim]")
|
||||
console.print("[dim]with actual infinite scroll pages like social media feeds.[/dim]\n")
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun(
|
||||
"https://example.com",
|
||||
config=config
|
||||
)
|
||||
|
||||
if result.success:
|
||||
console.print("[green]✓ Virtual scroll executed successfully![/green]")
|
||||
console.print(f" - Content length: {len(result.markdown)} chars")
|
||||
|
||||
# Show example of how to use with real infinite scroll sites
|
||||
console.print("\n[yellow]Example for real infinite scroll sites:[/yellow]")
|
||||
console.print("""
|
||||
# For Twitter-like feeds:
|
||||
scroll_config = VirtualScrollConfig(
|
||||
container_selector="[data-testid='primaryColumn']",
|
||||
scroll_count=20,
|
||||
scroll_by="container_height",
|
||||
wait_after_scroll=1.0
|
||||
)
|
||||
|
||||
# For Instagram-like grids:
|
||||
scroll_config = VirtualScrollConfig(
|
||||
container_selector="main article",
|
||||
scroll_count=15,
|
||||
scroll_by=1000, # Fixed pixel amount
|
||||
wait_after_scroll=1.5
|
||||
)""")
|
||||
|
||||
|
||||
async def demo_3_link_preview():
|
||||
"""Demo 3: Link Preview with 3-layer scoring"""
|
||||
print_section(
|
||||
"Demo 3: Link Preview & Scoring",
|
||||
"Advanced link analysis with intrinsic, contextual, and total scoring"
|
||||
)
|
||||
|
||||
# Configure link preview
|
||||
link_config = LinkPreviewConfig(
|
||||
include_internal=True,
|
||||
include_external=False,
|
||||
max_links=10,
|
||||
concurrency=5,
|
||||
query="python tutorial", # For contextual scoring
|
||||
score_threshold=0.3,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
link_preview_config=link_config,
|
||||
score_links=True, # Enable intrinsic scoring
|
||||
cache_mode=CacheMode.BYPASS
|
||||
)
|
||||
|
||||
console.print("[cyan]Analyzing links with 3-layer scoring system...[/cyan]")
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun("https://docs.python.org/3/", config=config)
|
||||
|
||||
if result.success and result.links:
|
||||
# Get scored links
|
||||
internal_links = result.links.get("internal", [])
|
||||
scored_links = [l for l in internal_links if l.get("total_score")]
|
||||
scored_links.sort(key=lambda x: x.get("total_score", 0), reverse=True)
|
||||
|
||||
# Create a scoring table
|
||||
table = Table(title="Link Scoring Results", box=box.ROUNDED)
|
||||
table.add_column("Link Text", style="cyan", width=40)
|
||||
table.add_column("Intrinsic Score", justify="center")
|
||||
table.add_column("Contextual Score", justify="center")
|
||||
table.add_column("Total Score", justify="center", style="bold green")
|
||||
|
||||
for link in scored_links[:5]:
|
||||
text = link.get('text', 'No text')[:40]
|
||||
table.add_row(
|
||||
text,
|
||||
f"{link.get('intrinsic_score', 0):.1f}/10",
|
||||
f"{link.get('contextual_score', 0):.2f}/1",
|
||||
f"{link.get('total_score', 0):.3f}"
|
||||
)
|
||||
|
||||
console.print(table)
|
||||
|
||||
|
||||
async def demo_4_url_seeder():
|
||||
"""Demo 4: URL Seeder - Smart URL discovery"""
|
||||
print_section(
|
||||
"Demo 4: URL Seeder",
|
||||
"Intelligent URL discovery and filtering"
|
||||
)
|
||||
|
||||
# Configure seeding
|
||||
seeding_config = SeedingConfig(
|
||||
source="cc+sitemap", # or "crawl"
|
||||
pattern="*tutorial*", # URL pattern filter
|
||||
max_urls=50,
|
||||
extract_head=True, # Get metadata
|
||||
query="python programming", # For relevance scoring
|
||||
scoring_method="bm25",
|
||||
score_threshold=0.2,
|
||||
force = True
|
||||
)
|
||||
|
||||
console.print("[cyan]URL Seeder Configuration:[/cyan]")
|
||||
console.print(f" - Source: {seeding_config.source}")
|
||||
console.print(f" - Pattern: {seeding_config.pattern}")
|
||||
console.print(f" - Max URLs: {seeding_config.max_urls}")
|
||||
console.print(f" - Query: {seeding_config.query}")
|
||||
console.print(f" - Scoring: {seeding_config.scoring_method}")
|
||||
|
||||
# Use URL seeder to discover URLs
|
||||
async with AsyncUrlSeeder() as seeder:
|
||||
console.print("\n[cyan]Discovering URLs from Python docs...[/cyan]")
|
||||
urls = await seeder.urls("docs.python.org", seeding_config)
|
||||
|
||||
console.print(f"\n[green]✓ Discovered {len(urls)} URLs[/green]")
|
||||
for i, url_info in enumerate(urls[:5], 1):
|
||||
console.print(f" {i}. {url_info['url']}")
|
||||
if url_info.get('relevance_score'):
|
||||
console.print(f" Relevance: {url_info['relevance_score']:.3f}")
|
||||
|
||||
|
||||
async def demo_5_c4a_script():
|
||||
"""Demo 5: C4A Script - Domain-specific language"""
|
||||
print_section(
|
||||
"Demo 5: C4A Script Language",
|
||||
"Domain-specific language for web automation"
|
||||
)
|
||||
|
||||
# Example C4A script
|
||||
c4a_script = """
|
||||
# Simple C4A script example
|
||||
WAIT `body` 3
|
||||
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
|
||||
CLICK `.search-button`
|
||||
TYPE "python tutorial"
|
||||
PRESS Enter
|
||||
WAIT `.results` 5
|
||||
"""
|
||||
|
||||
console.print("[cyan]C4A Script Example:[/cyan]")
|
||||
console.print(Panel(c4a_script, title="script.c4a", border_style="blue"))
|
||||
|
||||
# Compile the script
|
||||
compilation_result = c4a_compile(c4a_script)
|
||||
|
||||
if compilation_result.success:
|
||||
console.print("[green]✓ Script compiled successfully![/green]")
|
||||
console.print(f" - Generated {len(compilation_result.js_code)} JavaScript statements")
|
||||
console.print("\nFirst 3 JS statements:")
|
||||
for stmt in compilation_result.js_code[:3]:
|
||||
console.print(f" • {stmt}")
|
||||
else:
|
||||
console.print("[red]✗ Script compilation failed[/red]")
|
||||
if compilation_result.first_error:
|
||||
error = compilation_result.first_error
|
||||
console.print(f" Error at line {error.line}: {error.message}")
|
||||
|
||||
|
||||
async def demo_6_css_extraction():
|
||||
"""Demo 6: Enhanced CSS/JSON extraction"""
|
||||
print_section(
|
||||
"Demo 6: Enhanced Extraction",
|
||||
"Improved CSS selector and JSON extraction"
|
||||
)
|
||||
|
||||
# Define extraction schema
|
||||
schema = {
|
||||
"name": "Example Page Data",
|
||||
"baseSelector": "body",
|
||||
"fields": [
|
||||
{
|
||||
"name": "title",
|
||||
"selector": "h1",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "paragraphs",
|
||||
"selector": "p",
|
||||
"type": "list",
|
||||
"fields": [
|
||||
{"name": "text", "type": "text"}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
extraction_strategy = JsonCssExtractionStrategy(schema)
|
||||
|
||||
console.print("[cyan]Extraction Schema:[/cyan]")
|
||||
console.print(json.dumps(schema, indent=2))
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun(
|
||||
"https://example.com",
|
||||
config=CrawlerRunConfig(
|
||||
extraction_strategy=extraction_strategy,
|
||||
cache_mode=CacheMode.BYPASS
|
||||
)
|
||||
)
|
||||
|
||||
if result.success and result.extracted_content:
|
||||
console.print("\n[green]✓ Content extracted successfully![/green]")
|
||||
console.print(f"Extracted: {json.dumps(json.loads(result.extracted_content), indent=2)[:200]}...")
|
||||
|
||||
|
||||
async def demo_7_performance_improvements():
|
||||
"""Demo 7: Performance improvements"""
|
||||
print_section(
|
||||
"Demo 7: Performance Improvements",
|
||||
"Faster crawling with better resource management"
|
||||
)
|
||||
|
||||
# Performance-optimized configuration
|
||||
config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.ENABLED, # Use caching
|
||||
wait_until="domcontentloaded", # Faster than networkidle
|
||||
page_timeout=10000, # 10 second timeout
|
||||
exclude_external_links=True,
|
||||
exclude_social_media_links=True,
|
||||
exclude_external_images=True
|
||||
)
|
||||
|
||||
console.print("[cyan]Performance Configuration:[/cyan]")
|
||||
console.print(" - Cache: ENABLED")
|
||||
console.print(" - Wait: domcontentloaded (faster)")
|
||||
console.print(" - Timeout: 10s")
|
||||
console.print(" - Excluding: external links, images, social media")
|
||||
|
||||
# Measure performance
|
||||
import time
|
||||
start_time = time.time()
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun("https://example.com", config=config)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
if result.success:
|
||||
console.print(f"\n[green]✓ Page crawled in {elapsed:.2f} seconds[/green]")
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all demos"""
|
||||
console.print(Panel(
|
||||
"[bold cyan]Crawl4AI v0.7.0 Release Demo[/bold cyan]\n\n"
|
||||
"This demo showcases all major features introduced in v0.7.0.\n"
|
||||
"Each demo is self-contained and demonstrates a specific feature.",
|
||||
title="Welcome",
|
||||
border_style="blue"
|
||||
))
|
||||
|
||||
demos = [
|
||||
demo_1_adaptive_crawling,
|
||||
demo_2_virtual_scroll,
|
||||
demo_3_link_preview,
|
||||
demo_4_url_seeder,
|
||||
demo_5_c4a_script,
|
||||
demo_6_css_extraction,
|
||||
demo_7_performance_improvements
|
||||
]
|
||||
|
||||
for i, demo in enumerate(demos, 1):
|
||||
try:
|
||||
await demo()
|
||||
if i < len(demos):
|
||||
console.print("\n[dim]Press Enter to continue to next demo...[/dim]")
|
||||
input()
|
||||
except Exception as e:
|
||||
console.print(f"[red]Error in demo: {e}[/red]")
|
||||
continue
|
||||
|
||||
console.print(Panel(
|
||||
"[bold green]Demo Complete![/bold green]\n\n"
|
||||
"Thank you for trying Crawl4AI v0.7.0!\n"
|
||||
"For more examples and documentation, visit:\n"
|
||||
"https://github.com/unclecode/crawl4ai",
|
||||
title="Complete",
|
||||
border_style="green"
|
||||
))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,338 @@
|
||||
"""
|
||||
🚀 Crawl4AI v0.7.5 Release Demo - Working Examples
|
||||
==================================================
|
||||
This demo showcases key features introduced in v0.7.5 with real, executable examples.
|
||||
|
||||
Featured Demos:
|
||||
1. ✅ Docker Hooks System - Real API calls with custom hooks (string & function-based)
|
||||
2. ✅ Enhanced LLM Integration - Working LLM configurations
|
||||
3. ✅ HTTPS Preservation - Live crawling with HTTPS maintenance
|
||||
|
||||
Requirements:
|
||||
- crawl4ai v0.7.5 installed
|
||||
- Docker running with crawl4ai image (optional for Docker demos)
|
||||
- Valid API keys for LLM demos (optional)
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import requests
|
||||
import time
|
||||
import sys
|
||||
|
||||
from crawl4ai import (AsyncWebCrawler, CrawlerRunConfig, BrowserConfig,
|
||||
CacheMode, FilterChain, URLPatternFilter, BFSDeepCrawlStrategy,
|
||||
hooks_to_string)
|
||||
from crawl4ai.docker_client import Crawl4aiDockerClient
|
||||
|
||||
|
||||
def print_section(title: str, description: str = ""):
|
||||
"""Print a section header"""
|
||||
print(f"\n{'=' * 60}")
|
||||
print(f"{title}")
|
||||
if description:
|
||||
print(f"{description}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
|
||||
async def demo_1_docker_hooks_system():
|
||||
"""Demo 1: Docker Hooks System - Real API calls with custom hooks"""
|
||||
print_section(
|
||||
"Demo 1: Docker Hooks System",
|
||||
"Testing both string-based and function-based hooks (NEW in v0.7.5!)"
|
||||
)
|
||||
|
||||
# Check Docker service availability
|
||||
def check_docker_service():
|
||||
try:
|
||||
response = requests.get("http://localhost:11235/", timeout=3)
|
||||
return response.status_code == 200
|
||||
except:
|
||||
return False
|
||||
|
||||
print("Checking Docker service...")
|
||||
docker_running = check_docker_service()
|
||||
|
||||
if not docker_running:
|
||||
print("⚠️ Docker service not running on localhost:11235")
|
||||
print("To test Docker hooks:")
|
||||
print("1. Run: docker run -p 11235:11235 unclecode/crawl4ai:latest")
|
||||
print("2. Wait for service to start")
|
||||
print("3. Re-run this demo\n")
|
||||
return
|
||||
|
||||
print("✓ Docker service detected!")
|
||||
|
||||
# ============================================================================
|
||||
# PART 1: Traditional String-Based Hooks (Works with REST API)
|
||||
# ============================================================================
|
||||
print("\n" + "─" * 60)
|
||||
print("Part 1: String-Based Hooks (REST API)")
|
||||
print("─" * 60)
|
||||
|
||||
hooks_config_string = {
|
||||
"on_page_context_created": """
|
||||
async def hook(page, context, **kwargs):
|
||||
print("[String Hook] Setting up page context")
|
||||
await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
|
||||
return page
|
||||
""",
|
||||
"before_retrieve_html": """
|
||||
async def hook(page, context, **kwargs):
|
||||
print("[String Hook] Before retrieving HTML")
|
||||
await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
||||
await page.wait_for_timeout(1000)
|
||||
return page
|
||||
"""
|
||||
}
|
||||
|
||||
payload = {
|
||||
"urls": ["https://httpbin.org/html"],
|
||||
"hooks": {
|
||||
"code": hooks_config_string,
|
||||
"timeout": 30
|
||||
}
|
||||
}
|
||||
|
||||
print("🔧 Using string-based hooks for REST API...")
|
||||
try:
|
||||
start_time = time.time()
|
||||
response = requests.post("http://localhost:11235/crawl", json=payload, timeout=60)
|
||||
execution_time = time.time() - start_time
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
print(f"✅ String-based hooks executed in {execution_time:.2f}s")
|
||||
if result.get('results') and result['results'][0].get('success'):
|
||||
html_length = len(result['results'][0].get('html', ''))
|
||||
print(f" 📄 HTML length: {html_length} characters")
|
||||
else:
|
||||
print(f"❌ Request failed: {response.status_code}")
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {str(e)}")
|
||||
|
||||
# ============================================================================
|
||||
# PART 2: NEW Function-Based Hooks with Docker Client (v0.7.5)
|
||||
# ============================================================================
|
||||
print("\n" + "─" * 60)
|
||||
print("Part 2: Function-Based Hooks with Docker Client (✨ NEW!)")
|
||||
print("─" * 60)
|
||||
|
||||
# Define hooks as regular Python functions
|
||||
async def on_page_context_created_func(page, context, **kwargs):
|
||||
"""Block images to speed up crawling"""
|
||||
print("[Function Hook] Setting up page context")
|
||||
await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
|
||||
await page.set_viewport_size({"width": 1920, "height": 1080})
|
||||
return page
|
||||
|
||||
async def before_goto_func(page, context, url, **kwargs):
|
||||
"""Add custom headers before navigation"""
|
||||
print(f"[Function Hook] About to navigate to {url}")
|
||||
await page.set_extra_http_headers({
|
||||
'X-Crawl4AI': 'v0.7.5-function-hooks',
|
||||
'X-Test-Header': 'demo'
|
||||
})
|
||||
return page
|
||||
|
||||
async def before_retrieve_html_func(page, context, **kwargs):
|
||||
"""Scroll to load lazy content"""
|
||||
print("[Function Hook] Scrolling page for lazy-loaded content")
|
||||
await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
||||
await page.wait_for_timeout(500)
|
||||
await page.evaluate("window.scrollTo(0, 0)")
|
||||
return page
|
||||
|
||||
# Use the hooks_to_string utility (can be used standalone)
|
||||
print("\n📦 Converting functions to strings with hooks_to_string()...")
|
||||
hooks_as_strings = hooks_to_string({
|
||||
"on_page_context_created": on_page_context_created_func,
|
||||
"before_goto": before_goto_func,
|
||||
"before_retrieve_html": before_retrieve_html_func
|
||||
})
|
||||
print(f" ✓ Converted {len(hooks_as_strings)} hooks to string format")
|
||||
|
||||
# OR use Docker Client which does conversion automatically!
|
||||
print("\n🐳 Using Docker Client with automatic conversion...")
|
||||
try:
|
||||
client = Crawl4aiDockerClient(base_url="http://localhost:11235")
|
||||
|
||||
# Pass function objects directly - conversion happens automatically!
|
||||
results = await client.crawl(
|
||||
urls=["https://httpbin.org/html"],
|
||||
hooks={
|
||||
"on_page_context_created": on_page_context_created_func,
|
||||
"before_goto": before_goto_func,
|
||||
"before_retrieve_html": before_retrieve_html_func
|
||||
},
|
||||
hooks_timeout=30
|
||||
)
|
||||
|
||||
if results and results.success:
|
||||
print(f"✅ Function-based hooks executed successfully!")
|
||||
print(f" 📄 HTML length: {len(results.html)} characters")
|
||||
print(f" 🎯 URL: {results.url}")
|
||||
else:
|
||||
print("⚠️ Crawl completed but may have warnings")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Docker client error: {str(e)}")
|
||||
|
||||
# Show the benefits
|
||||
print("\n" + "=" * 60)
|
||||
print("✨ Benefits of Function-Based Hooks:")
|
||||
print("=" * 60)
|
||||
print("✓ Full IDE support (autocomplete, syntax highlighting)")
|
||||
print("✓ Type checking and linting")
|
||||
print("✓ Easier to test and debug")
|
||||
print("✓ Reusable across projects")
|
||||
print("✓ Automatic conversion in Docker client")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
async def demo_2_enhanced_llm_integration():
|
||||
"""Demo 2: Enhanced LLM Integration - Working LLM configurations"""
|
||||
print_section(
|
||||
"Demo 2: Enhanced LLM Integration",
|
||||
"Testing custom LLM providers and configurations"
|
||||
)
|
||||
|
||||
print("🤖 Testing Enhanced LLM Integration Features")
|
||||
|
||||
provider = "gemini/gemini-2.5-flash-lite"
|
||||
payload = {
|
||||
"url": "https://example.com",
|
||||
"f": "llm",
|
||||
"q": "Summarize this page in one sentence.",
|
||||
"provider": provider, # Explicitly set provider
|
||||
"temperature": 0.7
|
||||
}
|
||||
try:
|
||||
response = requests.post(
|
||||
"http://localhost:11235/md",
|
||||
json=payload,
|
||||
timeout=60
|
||||
)
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
print(f"✓ Request successful with provider: {provider}")
|
||||
print(f" - Response keys: {list(result.keys())}")
|
||||
print(f" - Content length: {len(result.get('markdown', ''))} characters")
|
||||
print(f" - Note: Actual LLM call may fail without valid API key")
|
||||
else:
|
||||
print(f"❌ Request failed: {response.status_code}")
|
||||
print(f" - Response: {response.text[:500]}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"[red]Error: {e}[/]")
|
||||
|
||||
|
||||
async def demo_3_https_preservation():
|
||||
"""Demo 3: HTTPS Preservation - Live crawling with HTTPS maintenance"""
|
||||
print_section(
|
||||
"Demo 3: HTTPS Preservation",
|
||||
"Testing HTTPS preservation for internal links"
|
||||
)
|
||||
|
||||
print("🔒 Testing HTTPS Preservation Feature")
|
||||
|
||||
# Test with HTTPS preservation enabled
|
||||
print("\nTest 1: HTTPS Preservation ENABLED")
|
||||
|
||||
url_filter = URLPatternFilter(
|
||||
patterns=["^(https:\/\/)?quotes\.toscrape\.com(\/.*)?$"]
|
||||
)
|
||||
config = CrawlerRunConfig(
|
||||
exclude_external_links=True,
|
||||
stream=True,
|
||||
verbose=False,
|
||||
preserve_https_for_internal_links=True,
|
||||
deep_crawl_strategy=BFSDeepCrawlStrategy(
|
||||
max_depth=2,
|
||||
max_pages=5,
|
||||
filter_chain=FilterChain([url_filter])
|
||||
)
|
||||
)
|
||||
|
||||
test_url = "https://quotes.toscrape.com"
|
||||
print(f"🎯 Testing URL: {test_url}")
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
async for result in await crawler.arun(url=test_url, config=config):
|
||||
print("✓ HTTPS Preservation Test Completed")
|
||||
internal_links = [i['href'] for i in result.links['internal']]
|
||||
for link in internal_links:
|
||||
print(f" → {link}")
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all demos"""
|
||||
print("\n" + "=" * 60)
|
||||
print("🚀 Crawl4AI v0.7.5 Working Demo")
|
||||
print("=" * 60)
|
||||
|
||||
# Check system requirements
|
||||
print("🔍 System Requirements Check:")
|
||||
print(f" - Python version: {sys.version.split()[0]} {'✓' if sys.version_info >= (3, 10) else '❌ (3.10+ required)'}")
|
||||
|
||||
try:
|
||||
import requests
|
||||
print(f" - Requests library: ✓")
|
||||
except ImportError:
|
||||
print(f" - Requests library: ❌")
|
||||
|
||||
print()
|
||||
|
||||
demos = [
|
||||
("Docker Hooks System", demo_1_docker_hooks_system),
|
||||
("Enhanced LLM Integration", demo_2_enhanced_llm_integration),
|
||||
("HTTPS Preservation", demo_3_https_preservation),
|
||||
]
|
||||
|
||||
for i, (name, demo_func) in enumerate(demos, 1):
|
||||
try:
|
||||
print(f"\n📍 Starting Demo {i}/{len(demos)}: {name}")
|
||||
await demo_func()
|
||||
|
||||
if i < len(demos):
|
||||
print(f"\n✨ Demo {i} complete! Press Enter for next demo...")
|
||||
input()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print(f"\n⏹️ Demo interrupted by user")
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"❌ Demo {i} error: {str(e)}")
|
||||
print("Continuing to next demo...")
|
||||
continue
|
||||
|
||||
print("\n" + "=" * 60)
|
||||
print("🎉 Demo Complete!")
|
||||
print("=" * 60)
|
||||
print("You've experienced the power of Crawl4AI v0.7.5!")
|
||||
print("")
|
||||
print("Key Features Demonstrated:")
|
||||
print("🔧 Docker Hooks - String-based & function-based (NEW!)")
|
||||
print(" • hooks_to_string() utility for function conversion")
|
||||
print(" • Docker client with automatic conversion")
|
||||
print(" • Full IDE support and type checking")
|
||||
print("🤖 Enhanced LLM - Better AI integration")
|
||||
print("🔒 HTTPS Preservation - Secure link handling")
|
||||
print("")
|
||||
print("Ready to build something amazing? 🚀")
|
||||
print("")
|
||||
print("📖 Docs: https://docs.crawl4ai.com/")
|
||||
print("🐙 GitHub: https://github.com/unclecode/crawl4ai")
|
||||
print("=" * 60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("🚀 Crawl4AI v0.7.5 Live Demo Starting...")
|
||||
print("Press Ctrl+C anytime to exit\n")
|
||||
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\n👋 Demo stopped by user. Thanks for trying Crawl4AI v0.7.5!")
|
||||
except Exception as e:
|
||||
print(f"\n❌ Demo error: {str(e)}")
|
||||
print("Make sure you have the required dependencies installed.")
|
||||
@@ -0,0 +1,359 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Crawl4AI v0.7.6 Release Demo
|
||||
============================
|
||||
|
||||
This demo showcases the major feature in v0.7.6:
|
||||
**Webhook Support for Docker Job Queue API**
|
||||
|
||||
Features Demonstrated:
|
||||
1. Asynchronous job processing with webhook notifications
|
||||
2. Webhook support for /crawl/job endpoint
|
||||
3. Webhook support for /llm/job endpoint
|
||||
4. Notification-only vs data-in-payload modes
|
||||
5. Custom webhook headers for authentication
|
||||
6. Structured extraction with JSON schemas
|
||||
7. Exponential backoff retry for reliable delivery
|
||||
|
||||
Prerequisites:
|
||||
- Crawl4AI Docker container running on localhost:11235
|
||||
- Flask installed: pip install flask requests
|
||||
- LLM API key configured (for LLM examples)
|
||||
|
||||
Usage:
|
||||
python docs/releases_review/demo_v0.7.6.py
|
||||
"""
|
||||
|
||||
import requests
|
||||
import json
|
||||
import time
|
||||
from flask import Flask, request, jsonify
|
||||
from threading import Thread
|
||||
|
||||
# Configuration
|
||||
CRAWL4AI_BASE_URL = "http://localhost:11235"
|
||||
WEBHOOK_BASE_URL = "http://localhost:8080"
|
||||
|
||||
# Flask app for webhook receiver
|
||||
app = Flask(__name__)
|
||||
received_webhooks = []
|
||||
|
||||
|
||||
@app.route('/webhook', methods=['POST'])
|
||||
def webhook_handler():
|
||||
"""Universal webhook handler for both crawl and LLM extraction jobs."""
|
||||
payload = request.json
|
||||
task_id = payload['task_id']
|
||||
task_type = payload['task_type']
|
||||
status = payload['status']
|
||||
|
||||
print(f"\n{'='*70}")
|
||||
print(f"📬 Webhook Received!")
|
||||
print(f" Task ID: {task_id}")
|
||||
print(f" Task Type: {task_type}")
|
||||
print(f" Status: {status}")
|
||||
print(f" Timestamp: {payload['timestamp']}")
|
||||
|
||||
if status == 'completed':
|
||||
if 'data' in payload:
|
||||
print(f" ✅ Data included in webhook")
|
||||
if task_type == 'crawl':
|
||||
results = payload['data'].get('results', [])
|
||||
print(f" 📊 Crawled {len(results)} URL(s)")
|
||||
elif task_type == 'llm_extraction':
|
||||
extracted = payload['data'].get('extracted_content', {})
|
||||
print(f" 🤖 Extracted: {json.dumps(extracted, indent=6)}")
|
||||
else:
|
||||
print(f" 📥 Notification only (fetch data separately)")
|
||||
elif status == 'failed':
|
||||
print(f" ❌ Error: {payload.get('error', 'Unknown')}")
|
||||
|
||||
print(f"{'='*70}\n")
|
||||
received_webhooks.append(payload)
|
||||
|
||||
return jsonify({"status": "received"}), 200
|
||||
|
||||
|
||||
def start_webhook_server():
|
||||
"""Start Flask webhook server in background."""
|
||||
app.run(host='0.0.0.0', port=8080, debug=False, use_reloader=False)
|
||||
|
||||
|
||||
def demo_1_crawl_webhook_notification_only():
|
||||
"""Demo 1: Crawl job with webhook notification (data fetched separately)."""
|
||||
print("\n" + "="*70)
|
||||
print("DEMO 1: Crawl Job - Webhook Notification Only")
|
||||
print("="*70)
|
||||
print("Submitting crawl job with webhook notification...")
|
||||
|
||||
payload = {
|
||||
"urls": ["https://example.com"],
|
||||
"browser_config": {"headless": True},
|
||||
"crawler_config": {"cache_mode": "bypass"},
|
||||
"webhook_config": {
|
||||
"webhook_url": f"{WEBHOOK_BASE_URL}/webhook",
|
||||
"webhook_data_in_payload": False,
|
||||
"webhook_headers": {
|
||||
"X-Demo": "v0.7.6",
|
||||
"X-Type": "crawl"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
response = requests.post(f"{CRAWL4AI_BASE_URL}/crawl/job", json=payload)
|
||||
if response.ok:
|
||||
task_id = response.json()['task_id']
|
||||
print(f"✅ Job submitted: {task_id}")
|
||||
print("⏳ Webhook will notify when complete...")
|
||||
return task_id
|
||||
else:
|
||||
print(f"❌ Failed: {response.text}")
|
||||
return None
|
||||
|
||||
|
||||
def demo_2_crawl_webhook_with_data():
|
||||
"""Demo 2: Crawl job with full data in webhook payload."""
|
||||
print("\n" + "="*70)
|
||||
print("DEMO 2: Crawl Job - Webhook with Full Data")
|
||||
print("="*70)
|
||||
print("Submitting crawl job with data included in webhook...")
|
||||
|
||||
payload = {
|
||||
"urls": ["https://www.python.org"],
|
||||
"browser_config": {"headless": True},
|
||||
"crawler_config": {"cache_mode": "bypass"},
|
||||
"webhook_config": {
|
||||
"webhook_url": f"{WEBHOOK_BASE_URL}/webhook",
|
||||
"webhook_data_in_payload": True,
|
||||
"webhook_headers": {
|
||||
"X-Demo": "v0.7.6",
|
||||
"X-Type": "crawl-with-data"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
response = requests.post(f"{CRAWL4AI_BASE_URL}/crawl/job", json=payload)
|
||||
if response.ok:
|
||||
task_id = response.json()['task_id']
|
||||
print(f"✅ Job submitted: {task_id}")
|
||||
print("⏳ Webhook will include full results...")
|
||||
return task_id
|
||||
else:
|
||||
print(f"❌ Failed: {response.text}")
|
||||
return None
|
||||
|
||||
|
||||
def demo_3_llm_webhook_notification_only():
|
||||
"""Demo 3: LLM extraction with webhook notification (NEW in v0.7.6!)."""
|
||||
print("\n" + "="*70)
|
||||
print("DEMO 3: LLM Extraction - Webhook Notification Only (NEW!)")
|
||||
print("="*70)
|
||||
print("Submitting LLM extraction job with webhook notification...")
|
||||
|
||||
payload = {
|
||||
"url": "https://www.example.com",
|
||||
"q": "Extract the main heading and description from this page",
|
||||
"provider": "openai/gpt-4o-mini",
|
||||
"cache": False,
|
||||
"webhook_config": {
|
||||
"webhook_url": f"{WEBHOOK_BASE_URL}/webhook",
|
||||
"webhook_data_in_payload": False,
|
||||
"webhook_headers": {
|
||||
"X-Demo": "v0.7.6",
|
||||
"X-Type": "llm"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
response = requests.post(f"{CRAWL4AI_BASE_URL}/llm/job", json=payload)
|
||||
if response.ok:
|
||||
task_id = response.json()['task_id']
|
||||
print(f"✅ Job submitted: {task_id}")
|
||||
print("⏳ Webhook will notify when LLM extraction completes...")
|
||||
return task_id
|
||||
else:
|
||||
print(f"❌ Failed: {response.text}")
|
||||
return None
|
||||
|
||||
|
||||
def demo_4_llm_webhook_with_schema():
|
||||
"""Demo 4: LLM extraction with JSON schema and data in webhook (NEW in v0.7.6!)."""
|
||||
print("\n" + "="*70)
|
||||
print("DEMO 4: LLM Extraction - Schema + Full Data in Webhook (NEW!)")
|
||||
print("="*70)
|
||||
print("Submitting LLM extraction with JSON schema...")
|
||||
|
||||
schema = {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"title": {"type": "string", "description": "Page title"},
|
||||
"description": {"type": "string", "description": "Page description"},
|
||||
"main_topics": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"description": "Main topics covered"
|
||||
}
|
||||
},
|
||||
"required": ["title"]
|
||||
}
|
||||
|
||||
payload = {
|
||||
"url": "https://www.python.org",
|
||||
"q": "Extract the title, description, and main topics from this website",
|
||||
"schema": json.dumps(schema),
|
||||
"provider": "openai/gpt-4o-mini",
|
||||
"cache": False,
|
||||
"webhook_config": {
|
||||
"webhook_url": f"{WEBHOOK_BASE_URL}/webhook",
|
||||
"webhook_data_in_payload": True,
|
||||
"webhook_headers": {
|
||||
"X-Demo": "v0.7.6",
|
||||
"X-Type": "llm-with-schema"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
response = requests.post(f"{CRAWL4AI_BASE_URL}/llm/job", json=payload)
|
||||
if response.ok:
|
||||
task_id = response.json()['task_id']
|
||||
print(f"✅ Job submitted: {task_id}")
|
||||
print("⏳ Webhook will include structured extraction results...")
|
||||
return task_id
|
||||
else:
|
||||
print(f"❌ Failed: {response.text}")
|
||||
return None
|
||||
|
||||
|
||||
def demo_5_global_webhook_config():
|
||||
"""Demo 5: Using global webhook configuration from config.yml."""
|
||||
print("\n" + "="*70)
|
||||
print("DEMO 5: Global Webhook Configuration")
|
||||
print("="*70)
|
||||
print("💡 You can configure a default webhook URL in config.yml:")
|
||||
print("""
|
||||
webhooks:
|
||||
enabled: true
|
||||
default_url: "https://myapp.com/webhooks/default"
|
||||
data_in_payload: false
|
||||
retry:
|
||||
max_attempts: 5
|
||||
initial_delay_ms: 1000
|
||||
max_delay_ms: 32000
|
||||
timeout_ms: 30000
|
||||
""")
|
||||
print("Then submit jobs WITHOUT webhook_config - they'll use the default!")
|
||||
print("This is useful for consistent webhook handling across all jobs.")
|
||||
|
||||
|
||||
def demo_6_webhook_retry_logic():
|
||||
"""Demo 6: Webhook retry mechanism with exponential backoff."""
|
||||
print("\n" + "="*70)
|
||||
print("DEMO 6: Webhook Retry Logic")
|
||||
print("="*70)
|
||||
print("🔄 Webhook delivery uses exponential backoff retry:")
|
||||
print(" • Max attempts: 5")
|
||||
print(" • Delays: 1s → 2s → 4s → 8s → 16s")
|
||||
print(" • Timeout: 30s per attempt")
|
||||
print(" • Retries on: 5xx errors, network errors, timeouts")
|
||||
print(" • No retry on: 4xx client errors")
|
||||
print("\nThis ensures reliable webhook delivery even with temporary failures!")
|
||||
|
||||
|
||||
def print_summary():
|
||||
"""Print demo summary and results."""
|
||||
print("\n" + "="*70)
|
||||
print("📊 DEMO SUMMARY")
|
||||
print("="*70)
|
||||
print(f"Total webhooks received: {len(received_webhooks)}")
|
||||
|
||||
crawl_webhooks = [w for w in received_webhooks if w['task_type'] == 'crawl']
|
||||
llm_webhooks = [w for w in received_webhooks if w['task_type'] == 'llm_extraction']
|
||||
|
||||
print(f"\nBreakdown:")
|
||||
print(f" 🕷️ Crawl jobs: {len(crawl_webhooks)}")
|
||||
print(f" 🤖 LLM extraction jobs: {len(llm_webhooks)}")
|
||||
|
||||
print(f"\nDetails:")
|
||||
for i, webhook in enumerate(received_webhooks, 1):
|
||||
icon = "🕷️" if webhook['task_type'] == 'crawl' else "🤖"
|
||||
print(f" {i}. {icon} {webhook['task_id']}: {webhook['status']}")
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("✨ v0.7.6 KEY FEATURES DEMONSTRATED:")
|
||||
print("="*70)
|
||||
print("✅ Webhook support for /crawl/job")
|
||||
print("✅ Webhook support for /llm/job (NEW!)")
|
||||
print("✅ Notification-only mode (fetch data separately)")
|
||||
print("✅ Data-in-payload mode (get full results in webhook)")
|
||||
print("✅ Custom headers for authentication")
|
||||
print("✅ JSON schema for structured LLM extraction")
|
||||
print("✅ Exponential backoff retry for reliable delivery")
|
||||
print("✅ Global webhook configuration support")
|
||||
print("✅ Universal webhook handler for both job types")
|
||||
print("\n💡 Benefits:")
|
||||
print(" • No more polling - get instant notifications")
|
||||
print(" • Better resource utilization")
|
||||
print(" • Reliable delivery with automatic retries")
|
||||
print(" • Consistent API across crawl and LLM jobs")
|
||||
print(" • Production-ready webhook infrastructure")
|
||||
|
||||
|
||||
def main():
|
||||
"""Run all demos."""
|
||||
print("\n" + "="*70)
|
||||
print("🚀 Crawl4AI v0.7.6 Release Demo")
|
||||
print("="*70)
|
||||
print("Feature: Webhook Support for Docker Job Queue API")
|
||||
print("="*70)
|
||||
|
||||
# Check if server is running
|
||||
try:
|
||||
health = requests.get(f"{CRAWL4AI_BASE_URL}/health", timeout=5)
|
||||
print(f"✅ Crawl4AI server is running")
|
||||
except:
|
||||
print(f"❌ Cannot connect to Crawl4AI at {CRAWL4AI_BASE_URL}")
|
||||
print("Please start Docker container:")
|
||||
print(" docker run -d -p 11235:11235 --env-file .llm.env unclecode/crawl4ai:0.7.6")
|
||||
return
|
||||
|
||||
# Start webhook server
|
||||
print(f"\n🌐 Starting webhook server at {WEBHOOK_BASE_URL}...")
|
||||
webhook_thread = Thread(target=start_webhook_server, daemon=True)
|
||||
webhook_thread.start()
|
||||
time.sleep(2)
|
||||
|
||||
# Run demos
|
||||
demo_1_crawl_webhook_notification_only()
|
||||
time.sleep(5)
|
||||
|
||||
demo_2_crawl_webhook_with_data()
|
||||
time.sleep(5)
|
||||
|
||||
demo_3_llm_webhook_notification_only()
|
||||
time.sleep(5)
|
||||
|
||||
demo_4_llm_webhook_with_schema()
|
||||
time.sleep(5)
|
||||
|
||||
demo_5_global_webhook_config()
|
||||
demo_6_webhook_retry_logic()
|
||||
|
||||
# Wait for webhooks
|
||||
print("\n⏳ Waiting for all webhooks to arrive...")
|
||||
time.sleep(30)
|
||||
|
||||
# Print summary
|
||||
print_summary()
|
||||
|
||||
print("\n" + "="*70)
|
||||
print("✅ Demo completed!")
|
||||
print("="*70)
|
||||
print("\n📚 Documentation:")
|
||||
print(" • deploy/docker/WEBHOOK_EXAMPLES.md")
|
||||
print(" • docs/examples/docker_webhook_example.py")
|
||||
print("\n🔗 Upgrade:")
|
||||
print(" docker pull unclecode/crawl4ai:0.7.6")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,628 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Crawl4AI v0.7.7 Release Demo
|
||||
============================
|
||||
|
||||
This demo showcases the major feature in v0.7.7:
|
||||
**Self-Hosting with Real-time Monitoring Dashboard**
|
||||
|
||||
Features Demonstrated:
|
||||
1. System health monitoring with live metrics
|
||||
2. Real-time request tracking (active & completed)
|
||||
3. Browser pool management (permanent/hot/cold pools)
|
||||
4. Monitor API endpoints for programmatic access
|
||||
5. WebSocket streaming for real-time updates
|
||||
6. Control actions (kill browser, cleanup, restart)
|
||||
7. Production metrics (efficiency, reuse rates, memory)
|
||||
|
||||
Prerequisites:
|
||||
- Crawl4AI Docker container running on localhost:11235
|
||||
- Python packages: pip install httpx websockets
|
||||
|
||||
Usage:
|
||||
python docs/releases_review/demo_v0.7.7.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import httpx
|
||||
import json
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any
|
||||
|
||||
# Configuration
|
||||
CRAWL4AI_BASE_URL = "http://localhost:11235"
|
||||
MONITOR_DASHBOARD_URL = f"{CRAWL4AI_BASE_URL}/dashboard"
|
||||
|
||||
|
||||
def print_section(title: str, description: str = ""):
|
||||
"""Print a formatted section header"""
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f"📊 {title}")
|
||||
if description:
|
||||
print(f"{description}")
|
||||
print(f"{'=' * 70}\n")
|
||||
|
||||
|
||||
def print_subsection(title: str):
|
||||
"""Print a formatted subsection header"""
|
||||
print(f"\n{'-' * 70}")
|
||||
print(f"{title}")
|
||||
print(f"{'-' * 70}")
|
||||
|
||||
|
||||
async def check_server_health():
|
||||
"""Check if Crawl4AI server is running"""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=5.0) as client:
|
||||
response = await client.get(f"{CRAWL4AI_BASE_URL}/health")
|
||||
return response.status_code == 200
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
async def demo_1_system_health_overview():
|
||||
"""Demo 1: System Health Overview - Live metrics and pool status"""
|
||||
print_section(
|
||||
"Demo 1: System Health Overview",
|
||||
"Real-time monitoring of system resources and browser pool"
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
print("🔍 Fetching system health metrics...")
|
||||
|
||||
try:
|
||||
response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/health")
|
||||
health = response.json()
|
||||
|
||||
print("\n✅ System Health Report:")
|
||||
print(f"\n🖥️ Container Metrics:")
|
||||
print(f" • CPU Usage: {health['container']['cpu_percent']:.1f}%")
|
||||
print(f" • Memory Usage: {health['container']['memory_percent']:.1f}% "
|
||||
f"({health['container']['memory_mb']:.0f} MB)")
|
||||
print(f" • Network RX: {health['container']['network_rx_mb']:.2f} MB")
|
||||
print(f" • Network TX: {health['container']['network_tx_mb']:.2f} MB")
|
||||
print(f" • Uptime: {health['container']['uptime_seconds']:.0f}s")
|
||||
|
||||
print(f"\n🌐 Browser Pool Status:")
|
||||
print(f" Permanent Browser:")
|
||||
print(f" • Active: {health['pool']['permanent']['active']}")
|
||||
print(f" • Total Requests: {health['pool']['permanent']['total_requests']}")
|
||||
|
||||
print(f" Hot Pool (Frequently Used Configs):")
|
||||
print(f" • Count: {health['pool']['hot']['count']}")
|
||||
print(f" • Total Requests: {health['pool']['hot']['total_requests']}")
|
||||
|
||||
print(f" Cold Pool (On-Demand Configs):")
|
||||
print(f" • Count: {health['pool']['cold']['count']}")
|
||||
print(f" • Total Requests: {health['pool']['cold']['total_requests']}")
|
||||
|
||||
print(f"\n📈 Overall Statistics:")
|
||||
print(f" • Total Requests: {health['stats']['total_requests']}")
|
||||
print(f" • Success Rate: {health['stats']['success_rate_percent']:.1f}%")
|
||||
print(f" • Avg Latency: {health['stats']['avg_latency_ms']:.0f}ms")
|
||||
|
||||
print(f"\n💡 Dashboard URL: {MONITOR_DASHBOARD_URL}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error fetching health: {e}")
|
||||
|
||||
|
||||
async def demo_2_request_tracking():
|
||||
"""Demo 2: Real-time Request Tracking - Generate and monitor requests"""
|
||||
print_section(
|
||||
"Demo 2: Real-time Request Tracking",
|
||||
"Submit crawl jobs and watch them in real-time"
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
print("🚀 Submitting crawl requests...")
|
||||
|
||||
# Submit multiple requests
|
||||
urls_to_crawl = [
|
||||
"https://httpbin.org/html",
|
||||
"https://httpbin.org/json",
|
||||
"https://example.com"
|
||||
]
|
||||
|
||||
tasks = []
|
||||
for url in urls_to_crawl:
|
||||
task = client.post(
|
||||
f"{CRAWL4AI_BASE_URL}/crawl",
|
||||
json={"urls": [url], "crawler_config": {}}
|
||||
)
|
||||
tasks.append(task)
|
||||
|
||||
print(f" • Submitting {len(urls_to_crawl)} requests in parallel...")
|
||||
results = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
successful = sum(1 for r in results if not isinstance(r, Exception) and r.status_code == 200)
|
||||
print(f" ✅ {successful}/{len(urls_to_crawl)} requests submitted")
|
||||
|
||||
# Check request tracking
|
||||
print("\n📊 Checking request tracking...")
|
||||
await asyncio.sleep(2) # Wait for requests to process
|
||||
|
||||
response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/requests")
|
||||
requests_data = response.json()
|
||||
|
||||
print(f"\n📋 Request Status:")
|
||||
print(f" • Active Requests: {len(requests_data['active'])}")
|
||||
print(f" • Completed Requests: {len(requests_data['completed'])}")
|
||||
|
||||
if requests_data['completed']:
|
||||
print(f"\n📝 Recent Completed Requests:")
|
||||
for req in requests_data['completed'][:3]:
|
||||
status_icon = "✅" if req['success'] else "❌"
|
||||
print(f" {status_icon} {req['endpoint']} - {req['latency_ms']:.0f}ms")
|
||||
|
||||
|
||||
async def demo_3_browser_pool_management():
|
||||
"""Demo 3: Browser Pool Management - 3-tier architecture in action"""
|
||||
print_section(
|
||||
"Demo 3: Browser Pool Management",
|
||||
"Understanding permanent, hot, and cold browser pools"
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
print("🌊 Testing browser pool with different configurations...")
|
||||
|
||||
# Test 1: Default config (permanent browser)
|
||||
print("\n🔥 Test 1: Default Config → Permanent Browser")
|
||||
for i in range(3):
|
||||
await client.post(
|
||||
f"{CRAWL4AI_BASE_URL}/crawl",
|
||||
json={"urls": [f"https://httpbin.org/html?req={i}"], "crawler_config": {}}
|
||||
)
|
||||
print(f" • Request {i+1}/3 sent (should use permanent browser)")
|
||||
|
||||
await asyncio.sleep(2)
|
||||
|
||||
# Test 2: Custom viewport (cold → hot promotion after 3 uses)
|
||||
print("\n♨️ Test 2: Custom Viewport → Cold Pool (promoting to Hot)")
|
||||
viewport_config = {"viewport": {"width": 1280, "height": 720}}
|
||||
for i in range(4):
|
||||
await client.post(
|
||||
f"{CRAWL4AI_BASE_URL}/crawl",
|
||||
json={
|
||||
"urls": [f"https://httpbin.org/json?viewport={i}"],
|
||||
"browser_config": viewport_config,
|
||||
"crawler_config": {}
|
||||
}
|
||||
)
|
||||
print(f" • Request {i+1}/4 sent (cold→hot promotion after 3rd use)")
|
||||
|
||||
await asyncio.sleep(2)
|
||||
|
||||
# Check browser pool status
|
||||
print("\n📊 Browser Pool Report:")
|
||||
response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/browsers")
|
||||
browsers = response.json()
|
||||
|
||||
print(f"\n🎯 Pool Summary:")
|
||||
print(f" • Total Browsers: {browsers['summary']['total_count']}")
|
||||
print(f" • Total Memory: {browsers['summary']['total_memory_mb']} MB")
|
||||
print(f" • Reuse Rate: {browsers['summary']['reuse_rate_percent']:.1f}%")
|
||||
|
||||
print(f"\n📋 Browser Pool Details:")
|
||||
if browsers['permanent']:
|
||||
for browser in browsers['permanent']:
|
||||
print(f" 🔥 Permanent: {browser['browser_id'][:8]}... | "
|
||||
f"Requests: {browser['request_count']} | "
|
||||
f"Memory: {browser['memory_mb']:.0f} MB")
|
||||
|
||||
if browsers['hot']:
|
||||
for browser in browsers['hot']:
|
||||
print(f" ♨️ Hot: {browser['browser_id'][:8]}... | "
|
||||
f"Requests: {browser['request_count']} | "
|
||||
f"Memory: {browser['memory_mb']:.0f} MB")
|
||||
|
||||
if browsers['cold']:
|
||||
for browser in browsers['cold']:
|
||||
print(f" ❄️ Cold: {browser['browser_id'][:8]}... | "
|
||||
f"Requests: {browser['request_count']} | "
|
||||
f"Memory: {browser['memory_mb']:.0f} MB")
|
||||
|
||||
|
||||
async def demo_4_monitor_api_endpoints():
|
||||
"""Demo 4: Monitor API Endpoints - Complete API surface"""
|
||||
print_section(
|
||||
"Demo 4: Monitor API Endpoints",
|
||||
"Programmatic access to all monitoring data"
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
print("🔌 Testing Monitor API endpoints...")
|
||||
|
||||
# Endpoint performance statistics
|
||||
print_subsection("Endpoint Performance Statistics")
|
||||
response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/endpoints/stats")
|
||||
endpoint_stats = response.json()
|
||||
|
||||
print("\n📊 Per-Endpoint Analytics:")
|
||||
for endpoint, stats in endpoint_stats.items():
|
||||
print(f" {endpoint}:")
|
||||
print(f" • Requests: {stats['count']}")
|
||||
print(f" • Avg Latency: {stats['avg_latency_ms']:.0f}ms")
|
||||
print(f" • Success Rate: {stats['success_rate_percent']:.1f}%")
|
||||
|
||||
# Timeline data for charts
|
||||
print_subsection("Timeline Data (for Charts)")
|
||||
response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/timeline?minutes=5")
|
||||
timeline = response.json()
|
||||
|
||||
print(f"\n📈 Timeline Metrics (last 5 minutes):")
|
||||
print(f" • Data Points: {len(timeline['memory'])}")
|
||||
if timeline['memory']:
|
||||
latest = timeline['memory'][-1]
|
||||
print(f" • Latest Memory: {latest['value']:.1f}%")
|
||||
print(f" • Timestamp: {latest['timestamp']}")
|
||||
|
||||
# Janitor logs
|
||||
print_subsection("Janitor Cleanup Events")
|
||||
response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/logs/janitor?limit=3")
|
||||
janitor_logs = response.json()
|
||||
|
||||
print(f"\n🧹 Recent Cleanup Activities:")
|
||||
if janitor_logs:
|
||||
for log in janitor_logs[:3]:
|
||||
print(f" • {log['timestamp']}: {log['message']}")
|
||||
else:
|
||||
print(" (No cleanup events yet - janitor runs periodically)")
|
||||
|
||||
# Error logs
|
||||
print_subsection("Error Monitoring")
|
||||
response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/logs/errors?limit=3")
|
||||
error_logs = response.json()
|
||||
|
||||
print(f"\n❌ Recent Errors:")
|
||||
if error_logs:
|
||||
for log in error_logs[:3]:
|
||||
print(f" • {log['timestamp']}: {log['error_type']}")
|
||||
print(f" {log['message'][:100]}...")
|
||||
else:
|
||||
print(" ✅ No recent errors!")
|
||||
|
||||
|
||||
async def demo_5_websocket_streaming():
|
||||
"""Demo 5: WebSocket Streaming - Real-time updates"""
|
||||
print_section(
|
||||
"Demo 5: WebSocket Streaming",
|
||||
"Live monitoring with 2-second update intervals"
|
||||
)
|
||||
|
||||
print("⚡ WebSocket Streaming Demo")
|
||||
print("\n💡 The monitoring dashboard uses WebSocket for real-time updates")
|
||||
print(f" • Connection: ws://localhost:11235/monitor/ws")
|
||||
print(f" • Update Interval: 2 seconds")
|
||||
print(f" • Data: Health, requests, browsers, memory, errors")
|
||||
|
||||
print("\n📝 Sample WebSocket Integration Code:")
|
||||
print("""
|
||||
import websockets
|
||||
import json
|
||||
|
||||
async def monitor_realtime():
|
||||
uri = "ws://localhost:11235/monitor/ws"
|
||||
async with websockets.connect(uri) as websocket:
|
||||
while True:
|
||||
data = await websocket.recv()
|
||||
update = json.loads(data)
|
||||
|
||||
print(f"Memory: {update['health']['container']['memory_percent']:.1f}%")
|
||||
print(f"Active Requests: {len(update['requests']['active'])}")
|
||||
print(f"Browser Pool: {update['health']['pool']['permanent']['active']}")
|
||||
""")
|
||||
|
||||
print("\n🌐 Open the dashboard to see WebSocket in action:")
|
||||
print(f" {MONITOR_DASHBOARD_URL}")
|
||||
|
||||
|
||||
async def demo_6_control_actions():
|
||||
"""Demo 6: Control Actions - Manual browser management"""
|
||||
print_section(
|
||||
"Demo 6: Control Actions",
|
||||
"Manual control over browser pool and cleanup"
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
print("🎮 Testing control actions...")
|
||||
|
||||
# Force cleanup
|
||||
print_subsection("Force Immediate Cleanup")
|
||||
print("🧹 Triggering manual cleanup...")
|
||||
try:
|
||||
response = await client.post(f"{CRAWL4AI_BASE_URL}/monitor/actions/cleanup")
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
print(f" ✅ Cleanup completed")
|
||||
print(f" • Browsers cleaned: {result.get('cleaned_count', 0)}")
|
||||
print(f" • Memory freed: {result.get('memory_freed_mb', 0):.1f} MB")
|
||||
else:
|
||||
print(f" ⚠️ Response: {response.status_code}")
|
||||
except Exception as e:
|
||||
print(f" ℹ️ Cleanup action: {e}")
|
||||
|
||||
# Get browser list for potential kill/restart
|
||||
print_subsection("Browser Management")
|
||||
response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/browsers")
|
||||
browsers = response.json()
|
||||
|
||||
cold_browsers = browsers.get('cold', [])
|
||||
if cold_browsers:
|
||||
browser_id = cold_browsers[0]['browser_id']
|
||||
print(f"\n🎯 Example: Kill specific browser")
|
||||
print(f" POST /monitor/actions/kill_browser")
|
||||
print(f" JSON: {{'browser_id': '{browser_id[:16]}...'}}")
|
||||
print(f" → Kills the browser and frees resources")
|
||||
|
||||
print(f"\n🔄 Example: Restart browser")
|
||||
print(f" POST /monitor/actions/restart_browser")
|
||||
print(f" JSON: {{'browser_id': 'browser_id_here'}}")
|
||||
print(f" → Restart a specific browser instance")
|
||||
|
||||
# Reset statistics
|
||||
print_subsection("Reset Statistics")
|
||||
print("📊 Statistics can be reset for fresh monitoring:")
|
||||
print(f" POST /monitor/stats/reset")
|
||||
print(f" → Clears all accumulated statistics")
|
||||
|
||||
|
||||
async def demo_7_production_metrics():
|
||||
"""Demo 7: Production Metrics - Key indicators for operations"""
|
||||
print_section(
|
||||
"Demo 7: Production Metrics",
|
||||
"Critical metrics for production monitoring"
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
print("📊 Key Production Metrics:")
|
||||
|
||||
# Overall health
|
||||
response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/health")
|
||||
health = response.json()
|
||||
|
||||
# Browser efficiency
|
||||
response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/browsers")
|
||||
browsers = response.json()
|
||||
|
||||
print("\n🎯 Critical Metrics to Track:")
|
||||
|
||||
print(f"\n1️⃣ Memory Usage Trends")
|
||||
print(f" • Current: {health['container']['memory_percent']:.1f}%")
|
||||
print(f" • Alert if: >80%")
|
||||
print(f" • Action: Trigger cleanup or scale")
|
||||
|
||||
print(f"\n2️⃣ Request Success Rate")
|
||||
print(f" • Current: {health['stats']['success_rate_percent']:.1f}%")
|
||||
print(f" • Target: >95%")
|
||||
print(f" • Alert if: <90%")
|
||||
|
||||
print(f"\n3️⃣ Average Latency")
|
||||
print(f" • Current: {health['stats']['avg_latency_ms']:.0f}ms")
|
||||
print(f" • Target: <2000ms")
|
||||
print(f" • Alert if: >5000ms")
|
||||
|
||||
print(f"\n4️⃣ Browser Pool Efficiency")
|
||||
print(f" • Reuse Rate: {browsers['summary']['reuse_rate_percent']:.1f}%")
|
||||
print(f" • Target: >80%")
|
||||
print(f" • Indicates: Effective browser pooling")
|
||||
|
||||
print(f"\n5️⃣ Total Browsers")
|
||||
print(f" • Current: {browsers['summary']['total_count']}")
|
||||
print(f" • Alert if: >20 (possible leak)")
|
||||
print(f" • Check: Janitor is running correctly")
|
||||
|
||||
print(f"\n6️⃣ Error Frequency")
|
||||
response = await client.get(f"{CRAWL4AI_BASE_URL}/monitor/logs/errors?limit=10")
|
||||
errors = response.json()
|
||||
print(f" • Recent Errors: {len(errors)}")
|
||||
print(f" • Alert if: >10 in last hour")
|
||||
print(f" • Action: Review error patterns")
|
||||
|
||||
print("\n💡 Integration Examples:")
|
||||
print(" • Prometheus: Scrape /monitor/health")
|
||||
print(" • Alerting: Monitor memory, success rate, latency")
|
||||
print(" • Dashboards: WebSocket streaming to custom UI")
|
||||
print(" • Log Aggregation: Collect /monitor/logs/* endpoints")
|
||||
|
||||
|
||||
async def demo_8_self_hosting_value():
|
||||
"""Demo 8: Self-Hosting Value Proposition"""
|
||||
print_section(
|
||||
"Demo 8: Why Self-Host Crawl4AI?",
|
||||
"The value proposition of owning your infrastructure"
|
||||
)
|
||||
|
||||
print("🎯 Self-Hosting Benefits:\n")
|
||||
|
||||
print("🔒 Data Privacy & Security")
|
||||
print(" • Your data never leaves your infrastructure")
|
||||
print(" • No third-party access to crawled content")
|
||||
print(" • Keep sensitive workflows behind your firewall")
|
||||
|
||||
print("\n💰 Cost Control")
|
||||
print(" • No per-request pricing or rate limits")
|
||||
print(" • Predictable infrastructure costs")
|
||||
print(" • Scale based on your actual needs")
|
||||
|
||||
print("\n🎯 Full Customization")
|
||||
print(" • Complete control over browser configs")
|
||||
print(" • Custom hooks and strategies")
|
||||
print(" • Tailored monitoring and alerting")
|
||||
|
||||
print("\n📊 Complete Transparency")
|
||||
print(" • Real-time monitoring dashboard")
|
||||
print(" • Full visibility into system performance")
|
||||
print(" • Detailed request and error tracking")
|
||||
|
||||
print("\n⚡ Performance & Flexibility")
|
||||
print(" • Direct access, no network overhead")
|
||||
print(" • Integrate with existing infrastructure")
|
||||
print(" • Custom resource allocation")
|
||||
|
||||
print("\n🛡️ Enterprise-Grade Operations")
|
||||
print(" • Prometheus integration ready")
|
||||
print(" • WebSocket for real-time dashboards")
|
||||
print(" • Full API for automation")
|
||||
print(" • Manual controls for troubleshooting")
|
||||
|
||||
print(f"\n🌐 Get Started:")
|
||||
print(f" docker pull unclecode/crawl4ai:0.7.7")
|
||||
print(f" docker run -d -p 11235:11235 --shm-size=1g unclecode/crawl4ai:0.7.7")
|
||||
print(f" # Visit: {MONITOR_DASHBOARD_URL}")
|
||||
|
||||
|
||||
def print_summary():
|
||||
"""Print comprehensive demo summary"""
|
||||
print("\n" + "=" * 70)
|
||||
print("📊 DEMO SUMMARY - Crawl4AI v0.7.7")
|
||||
print("=" * 70)
|
||||
|
||||
print("\n✨ Features Demonstrated:")
|
||||
print("=" * 70)
|
||||
print("✅ System Health Overview")
|
||||
print(" → Real-time CPU, memory, network, and uptime monitoring")
|
||||
|
||||
print("\n✅ Request Tracking")
|
||||
print(" → Active and completed request monitoring with full details")
|
||||
|
||||
print("\n✅ Browser Pool Management")
|
||||
print(" → 3-tier architecture: Permanent, Hot, and Cold pools")
|
||||
print(" → Automatic promotion and cleanup")
|
||||
|
||||
print("\n✅ Monitor API Endpoints")
|
||||
print(" → Complete REST API for programmatic access")
|
||||
print(" → Health, requests, browsers, timeline, logs, errors")
|
||||
|
||||
print("\n✅ WebSocket Streaming")
|
||||
print(" → Real-time updates every 2 seconds")
|
||||
print(" → Build custom dashboards with live data")
|
||||
|
||||
print("\n✅ Control Actions")
|
||||
print(" → Manual browser management (kill, restart)")
|
||||
print(" → Force cleanup and statistics reset")
|
||||
|
||||
print("\n✅ Production Metrics")
|
||||
print(" → 6 critical metrics for operational excellence")
|
||||
print(" → Prometheus integration patterns")
|
||||
|
||||
print("\n✅ Self-Hosting Value")
|
||||
print(" → Data privacy, cost control, full customization")
|
||||
print(" → Enterprise-grade transparency and control")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("🎯 What's New in v0.7.7?")
|
||||
print("=" * 70)
|
||||
print("• 📊 Complete Real-time Monitoring System")
|
||||
print("• 🌐 Interactive Web Dashboard (/dashboard)")
|
||||
print("• 🔌 Comprehensive Monitor API")
|
||||
print("• ⚡ WebSocket Streaming (2-second updates)")
|
||||
print("• 🎮 Manual Control Actions")
|
||||
print("• 📈 Production Integration Examples")
|
||||
print("• 🏭 Prometheus, Alerting, Log Aggregation")
|
||||
print("• 🔥 Smart Browser Pool (Permanent/Hot/Cold)")
|
||||
print("• 🧹 Automatic Janitor Cleanup")
|
||||
print("• 📋 Full Request & Error Tracking")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("💡 Why This Matters")
|
||||
print("=" * 70)
|
||||
print("Before v0.7.7: Docker was just a containerized crawler")
|
||||
print("After v0.7.7: Complete self-hosting platform with enterprise monitoring")
|
||||
print("\nYou now have:")
|
||||
print(" • Full visibility into what's happening inside")
|
||||
print(" • Real-time operational dashboards")
|
||||
print(" • Complete control over browser resources")
|
||||
print(" • Production-ready observability")
|
||||
print(" • Zero external dependencies")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("📚 Next Steps")
|
||||
print("=" * 70)
|
||||
print(f"1. Open the dashboard: {MONITOR_DASHBOARD_URL}")
|
||||
print("2. Read the docs: https://docs.crawl4ai.com/basic/self-hosting/")
|
||||
print("3. Try the Monitor API endpoints yourself")
|
||||
print("4. Set up Prometheus integration for production")
|
||||
print("5. Build custom dashboards with WebSocket streaming")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("🔗 Resources")
|
||||
print("=" * 70)
|
||||
print(f"• Dashboard: {MONITOR_DASHBOARD_URL}")
|
||||
print(f"• Health API: {CRAWL4AI_BASE_URL}/monitor/health")
|
||||
print(f"• Documentation: https://docs.crawl4ai.com/")
|
||||
print(f"• GitHub: https://github.com/unclecode/crawl4ai")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("🎉 You're now in control of your web crawling destiny!")
|
||||
print("=" * 70)
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all demos"""
|
||||
print("\n" + "=" * 70)
|
||||
print("🚀 Crawl4AI v0.7.7 Release Demo")
|
||||
print("=" * 70)
|
||||
print("Feature: Self-Hosting with Real-time Monitoring Dashboard")
|
||||
print("=" * 70)
|
||||
|
||||
# Check if server is running
|
||||
print("\n🔍 Checking Crawl4AI server...")
|
||||
server_running = await check_server_health()
|
||||
|
||||
if not server_running:
|
||||
print(f"❌ Cannot connect to Crawl4AI at {CRAWL4AI_BASE_URL}")
|
||||
print("\nPlease start the Docker container:")
|
||||
print(" docker pull unclecode/crawl4ai:0.7.7")
|
||||
print(" docker run -d -p 11235:11235 --shm-size=1g unclecode/crawl4ai:0.7.7")
|
||||
print("\nThen re-run this demo.")
|
||||
return
|
||||
|
||||
print(f"✅ Crawl4AI server is running!")
|
||||
print(f"📊 Dashboard available at: {MONITOR_DASHBOARD_URL}")
|
||||
|
||||
# Run all demos
|
||||
demos = [
|
||||
demo_1_system_health_overview,
|
||||
demo_2_request_tracking,
|
||||
demo_3_browser_pool_management,
|
||||
demo_4_monitor_api_endpoints,
|
||||
demo_5_websocket_streaming,
|
||||
demo_6_control_actions,
|
||||
demo_7_production_metrics,
|
||||
demo_8_self_hosting_value,
|
||||
]
|
||||
|
||||
for i, demo_func in enumerate(demos, 1):
|
||||
try:
|
||||
await demo_func()
|
||||
|
||||
if i < len(demos):
|
||||
await asyncio.sleep(2) # Brief pause between demos
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print(f"\n\n⚠️ Demo interrupted by user")
|
||||
return
|
||||
except Exception as e:
|
||||
print(f"\n❌ Demo {i} error: {e}")
|
||||
print("Continuing to next demo...\n")
|
||||
continue
|
||||
|
||||
# Print comprehensive summary
|
||||
print_summary()
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("✅ Demo completed!")
|
||||
print("=" * 70)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n👋 Demo stopped by user. Thanks for trying Crawl4AI v0.7.7!")
|
||||
except Exception as e:
|
||||
print(f"\n\n❌ Demo failed: {e}")
|
||||
print("Make sure the Docker container is running:")
|
||||
print(" docker run -d -p 11235:11235 --shm-size=1g unclecode/crawl4ai:0.7.7")
|
||||
@@ -0,0 +1,910 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Crawl4AI v0.7.8 Release Demo - Verification Tests
|
||||
==================================================
|
||||
|
||||
This demo ACTUALLY RUNS and VERIFIES the bug fixes in v0.7.8.
|
||||
Each test executes real code and validates the fix is working.
|
||||
|
||||
Bug Fixes Verified:
|
||||
1. ProxyConfig JSON serialization (#1629)
|
||||
2. Configurable backoff parameters (#1269)
|
||||
3. LLM Strategy input_format support (#1178)
|
||||
4. Raw HTML URL variable (#1116)
|
||||
5. Relative URLs after redirects (#1268)
|
||||
6. pypdf migration (#1412)
|
||||
7. Pydantic v2 ConfigDict (#678)
|
||||
8. Docker ContentRelevanceFilter (#1642) - requires Docker
|
||||
9. Docker .cache permissions (#1638) - requires Docker
|
||||
10. AdaptiveCrawler query expansion (#1621) - requires LLM API key
|
||||
11. Import statement formatting (#1181)
|
||||
|
||||
Usage:
|
||||
python docs/releases_review/demo_v0.7.8.py
|
||||
|
||||
For Docker tests:
|
||||
docker run -d -p 11235:11235 --shm-size=1g unclecode/crawl4ai:0.7.8
|
||||
python docs/releases_review/demo_v0.7.8.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import warnings
|
||||
import os
|
||||
import tempfile
|
||||
from typing import Tuple, Optional
|
||||
from dataclasses import dataclass
|
||||
|
||||
# Test results tracking
|
||||
@dataclass
|
||||
class TestResult:
|
||||
name: str
|
||||
issue: str
|
||||
passed: bool
|
||||
message: str
|
||||
skipped: bool = False
|
||||
|
||||
|
||||
results: list[TestResult] = []
|
||||
|
||||
|
||||
def print_header(title: str):
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f"{title}")
|
||||
print(f"{'=' * 70}")
|
||||
|
||||
|
||||
def print_test(name: str, issue: str):
|
||||
print(f"\n[TEST] {name} ({issue})")
|
||||
print("-" * 50)
|
||||
|
||||
|
||||
def record_result(name: str, issue: str, passed: bool, message: str, skipped: bool = False):
|
||||
results.append(TestResult(name, issue, passed, message, skipped))
|
||||
if skipped:
|
||||
print(f" SKIPPED: {message}")
|
||||
elif passed:
|
||||
print(f" PASSED: {message}")
|
||||
else:
|
||||
print(f" FAILED: {message}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 1: ProxyConfig JSON Serialization (#1629)
|
||||
# =============================================================================
|
||||
async def test_proxy_config_serialization():
|
||||
"""
|
||||
Verify BrowserConfig.to_dict() properly serializes ProxyConfig to JSON.
|
||||
|
||||
BEFORE: ProxyConfig was included as object, causing JSON serialization to fail
|
||||
AFTER: ProxyConfig.to_dict() is called, producing valid JSON
|
||||
"""
|
||||
print_test("ProxyConfig JSON Serialization", "#1629")
|
||||
|
||||
try:
|
||||
from crawl4ai import BrowserConfig
|
||||
from crawl4ai.async_configs import ProxyConfig
|
||||
|
||||
# Create config with ProxyConfig
|
||||
proxy = ProxyConfig(
|
||||
server="http://proxy.example.com:8080",
|
||||
username="testuser",
|
||||
password="testpass"
|
||||
)
|
||||
browser_config = BrowserConfig(headless=True, proxy_config=proxy)
|
||||
|
||||
# Test 1: to_dict() should return dict for proxy_config
|
||||
config_dict = browser_config.to_dict()
|
||||
proxy_dict = config_dict.get('proxy_config')
|
||||
|
||||
if not isinstance(proxy_dict, dict):
|
||||
record_result("ProxyConfig Serialization", "#1629", False,
|
||||
f"proxy_config is {type(proxy_dict)}, expected dict")
|
||||
return
|
||||
|
||||
# Test 2: Should be JSON serializable
|
||||
try:
|
||||
json_str = json.dumps(config_dict)
|
||||
json.loads(json_str) # Verify valid JSON
|
||||
except (TypeError, json.JSONDecodeError) as e:
|
||||
record_result("ProxyConfig Serialization", "#1629", False,
|
||||
f"JSON serialization failed: {e}")
|
||||
return
|
||||
|
||||
# Test 3: Verify proxy data is preserved
|
||||
if proxy_dict.get('server') != "http://proxy.example.com:8080":
|
||||
record_result("ProxyConfig Serialization", "#1629", False,
|
||||
"Proxy server not preserved in serialization")
|
||||
return
|
||||
|
||||
record_result("ProxyConfig Serialization", "#1629", True,
|
||||
"BrowserConfig with ProxyConfig serializes to valid JSON")
|
||||
|
||||
except Exception as e:
|
||||
record_result("ProxyConfig Serialization", "#1629", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 2: Configurable Backoff Parameters (#1269)
|
||||
# =============================================================================
|
||||
async def test_configurable_backoff():
|
||||
"""
|
||||
Verify LLMConfig accepts and stores backoff configuration parameters.
|
||||
|
||||
BEFORE: Backoff was hardcoded (delay=2, attempts=3, factor=2)
|
||||
AFTER: LLMConfig accepts backoff_base_delay, backoff_max_attempts, backoff_exponential_factor
|
||||
"""
|
||||
print_test("Configurable Backoff Parameters", "#1269")
|
||||
|
||||
try:
|
||||
from crawl4ai import LLMConfig
|
||||
|
||||
# Test 1: Default values
|
||||
default_config = LLMConfig(provider="openai/gpt-4o-mini")
|
||||
|
||||
if default_config.backoff_base_delay != 2:
|
||||
record_result("Configurable Backoff", "#1269", False,
|
||||
f"Default base_delay is {default_config.backoff_base_delay}, expected 2")
|
||||
return
|
||||
|
||||
if default_config.backoff_max_attempts != 3:
|
||||
record_result("Configurable Backoff", "#1269", False,
|
||||
f"Default max_attempts is {default_config.backoff_max_attempts}, expected 3")
|
||||
return
|
||||
|
||||
if default_config.backoff_exponential_factor != 2:
|
||||
record_result("Configurable Backoff", "#1269", False,
|
||||
f"Default exponential_factor is {default_config.backoff_exponential_factor}, expected 2")
|
||||
return
|
||||
|
||||
# Test 2: Custom values
|
||||
custom_config = LLMConfig(
|
||||
provider="openai/gpt-4o-mini",
|
||||
backoff_base_delay=5,
|
||||
backoff_max_attempts=10,
|
||||
backoff_exponential_factor=3
|
||||
)
|
||||
|
||||
if custom_config.backoff_base_delay != 5:
|
||||
record_result("Configurable Backoff", "#1269", False,
|
||||
f"Custom base_delay is {custom_config.backoff_base_delay}, expected 5")
|
||||
return
|
||||
|
||||
if custom_config.backoff_max_attempts != 10:
|
||||
record_result("Configurable Backoff", "#1269", False,
|
||||
f"Custom max_attempts is {custom_config.backoff_max_attempts}, expected 10")
|
||||
return
|
||||
|
||||
if custom_config.backoff_exponential_factor != 3:
|
||||
record_result("Configurable Backoff", "#1269", False,
|
||||
f"Custom exponential_factor is {custom_config.backoff_exponential_factor}, expected 3")
|
||||
return
|
||||
|
||||
# Test 3: to_dict() includes backoff params
|
||||
config_dict = custom_config.to_dict()
|
||||
if 'backoff_base_delay' not in config_dict:
|
||||
record_result("Configurable Backoff", "#1269", False,
|
||||
"backoff_base_delay missing from to_dict()")
|
||||
return
|
||||
|
||||
record_result("Configurable Backoff", "#1269", True,
|
||||
"LLMConfig accepts and stores custom backoff parameters")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Configurable Backoff", "#1269", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 3: LLM Strategy Input Format (#1178)
|
||||
# =============================================================================
|
||||
async def test_llm_input_format():
|
||||
"""
|
||||
Verify LLMExtractionStrategy accepts input_format parameter.
|
||||
|
||||
BEFORE: Always used markdown input
|
||||
AFTER: Supports "markdown", "html", "fit_markdown", "cleaned_html", "fit_html"
|
||||
"""
|
||||
print_test("LLM Strategy Input Format", "#1178")
|
||||
|
||||
try:
|
||||
from crawl4ai import LLMExtractionStrategy, LLMConfig
|
||||
|
||||
llm_config = LLMConfig(provider="openai/gpt-4o-mini")
|
||||
|
||||
# Test 1: Default is markdown
|
||||
default_strategy = LLMExtractionStrategy(
|
||||
llm_config=llm_config,
|
||||
instruction="Extract data"
|
||||
)
|
||||
|
||||
if default_strategy.input_format != "markdown":
|
||||
record_result("LLM Input Format", "#1178", False,
|
||||
f"Default input_format is '{default_strategy.input_format}', expected 'markdown'")
|
||||
return
|
||||
|
||||
# Test 2: Can set to html
|
||||
html_strategy = LLMExtractionStrategy(
|
||||
llm_config=llm_config,
|
||||
instruction="Extract data",
|
||||
input_format="html"
|
||||
)
|
||||
|
||||
if html_strategy.input_format != "html":
|
||||
record_result("LLM Input Format", "#1178", False,
|
||||
f"HTML input_format is '{html_strategy.input_format}', expected 'html'")
|
||||
return
|
||||
|
||||
# Test 3: Can set to fit_markdown
|
||||
fit_strategy = LLMExtractionStrategy(
|
||||
llm_config=llm_config,
|
||||
instruction="Extract data",
|
||||
input_format="fit_markdown"
|
||||
)
|
||||
|
||||
if fit_strategy.input_format != "fit_markdown":
|
||||
record_result("LLM Input Format", "#1178", False,
|
||||
f"fit_markdown input_format is '{fit_strategy.input_format}'")
|
||||
return
|
||||
|
||||
record_result("LLM Input Format", "#1178", True,
|
||||
"LLMExtractionStrategy accepts all input_format options")
|
||||
|
||||
except Exception as e:
|
||||
record_result("LLM Input Format", "#1178", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 4: Raw HTML URL Variable (#1116)
|
||||
# =============================================================================
|
||||
async def test_raw_html_url_variable():
|
||||
"""
|
||||
Verify that raw: prefix URLs pass "Raw HTML" to extraction strategy.
|
||||
|
||||
BEFORE: Entire HTML blob was passed as URL parameter
|
||||
AFTER: "Raw HTML" string is passed as URL parameter
|
||||
"""
|
||||
print_test("Raw HTML URL Variable", "#1116")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
from crawl4ai.extraction_strategy import ExtractionStrategy
|
||||
|
||||
# Custom strategy to capture what URL is passed
|
||||
class URLCapturingStrategy(ExtractionStrategy):
|
||||
captured_url = None
|
||||
|
||||
def extract(self, url: str, html: str, *args, **kwargs):
|
||||
URLCapturingStrategy.captured_url = url
|
||||
return [{"content": "test"}]
|
||||
|
||||
html_content = "<html><body><h1>Test</h1></body></html>"
|
||||
strategy = URLCapturingStrategy()
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun(
|
||||
url=f"raw:{html_content}",
|
||||
config=CrawlerRunConfig(
|
||||
extraction_strategy=strategy
|
||||
)
|
||||
)
|
||||
|
||||
captured = URLCapturingStrategy.captured_url
|
||||
|
||||
if captured is None:
|
||||
record_result("Raw HTML URL Variable", "#1116", False,
|
||||
"Extraction strategy was not called")
|
||||
return
|
||||
|
||||
if captured == html_content or captured.startswith("<html"):
|
||||
record_result("Raw HTML URL Variable", "#1116", False,
|
||||
f"URL contains HTML content instead of 'Raw HTML': {captured[:50]}...")
|
||||
return
|
||||
|
||||
if captured != "Raw HTML":
|
||||
record_result("Raw HTML URL Variable", "#1116", False,
|
||||
f"URL is '{captured}', expected 'Raw HTML'")
|
||||
return
|
||||
|
||||
record_result("Raw HTML URL Variable", "#1116", True,
|
||||
"Extraction strategy receives 'Raw HTML' as URL for raw: prefix")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Raw HTML URL Variable", "#1116", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 5: Relative URLs After Redirects (#1268)
|
||||
# =============================================================================
|
||||
async def test_redirect_url_handling():
|
||||
"""
|
||||
Verify that redirected_url reflects the final URL after JS navigation.
|
||||
|
||||
BEFORE: redirected_url was the original URL, not the final URL
|
||||
AFTER: redirected_url is captured after JS execution completes
|
||||
"""
|
||||
print_test("Relative URLs After Redirects", "#1268")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
|
||||
# Test with a URL that we know the final state of
|
||||
# We'll use httpbin which doesn't redirect, but verify the mechanism works
|
||||
test_url = "https://httpbin.org/html"
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun(
|
||||
url=test_url,
|
||||
config=CrawlerRunConfig()
|
||||
)
|
||||
|
||||
# Verify redirected_url is populated
|
||||
if not result.redirected_url:
|
||||
record_result("Redirect URL Handling", "#1268", False,
|
||||
"redirected_url is empty")
|
||||
return
|
||||
|
||||
# For non-redirecting URL, should match original or be the final URL
|
||||
if not result.redirected_url.startswith("https://httpbin.org"):
|
||||
record_result("Redirect URL Handling", "#1268", False,
|
||||
f"redirected_url is unexpected: {result.redirected_url}")
|
||||
return
|
||||
|
||||
# Verify links are present and resolved
|
||||
if result.links:
|
||||
# Check that internal links have full URLs
|
||||
internal_links = result.links.get('internal', [])
|
||||
external_links = result.links.get('external', [])
|
||||
all_links = internal_links + external_links
|
||||
|
||||
for link in all_links[:5]: # Check first 5 links
|
||||
href = link.get('href', '')
|
||||
if href and not href.startswith(('http://', 'https://', 'mailto:', 'tel:', '#', 'javascript:')):
|
||||
record_result("Redirect URL Handling", "#1268", False,
|
||||
f"Link not resolved to absolute URL: {href}")
|
||||
return
|
||||
|
||||
record_result("Redirect URL Handling", "#1268", True,
|
||||
f"redirected_url correctly captured: {result.redirected_url}")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Redirect URL Handling", "#1268", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 6: pypdf Migration (#1412)
|
||||
# =============================================================================
|
||||
async def test_pypdf_migration():
|
||||
"""
|
||||
Verify pypdf is used instead of deprecated PyPDF2.
|
||||
|
||||
BEFORE: Used PyPDF2 (deprecated since 2022)
|
||||
AFTER: Uses pypdf (actively maintained)
|
||||
"""
|
||||
print_test("pypdf Migration", "#1412")
|
||||
|
||||
try:
|
||||
# Test 1: pypdf should be importable (if pdf extra is installed)
|
||||
try:
|
||||
import pypdf
|
||||
pypdf_available = True
|
||||
pypdf_version = pypdf.__version__
|
||||
except ImportError:
|
||||
pypdf_available = False
|
||||
pypdf_version = None
|
||||
|
||||
# Test 2: PyPDF2 should NOT be imported by crawl4ai
|
||||
# Check if the processor uses pypdf
|
||||
try:
|
||||
from crawl4ai.processors.pdf import processor
|
||||
processor_source = open(processor.__file__).read()
|
||||
|
||||
uses_pypdf = 'from pypdf' in processor_source or 'import pypdf' in processor_source
|
||||
uses_pypdf2 = 'from PyPDF2' in processor_source or 'import PyPDF2' in processor_source
|
||||
|
||||
if uses_pypdf2 and not uses_pypdf:
|
||||
record_result("pypdf Migration", "#1412", False,
|
||||
"PDF processor still uses PyPDF2")
|
||||
return
|
||||
|
||||
if uses_pypdf:
|
||||
record_result("pypdf Migration", "#1412", True,
|
||||
f"PDF processor uses pypdf{' v' + pypdf_version if pypdf_version else ''}")
|
||||
return
|
||||
else:
|
||||
record_result("pypdf Migration", "#1412", True,
|
||||
"PDF processor found, pypdf dependency updated", skipped=not pypdf_available)
|
||||
return
|
||||
|
||||
except ImportError:
|
||||
# PDF processor not available
|
||||
if pypdf_available:
|
||||
record_result("pypdf Migration", "#1412", True,
|
||||
f"pypdf v{pypdf_version} is installed (PDF processor not loaded)")
|
||||
else:
|
||||
record_result("pypdf Migration", "#1412", True,
|
||||
"PDF support not installed (optional feature)", skipped=True)
|
||||
return
|
||||
|
||||
except Exception as e:
|
||||
record_result("pypdf Migration", "#1412", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 7: Pydantic v2 ConfigDict (#678)
|
||||
# =============================================================================
|
||||
async def test_pydantic_configdict():
|
||||
"""
|
||||
Verify no Pydantic deprecation warnings for Config class.
|
||||
|
||||
BEFORE: Used deprecated 'class Config' syntax
|
||||
AFTER: Uses ConfigDict for Pydantic v2 compatibility
|
||||
"""
|
||||
print_test("Pydantic v2 ConfigDict", "#678")
|
||||
|
||||
try:
|
||||
import pydantic
|
||||
from pydantic import __version__ as pydantic_version
|
||||
|
||||
# Capture warnings during import
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
warnings.simplefilter("always", DeprecationWarning)
|
||||
|
||||
# Import models that might have Config classes
|
||||
from crawl4ai.models import CrawlResult, MarkdownGenerationResult
|
||||
from crawl4ai.async_configs import CrawlerRunConfig, BrowserConfig
|
||||
|
||||
# Filter for Pydantic-related deprecation warnings
|
||||
pydantic_warnings = [
|
||||
warning for warning in w
|
||||
if 'pydantic' in str(warning.message).lower()
|
||||
or 'config' in str(warning.message).lower()
|
||||
]
|
||||
|
||||
if pydantic_warnings:
|
||||
warning_msgs = [str(w.message) for w in pydantic_warnings[:3]]
|
||||
record_result("Pydantic ConfigDict", "#678", False,
|
||||
f"Deprecation warnings: {warning_msgs}")
|
||||
return
|
||||
|
||||
# Verify models work correctly
|
||||
try:
|
||||
# Test that models can be instantiated without issues
|
||||
config = CrawlerRunConfig()
|
||||
browser = BrowserConfig()
|
||||
|
||||
record_result("Pydantic ConfigDict", "#678", True,
|
||||
f"No deprecation warnings with Pydantic v{pydantic_version}")
|
||||
except Exception as e:
|
||||
record_result("Pydantic ConfigDict", "#678", False,
|
||||
f"Model instantiation failed: {e}")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Pydantic ConfigDict", "#678", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 8: Docker ContentRelevanceFilter (#1642)
|
||||
# =============================================================================
|
||||
async def test_docker_content_filter():
|
||||
"""
|
||||
Verify ContentRelevanceFilter deserializes correctly in Docker API.
|
||||
|
||||
BEFORE: Docker API failed to import/instantiate ContentRelevanceFilter
|
||||
AFTER: Filter is properly exported and deserializable
|
||||
"""
|
||||
print_test("Docker ContentRelevanceFilter", "#1642")
|
||||
|
||||
# First verify the fix in local code
|
||||
try:
|
||||
# Test 1: ContentRelevanceFilter should be importable from crawl4ai
|
||||
from crawl4ai import ContentRelevanceFilter
|
||||
|
||||
# Test 2: Should be instantiable
|
||||
filter_instance = ContentRelevanceFilter(
|
||||
query="test query",
|
||||
threshold=0.3
|
||||
)
|
||||
|
||||
if not hasattr(filter_instance, 'query'):
|
||||
record_result("Docker ContentRelevanceFilter", "#1642", False,
|
||||
"ContentRelevanceFilter missing query attribute")
|
||||
return
|
||||
|
||||
except ImportError as e:
|
||||
record_result("Docker ContentRelevanceFilter", "#1642", False,
|
||||
f"ContentRelevanceFilter not exported: {e}")
|
||||
return
|
||||
except Exception as e:
|
||||
record_result("Docker ContentRelevanceFilter", "#1642", False,
|
||||
f"ContentRelevanceFilter instantiation failed: {e}")
|
||||
return
|
||||
|
||||
# Test Docker API if available
|
||||
try:
|
||||
import httpx
|
||||
|
||||
async with httpx.AsyncClient(timeout=5.0) as client:
|
||||
response = await client.get("http://localhost:11235/health")
|
||||
if response.status_code != 200:
|
||||
raise Exception("Docker not available")
|
||||
|
||||
# Docker is running, test the API
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
request = {
|
||||
"urls": ["https://httpbin.org/html"],
|
||||
"crawler_config": {
|
||||
"deep_crawl_strategy": {
|
||||
"type": "BFSDeepCrawlStrategy",
|
||||
"max_depth": 1,
|
||||
"filter_chain": [
|
||||
{
|
||||
"type": "ContentTypeFilter",
|
||||
"allowed_types": ["text/html"]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
response = await client.post(
|
||||
"http://localhost:11235/crawl",
|
||||
json=request
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
record_result("Docker ContentRelevanceFilter", "#1642", True,
|
||||
"Filter deserializes correctly in Docker API")
|
||||
else:
|
||||
record_result("Docker ContentRelevanceFilter", "#1642", False,
|
||||
f"Docker API returned {response.status_code}: {response.text[:100]}")
|
||||
|
||||
except ImportError:
|
||||
record_result("Docker ContentRelevanceFilter", "#1642", True,
|
||||
"ContentRelevanceFilter exportable (Docker test skipped - httpx not installed)",
|
||||
skipped=True)
|
||||
except Exception as e:
|
||||
record_result("Docker ContentRelevanceFilter", "#1642", True,
|
||||
f"ContentRelevanceFilter exportable (Docker test skipped: {e})",
|
||||
skipped=True)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 9: Docker Cache Permissions (#1638)
|
||||
# =============================================================================
|
||||
async def test_docker_cache_permissions():
|
||||
"""
|
||||
Verify Docker image has correct .cache folder permissions.
|
||||
|
||||
This test requires Docker container to be running.
|
||||
"""
|
||||
print_test("Docker Cache Permissions", "#1638")
|
||||
|
||||
try:
|
||||
import httpx
|
||||
|
||||
async with httpx.AsyncClient(timeout=5.0) as client:
|
||||
response = await client.get("http://localhost:11235/health")
|
||||
if response.status_code != 200:
|
||||
raise Exception("Docker not available")
|
||||
|
||||
# Test by making a crawl request with caching
|
||||
async with httpx.AsyncClient(timeout=60.0) as client:
|
||||
request = {
|
||||
"urls": ["https://httpbin.org/html"],
|
||||
"crawler_config": {
|
||||
"cache_mode": "enabled"
|
||||
}
|
||||
}
|
||||
|
||||
response = await client.post(
|
||||
"http://localhost:11235/crawl",
|
||||
json=request
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
# Check if there were permission errors
|
||||
if "permission" in str(result).lower() and "denied" in str(result).lower():
|
||||
record_result("Docker Cache Permissions", "#1638", False,
|
||||
"Permission denied error in response")
|
||||
else:
|
||||
record_result("Docker Cache Permissions", "#1638", True,
|
||||
"Crawl with caching succeeded in Docker")
|
||||
else:
|
||||
error_text = response.text[:200]
|
||||
if "permission" in error_text.lower():
|
||||
record_result("Docker Cache Permissions", "#1638", False,
|
||||
f"Permission error: {error_text}")
|
||||
else:
|
||||
record_result("Docker Cache Permissions", "#1638", False,
|
||||
f"Request failed: {response.status_code}")
|
||||
|
||||
except ImportError:
|
||||
record_result("Docker Cache Permissions", "#1638", True,
|
||||
"Skipped - httpx not installed", skipped=True)
|
||||
except Exception as e:
|
||||
record_result("Docker Cache Permissions", "#1638", True,
|
||||
f"Skipped - Docker not available: {e}", skipped=True)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 10: AdaptiveCrawler Query Expansion (#1621)
|
||||
# =============================================================================
|
||||
async def test_adaptive_crawler_embedding():
|
||||
"""
|
||||
Verify EmbeddingStrategy LLM code is uncommented and functional.
|
||||
|
||||
BEFORE: LLM call was commented out, using hardcoded mock data
|
||||
AFTER: Actually calls LLM for query expansion
|
||||
"""
|
||||
print_test("AdaptiveCrawler Query Expansion", "#1621")
|
||||
|
||||
try:
|
||||
# Read the source file to verify the fix
|
||||
import crawl4ai.adaptive_crawler as adaptive_module
|
||||
source_file = adaptive_module.__file__
|
||||
|
||||
with open(source_file, 'r') as f:
|
||||
source_code = f.read()
|
||||
|
||||
# Check that the LLM call is NOT commented out
|
||||
# Look for the perform_completion_with_backoff call
|
||||
|
||||
# Find the EmbeddingStrategy section
|
||||
if 'class EmbeddingStrategy' not in source_code:
|
||||
record_result("AdaptiveCrawler Query Expansion", "#1621", True,
|
||||
"EmbeddingStrategy not in adaptive_crawler (may have moved)",
|
||||
skipped=True)
|
||||
return
|
||||
|
||||
# Check if the mock data line is commented out
|
||||
# and the actual LLM call is NOT commented out
|
||||
lines = source_code.split('\n')
|
||||
in_embedding_strategy = False
|
||||
found_llm_call = False
|
||||
mock_data_commented = False
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
if 'class EmbeddingStrategy' in line:
|
||||
in_embedding_strategy = True
|
||||
elif in_embedding_strategy and line.strip().startswith('class '):
|
||||
in_embedding_strategy = False
|
||||
|
||||
if in_embedding_strategy:
|
||||
# Check for uncommented LLM call
|
||||
if 'perform_completion_with_backoff' in line and not line.strip().startswith('#'):
|
||||
found_llm_call = True
|
||||
# Check for commented mock data
|
||||
if "variations ={'queries'" in line or 'variations = {\'queries\'' in line:
|
||||
if line.strip().startswith('#'):
|
||||
mock_data_commented = True
|
||||
|
||||
if found_llm_call:
|
||||
record_result("AdaptiveCrawler Query Expansion", "#1621", True,
|
||||
"LLM call is active in EmbeddingStrategy")
|
||||
else:
|
||||
# Check if the entire embedding strategy exists but might be structured differently
|
||||
if 'perform_completion_with_backoff' in source_code:
|
||||
record_result("AdaptiveCrawler Query Expansion", "#1621", True,
|
||||
"perform_completion_with_backoff found in module")
|
||||
else:
|
||||
record_result("AdaptiveCrawler Query Expansion", "#1621", False,
|
||||
"LLM call not found or still commented out")
|
||||
|
||||
except Exception as e:
|
||||
record_result("AdaptiveCrawler Query Expansion", "#1621", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 11: Import Statement Formatting (#1181)
|
||||
# =============================================================================
|
||||
async def test_import_formatting():
|
||||
"""
|
||||
Verify code extraction properly formats import statements.
|
||||
|
||||
BEFORE: Import statements were concatenated without newlines
|
||||
AFTER: Import statements have proper newline separation
|
||||
"""
|
||||
print_test("Import Statement Formatting", "#1181")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
|
||||
# Create HTML with code containing imports
|
||||
html_with_code = """
|
||||
<html>
|
||||
<body>
|
||||
<pre><code>
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Dict
|
||||
|
||||
def main():
|
||||
pass
|
||||
</code></pre>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun(
|
||||
url=f"raw:{html_with_code}",
|
||||
config=CrawlerRunConfig()
|
||||
)
|
||||
|
||||
markdown = result.markdown.raw_markdown if result.markdown else ""
|
||||
|
||||
# Check that imports are not concatenated on the same line
|
||||
# Bad: "import osimport sys" (no newline between statements)
|
||||
# This is the actual bug - statements getting merged on same line
|
||||
bad_patterns = [
|
||||
"import os import sys", # Space but no newline
|
||||
"import osimport sys", # No space or newline
|
||||
"import os from pathlib", # Space but no newline
|
||||
"import osfrom pathlib", # No space or newline
|
||||
]
|
||||
|
||||
markdown_single_line = markdown.replace('\n', ' ') # Convert newlines to spaces
|
||||
|
||||
for pattern in bad_patterns:
|
||||
# Check if pattern exists without proper line separation
|
||||
if pattern.replace(' ', '') in markdown_single_line.replace(' ', ''):
|
||||
# Verify it's actually on same line (not just adjacent after newline removal)
|
||||
lines = markdown.split('\n')
|
||||
for line in lines:
|
||||
if 'import' in line.lower():
|
||||
# Count import statements on this line
|
||||
import_count = line.lower().count('import ')
|
||||
if import_count > 1:
|
||||
record_result("Import Formatting", "#1181", False,
|
||||
f"Multiple imports on same line: {line[:60]}...")
|
||||
return
|
||||
|
||||
# Verify imports are present
|
||||
if "import" in markdown.lower():
|
||||
record_result("Import Formatting", "#1181", True,
|
||||
"Import statements are properly line-separated")
|
||||
else:
|
||||
record_result("Import Formatting", "#1181", True,
|
||||
"No import statements found to verify (test HTML may have changed)")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Import Formatting", "#1181", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# COMPREHENSIVE CRAWL TEST
|
||||
# =============================================================================
|
||||
async def test_comprehensive_crawl():
|
||||
"""
|
||||
Run a comprehensive crawl to verify overall stability.
|
||||
"""
|
||||
print_test("Comprehensive Crawl Test", "Overall")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
|
||||
|
||||
async with AsyncWebCrawler(config=BrowserConfig(headless=True)) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://httpbin.org/html",
|
||||
config=CrawlerRunConfig()
|
||||
)
|
||||
|
||||
# Verify result
|
||||
checks = []
|
||||
|
||||
if result.success:
|
||||
checks.append("success=True")
|
||||
else:
|
||||
record_result("Comprehensive Crawl", "Overall", False,
|
||||
f"Crawl failed: {result.error_message}")
|
||||
return
|
||||
|
||||
if result.html and len(result.html) > 100:
|
||||
checks.append(f"html={len(result.html)} chars")
|
||||
|
||||
if result.markdown and result.markdown.raw_markdown:
|
||||
checks.append(f"markdown={len(result.markdown.raw_markdown)} chars")
|
||||
|
||||
if result.redirected_url:
|
||||
checks.append("redirected_url present")
|
||||
|
||||
record_result("Comprehensive Crawl", "Overall", True,
|
||||
f"All checks passed: {', '.join(checks)}")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Comprehensive Crawl", "Overall", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MAIN
|
||||
# =============================================================================
|
||||
|
||||
def print_summary():
|
||||
"""Print test results summary"""
|
||||
print_header("TEST RESULTS SUMMARY")
|
||||
|
||||
passed = sum(1 for r in results if r.passed and not r.skipped)
|
||||
failed = sum(1 for r in results if not r.passed and not r.skipped)
|
||||
skipped = sum(1 for r in results if r.skipped)
|
||||
|
||||
print(f"\nTotal: {len(results)} tests")
|
||||
print(f" Passed: {passed}")
|
||||
print(f" Failed: {failed}")
|
||||
print(f" Skipped: {skipped}")
|
||||
|
||||
if failed > 0:
|
||||
print("\nFailed Tests:")
|
||||
for r in results:
|
||||
if not r.passed and not r.skipped:
|
||||
print(f" - {r.name} ({r.issue}): {r.message}")
|
||||
|
||||
if skipped > 0:
|
||||
print("\nSkipped Tests:")
|
||||
for r in results:
|
||||
if r.skipped:
|
||||
print(f" - {r.name} ({r.issue}): {r.message}")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
if failed == 0:
|
||||
print("All tests passed! v0.7.8 bug fixes verified.")
|
||||
else:
|
||||
print(f"WARNING: {failed} test(s) failed!")
|
||||
print("=" * 70)
|
||||
|
||||
return failed == 0
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all verification tests"""
|
||||
print_header("Crawl4AI v0.7.8 - Bug Fix Verification Tests")
|
||||
print("Running actual tests to verify bug fixes...")
|
||||
|
||||
# Run all tests
|
||||
tests = [
|
||||
test_proxy_config_serialization, # #1629
|
||||
test_configurable_backoff, # #1269
|
||||
test_llm_input_format, # #1178
|
||||
test_raw_html_url_variable, # #1116
|
||||
test_redirect_url_handling, # #1268
|
||||
test_pypdf_migration, # #1412
|
||||
test_pydantic_configdict, # #678
|
||||
test_docker_content_filter, # #1642
|
||||
test_docker_cache_permissions, # #1638
|
||||
test_adaptive_crawler_embedding, # #1621
|
||||
test_import_formatting, # #1181
|
||||
test_comprehensive_crawl, # Overall
|
||||
]
|
||||
|
||||
for test_func in tests:
|
||||
try:
|
||||
await test_func()
|
||||
except Exception as e:
|
||||
print(f"\nTest {test_func.__name__} crashed: {e}")
|
||||
results.append(TestResult(
|
||||
test_func.__name__,
|
||||
"Unknown",
|
||||
False,
|
||||
f"Crashed: {e}"
|
||||
))
|
||||
|
||||
# Print summary
|
||||
all_passed = print_summary()
|
||||
|
||||
return 0 if all_passed else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
exit_code = asyncio.run(main())
|
||||
sys.exit(exit_code)
|
||||
except KeyboardInterrupt:
|
||||
print("\n\nTests interrupted by user.")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"\n\nTest suite failed: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,633 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Crawl4AI v0.8.0 Release Demo - Feature Verification Tests
|
||||
==========================================================
|
||||
|
||||
This demo ACTUALLY RUNS and VERIFIES the new features in v0.8.0.
|
||||
Each test executes real code and validates the feature is working.
|
||||
|
||||
New Features Verified:
|
||||
1. Crash Recovery - on_state_change callback for real-time state persistence
|
||||
2. Crash Recovery - resume_state for resuming from checkpoint
|
||||
3. Crash Recovery - State is JSON serializable
|
||||
4. Prefetch Mode - Returns HTML and links only
|
||||
5. Prefetch Mode - Skips heavy processing (markdown, extraction)
|
||||
6. Prefetch Mode - Two-phase crawl pattern
|
||||
7. Security - Hooks disabled by default (Docker API)
|
||||
|
||||
Breaking Changes in v0.8.0:
|
||||
- Docker API hooks disabled by default (CRAWL4AI_HOOKS_ENABLED=false)
|
||||
- file:// URLs blocked on Docker API endpoints
|
||||
|
||||
Usage:
|
||||
python docs/releases_review/demo_v0.8.0.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from typing import Dict, Any, List, Optional
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
# Test results tracking
|
||||
@dataclass
|
||||
class TestResult:
|
||||
name: str
|
||||
feature: str
|
||||
passed: bool
|
||||
message: str
|
||||
skipped: bool = False
|
||||
|
||||
|
||||
results: list[TestResult] = []
|
||||
|
||||
|
||||
def print_header(title: str):
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f"{title}")
|
||||
print(f"{'=' * 70}")
|
||||
|
||||
|
||||
def print_test(name: str, feature: str):
|
||||
print(f"\n[TEST] {name} ({feature})")
|
||||
print("-" * 50)
|
||||
|
||||
|
||||
def record_result(name: str, feature: str, passed: bool, message: str, skipped: bool = False):
|
||||
results.append(TestResult(name, feature, passed, message, skipped))
|
||||
if skipped:
|
||||
print(f" SKIPPED: {message}")
|
||||
elif passed:
|
||||
print(f" PASSED: {message}")
|
||||
else:
|
||||
print(f" FAILED: {message}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 1: Crash Recovery - State Capture with on_state_change
|
||||
# =============================================================================
|
||||
async def test_crash_recovery_state_capture():
|
||||
"""
|
||||
Verify on_state_change callback is called after each URL is processed.
|
||||
|
||||
NEW in v0.8.0: Deep crawl strategies support on_state_change callback
|
||||
for real-time state persistence (useful for cloud deployments).
|
||||
"""
|
||||
print_test("Crash Recovery - State Capture", "on_state_change")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
|
||||
|
||||
captured_states: List[Dict[str, Any]] = []
|
||||
|
||||
async def capture_state(state: Dict[str, Any]):
|
||||
"""Callback that fires after each URL is processed."""
|
||||
captured_states.append(state.copy())
|
||||
|
||||
strategy = BFSDeepCrawlStrategy(
|
||||
max_depth=1,
|
||||
max_pages=3,
|
||||
on_state_change=capture_state,
|
||||
)
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
deep_crawl_strategy=strategy,
|
||||
verbose=False,
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
await crawler.arun("https://books.toscrape.com", config=config)
|
||||
|
||||
# Verify states were captured
|
||||
if len(captured_states) == 0:
|
||||
record_result("State Capture", "on_state_change", False,
|
||||
"No states captured - callback not called")
|
||||
return
|
||||
|
||||
# Verify callback was called for each page
|
||||
pages_crawled = captured_states[-1].get("pages_crawled", 0)
|
||||
if pages_crawled != len(captured_states):
|
||||
record_result("State Capture", "on_state_change", False,
|
||||
f"Callback count {len(captured_states)} != pages_crawled {pages_crawled}")
|
||||
return
|
||||
|
||||
record_result("State Capture", "on_state_change", True,
|
||||
f"Callback fired {len(captured_states)} times (once per URL)")
|
||||
|
||||
except Exception as e:
|
||||
record_result("State Capture", "on_state_change", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 2: Crash Recovery - Resume from Checkpoint
|
||||
# =============================================================================
|
||||
async def test_crash_recovery_resume():
|
||||
"""
|
||||
Verify crawl can resume from a saved checkpoint without re-crawling visited URLs.
|
||||
|
||||
NEW in v0.8.0: BFSDeepCrawlStrategy accepts resume_state parameter
|
||||
to continue from a previously saved checkpoint.
|
||||
"""
|
||||
print_test("Crash Recovery - Resume from Checkpoint", "resume_state")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
|
||||
|
||||
# Phase 1: Start crawl and capture state after 2 pages
|
||||
crash_after = 2
|
||||
captured_states: List[Dict] = []
|
||||
phase1_urls: List[str] = []
|
||||
|
||||
async def capture_until_crash(state: Dict[str, Any]):
|
||||
captured_states.append(state.copy())
|
||||
phase1_urls.clear()
|
||||
phase1_urls.extend(state["visited"])
|
||||
if state["pages_crawled"] >= crash_after:
|
||||
raise Exception("Simulated crash")
|
||||
|
||||
strategy1 = BFSDeepCrawlStrategy(
|
||||
max_depth=1,
|
||||
max_pages=5,
|
||||
on_state_change=capture_until_crash,
|
||||
)
|
||||
|
||||
config1 = CrawlerRunConfig(
|
||||
deep_crawl_strategy=strategy1,
|
||||
verbose=False,
|
||||
)
|
||||
|
||||
# Run until "crash"
|
||||
try:
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
await crawler.arun("https://books.toscrape.com", config=config1)
|
||||
except Exception:
|
||||
pass # Expected crash
|
||||
|
||||
if not captured_states:
|
||||
record_result("Resume from Checkpoint", "resume_state", False,
|
||||
"No state captured before crash")
|
||||
return
|
||||
|
||||
saved_state = captured_states[-1]
|
||||
print(f" Phase 1: Crawled {len(phase1_urls)} URLs before crash")
|
||||
|
||||
# Phase 2: Resume from checkpoint
|
||||
phase2_urls: List[str] = []
|
||||
|
||||
async def track_phase2(state: Dict[str, Any]):
|
||||
new_urls = set(state["visited"]) - set(saved_state["visited"])
|
||||
for url in new_urls:
|
||||
if url not in phase2_urls:
|
||||
phase2_urls.append(url)
|
||||
|
||||
strategy2 = BFSDeepCrawlStrategy(
|
||||
max_depth=1,
|
||||
max_pages=5,
|
||||
resume_state=saved_state, # Resume from checkpoint!
|
||||
on_state_change=track_phase2,
|
||||
)
|
||||
|
||||
config2 = CrawlerRunConfig(
|
||||
deep_crawl_strategy=strategy2,
|
||||
verbose=False,
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
await crawler.arun("https://books.toscrape.com", config=config2)
|
||||
|
||||
print(f" Phase 2: Crawled {len(phase2_urls)} new URLs after resume")
|
||||
|
||||
# Verify no duplicates
|
||||
duplicates = set(phase2_urls) & set(phase1_urls)
|
||||
if duplicates:
|
||||
record_result("Resume from Checkpoint", "resume_state", False,
|
||||
f"Re-crawled {len(duplicates)} URLs: {list(duplicates)[:2]}")
|
||||
return
|
||||
|
||||
record_result("Resume from Checkpoint", "resume_state", True,
|
||||
f"Resumed successfully, no duplicate crawls")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Resume from Checkpoint", "resume_state", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 3: Crash Recovery - State is JSON Serializable
|
||||
# =============================================================================
|
||||
async def test_crash_recovery_json_serializable():
|
||||
"""
|
||||
Verify the state dictionary can be serialized to JSON (for Redis/DB storage).
|
||||
|
||||
NEW in v0.8.0: State dictionary is designed to be JSON-serializable
|
||||
for easy storage in Redis, databases, or files.
|
||||
"""
|
||||
print_test("Crash Recovery - JSON Serializable", "State Structure")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
|
||||
|
||||
captured_state: Optional[Dict] = None
|
||||
|
||||
async def capture_state(state: Dict[str, Any]):
|
||||
nonlocal captured_state
|
||||
captured_state = state
|
||||
|
||||
strategy = BFSDeepCrawlStrategy(
|
||||
max_depth=1,
|
||||
max_pages=2,
|
||||
on_state_change=capture_state,
|
||||
)
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
deep_crawl_strategy=strategy,
|
||||
verbose=False,
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
await crawler.arun("https://books.toscrape.com", config=config)
|
||||
|
||||
if not captured_state:
|
||||
record_result("JSON Serializable", "State Structure", False,
|
||||
"No state captured")
|
||||
return
|
||||
|
||||
# Test JSON serialization round-trip
|
||||
try:
|
||||
json_str = json.dumps(captured_state)
|
||||
restored = json.loads(json_str)
|
||||
except (TypeError, json.JSONDecodeError) as e:
|
||||
record_result("JSON Serializable", "State Structure", False,
|
||||
f"JSON serialization failed: {e}")
|
||||
return
|
||||
|
||||
# Verify state structure
|
||||
required_fields = ["strategy_type", "visited", "pending", "depths", "pages_crawled"]
|
||||
missing = [f for f in required_fields if f not in restored]
|
||||
if missing:
|
||||
record_result("JSON Serializable", "State Structure", False,
|
||||
f"Missing fields: {missing}")
|
||||
return
|
||||
|
||||
# Verify types
|
||||
if not isinstance(restored["visited"], list):
|
||||
record_result("JSON Serializable", "State Structure", False,
|
||||
"visited is not a list")
|
||||
return
|
||||
|
||||
if not isinstance(restored["pages_crawled"], int):
|
||||
record_result("JSON Serializable", "State Structure", False,
|
||||
"pages_crawled is not an int")
|
||||
return
|
||||
|
||||
record_result("JSON Serializable", "State Structure", True,
|
||||
f"State serializes to {len(json_str)} bytes, all fields present")
|
||||
|
||||
except Exception as e:
|
||||
record_result("JSON Serializable", "State Structure", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 4: Prefetch Mode - Returns HTML and Links Only
|
||||
# =============================================================================
|
||||
async def test_prefetch_returns_html_links():
|
||||
"""
|
||||
Verify prefetch mode returns HTML and links but skips markdown generation.
|
||||
|
||||
NEW in v0.8.0: CrawlerRunConfig accepts prefetch=True for fast
|
||||
URL discovery without heavy processing.
|
||||
"""
|
||||
print_test("Prefetch Mode - HTML and Links", "prefetch=True")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
|
||||
config = CrawlerRunConfig(prefetch=True)
|
||||
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
result = await crawler.arun("https://books.toscrape.com", config=config)
|
||||
|
||||
# Verify HTML is present
|
||||
if not result.html or len(result.html) < 100:
|
||||
record_result("Prefetch HTML/Links", "prefetch=True", False,
|
||||
"HTML not returned or too short")
|
||||
return
|
||||
|
||||
# Verify links are present
|
||||
if not result.links:
|
||||
record_result("Prefetch HTML/Links", "prefetch=True", False,
|
||||
"Links not returned")
|
||||
return
|
||||
|
||||
internal_count = len(result.links.get("internal", []))
|
||||
external_count = len(result.links.get("external", []))
|
||||
|
||||
if internal_count == 0:
|
||||
record_result("Prefetch HTML/Links", "prefetch=True", False,
|
||||
"No internal links extracted")
|
||||
return
|
||||
|
||||
record_result("Prefetch HTML/Links", "prefetch=True", True,
|
||||
f"HTML: {len(result.html)} chars, Links: {internal_count} internal, {external_count} external")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Prefetch HTML/Links", "prefetch=True", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 5: Prefetch Mode - Skips Heavy Processing
|
||||
# =============================================================================
|
||||
async def test_prefetch_skips_processing():
|
||||
"""
|
||||
Verify prefetch mode skips markdown generation and content extraction.
|
||||
|
||||
NEW in v0.8.0: prefetch=True skips markdown generation, content scraping,
|
||||
media extraction, and LLM extraction for maximum speed.
|
||||
"""
|
||||
print_test("Prefetch Mode - Skips Processing", "prefetch=True")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
|
||||
config = CrawlerRunConfig(prefetch=True)
|
||||
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
result = await crawler.arun("https://books.toscrape.com", config=config)
|
||||
|
||||
# Check that heavy processing was skipped
|
||||
checks = []
|
||||
|
||||
# Markdown should be None or empty
|
||||
if result.markdown is None:
|
||||
checks.append("markdown=None")
|
||||
elif hasattr(result.markdown, 'raw_markdown') and result.markdown.raw_markdown is None:
|
||||
checks.append("raw_markdown=None")
|
||||
else:
|
||||
record_result("Prefetch Skips Processing", "prefetch=True", False,
|
||||
f"Markdown was generated (should be skipped)")
|
||||
return
|
||||
|
||||
# cleaned_html should be None
|
||||
if result.cleaned_html is None:
|
||||
checks.append("cleaned_html=None")
|
||||
else:
|
||||
record_result("Prefetch Skips Processing", "prefetch=True", False,
|
||||
"cleaned_html was generated (should be skipped)")
|
||||
return
|
||||
|
||||
# extracted_content should be None
|
||||
if result.extracted_content is None:
|
||||
checks.append("extracted_content=None")
|
||||
|
||||
record_result("Prefetch Skips Processing", "prefetch=True", True,
|
||||
f"Heavy processing skipped: {', '.join(checks)}")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Prefetch Skips Processing", "prefetch=True", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 6: Prefetch Mode - Two-Phase Crawl Pattern
|
||||
# =============================================================================
|
||||
async def test_prefetch_two_phase():
|
||||
"""
|
||||
Verify the two-phase crawl pattern: prefetch for discovery, then full processing.
|
||||
|
||||
NEW in v0.8.0: Prefetch mode enables efficient two-phase crawling where
|
||||
you discover URLs quickly, then selectively process important ones.
|
||||
"""
|
||||
print_test("Prefetch Mode - Two-Phase Crawl", "Two-Phase Pattern")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
# Phase 1: Fast discovery with prefetch
|
||||
prefetch_config = CrawlerRunConfig(prefetch=True)
|
||||
|
||||
start = time.time()
|
||||
discovery = await crawler.arun("https://books.toscrape.com", config=prefetch_config)
|
||||
prefetch_time = time.time() - start
|
||||
|
||||
all_urls = [link["href"] for link in discovery.links.get("internal", [])]
|
||||
|
||||
# Filter to specific pages (e.g., book detail pages)
|
||||
book_urls = [
|
||||
url for url in all_urls
|
||||
if "catalogue/" in url and "category/" not in url
|
||||
][:2] # Just 2 for demo
|
||||
|
||||
print(f" Phase 1: Found {len(all_urls)} URLs in {prefetch_time:.2f}s")
|
||||
print(f" Filtered to {len(book_urls)} book pages for full processing")
|
||||
|
||||
if len(book_urls) == 0:
|
||||
record_result("Two-Phase Crawl", "Two-Phase Pattern", False,
|
||||
"No book URLs found to process")
|
||||
return
|
||||
|
||||
# Phase 2: Full processing on selected URLs
|
||||
full_config = CrawlerRunConfig() # Normal mode
|
||||
|
||||
start = time.time()
|
||||
processed = []
|
||||
for url in book_urls:
|
||||
result = await crawler.arun(url, config=full_config)
|
||||
if result.success and result.markdown:
|
||||
processed.append(result)
|
||||
|
||||
full_time = time.time() - start
|
||||
|
||||
print(f" Phase 2: Processed {len(processed)} pages in {full_time:.2f}s")
|
||||
|
||||
if len(processed) == 0:
|
||||
record_result("Two-Phase Crawl", "Two-Phase Pattern", False,
|
||||
"No pages successfully processed in phase 2")
|
||||
return
|
||||
|
||||
# Verify full processing includes markdown
|
||||
if not processed[0].markdown or not processed[0].markdown.raw_markdown:
|
||||
record_result("Two-Phase Crawl", "Two-Phase Pattern", False,
|
||||
"Full processing did not generate markdown")
|
||||
return
|
||||
|
||||
record_result("Two-Phase Crawl", "Two-Phase Pattern", True,
|
||||
f"Discovered {len(all_urls)} URLs (prefetch), processed {len(processed)} (full)")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Two-Phase Crawl", "Two-Phase Pattern", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 7: Security - Hooks Disabled by Default
|
||||
# =============================================================================
|
||||
async def test_security_hooks_disabled():
|
||||
"""
|
||||
Verify hooks are disabled by default in Docker API for security.
|
||||
|
||||
NEW in v0.8.0: Docker API hooks are disabled by default to prevent
|
||||
Remote Code Execution. Set CRAWL4AI_HOOKS_ENABLED=true to enable.
|
||||
"""
|
||||
print_test("Security - Hooks Disabled", "CRAWL4AI_HOOKS_ENABLED")
|
||||
|
||||
try:
|
||||
import os
|
||||
|
||||
# Check the default environment variable
|
||||
hooks_enabled = os.environ.get("CRAWL4AI_HOOKS_ENABLED", "false").lower()
|
||||
|
||||
if hooks_enabled == "true":
|
||||
record_result("Hooks Disabled Default", "Security", True,
|
||||
"CRAWL4AI_HOOKS_ENABLED is explicitly set to 'true' (user override)",
|
||||
skipped=True)
|
||||
return
|
||||
|
||||
# Verify default is "false"
|
||||
if hooks_enabled == "false":
|
||||
record_result("Hooks Disabled Default", "Security", True,
|
||||
"Hooks disabled by default (CRAWL4AI_HOOKS_ENABLED=false)")
|
||||
else:
|
||||
record_result("Hooks Disabled Default", "Security", True,
|
||||
f"CRAWL4AI_HOOKS_ENABLED='{hooks_enabled}' (not 'true', hooks disabled)")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Hooks Disabled Default", "Security", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 8: Comprehensive Crawl Test
|
||||
# =============================================================================
|
||||
async def test_comprehensive_crawl():
|
||||
"""
|
||||
Run a comprehensive crawl to verify overall stability with new features.
|
||||
"""
|
||||
print_test("Comprehensive Crawl Test", "Overall")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
|
||||
|
||||
async with AsyncWebCrawler(config=BrowserConfig(headless=True), verbose=False) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://httpbin.org/html",
|
||||
config=CrawlerRunConfig()
|
||||
)
|
||||
|
||||
checks = []
|
||||
|
||||
if result.success:
|
||||
checks.append("success=True")
|
||||
else:
|
||||
record_result("Comprehensive Crawl", "Overall", False,
|
||||
f"Crawl failed: {result.error_message}")
|
||||
return
|
||||
|
||||
if result.html and len(result.html) > 100:
|
||||
checks.append(f"html={len(result.html)} chars")
|
||||
|
||||
if result.markdown and result.markdown.raw_markdown:
|
||||
checks.append(f"markdown={len(result.markdown.raw_markdown)} chars")
|
||||
|
||||
if result.links:
|
||||
total_links = len(result.links.get("internal", [])) + len(result.links.get("external", []))
|
||||
checks.append(f"links={total_links}")
|
||||
|
||||
record_result("Comprehensive Crawl", "Overall", True,
|
||||
f"All checks passed: {', '.join(checks)}")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Comprehensive Crawl", "Overall", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MAIN
|
||||
# =============================================================================
|
||||
|
||||
def print_summary():
|
||||
"""Print test results summary"""
|
||||
print_header("TEST RESULTS SUMMARY")
|
||||
|
||||
passed = sum(1 for r in results if r.passed and not r.skipped)
|
||||
failed = sum(1 for r in results if not r.passed and not r.skipped)
|
||||
skipped = sum(1 for r in results if r.skipped)
|
||||
|
||||
print(f"\nTotal: {len(results)} tests")
|
||||
print(f" Passed: {passed}")
|
||||
print(f" Failed: {failed}")
|
||||
print(f" Skipped: {skipped}")
|
||||
|
||||
if failed > 0:
|
||||
print("\nFailed Tests:")
|
||||
for r in results:
|
||||
if not r.passed and not r.skipped:
|
||||
print(f" - {r.name} ({r.feature}): {r.message}")
|
||||
|
||||
if skipped > 0:
|
||||
print("\nSkipped Tests:")
|
||||
for r in results:
|
||||
if r.skipped:
|
||||
print(f" - {r.name} ({r.feature}): {r.message}")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
if failed == 0:
|
||||
print("All tests passed! v0.8.0 features verified.")
|
||||
else:
|
||||
print(f"WARNING: {failed} test(s) failed!")
|
||||
print("=" * 70)
|
||||
|
||||
return failed == 0
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all verification tests"""
|
||||
print_header("Crawl4AI v0.8.0 - Feature Verification Tests")
|
||||
print("Running actual tests to verify new features...")
|
||||
print("\nKey Features in v0.8.0:")
|
||||
print(" - Crash Recovery for Deep Crawl (resume_state, on_state_change)")
|
||||
print(" - Prefetch Mode for Fast URL Discovery (prefetch=True)")
|
||||
print(" - Security: Hooks disabled by default on Docker API")
|
||||
|
||||
# Run all tests
|
||||
tests = [
|
||||
test_crash_recovery_state_capture, # on_state_change
|
||||
test_crash_recovery_resume, # resume_state
|
||||
test_crash_recovery_json_serializable, # State structure
|
||||
test_prefetch_returns_html_links, # prefetch=True basics
|
||||
test_prefetch_skips_processing, # prefetch skips heavy work
|
||||
test_prefetch_two_phase, # Two-phase pattern
|
||||
test_security_hooks_disabled, # Security check
|
||||
test_comprehensive_crawl, # Overall stability
|
||||
]
|
||||
|
||||
for test_func in tests:
|
||||
try:
|
||||
await test_func()
|
||||
except Exception as e:
|
||||
print(f"\nTest {test_func.__name__} crashed: {e}")
|
||||
results.append(TestResult(
|
||||
test_func.__name__,
|
||||
"Unknown",
|
||||
False,
|
||||
f"Crashed: {e}"
|
||||
))
|
||||
|
||||
# Print summary
|
||||
all_passed = print_summary()
|
||||
|
||||
return 0 if all_passed else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
exit_code = asyncio.run(main())
|
||||
sys.exit(exit_code)
|
||||
except KeyboardInterrupt:
|
||||
print("\n\nTests interrupted by user.")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"\n\nTest suite failed: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,913 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Crawl4AI v0.8.5 Release Demo - Feature Verification Tests
|
||||
==========================================================
|
||||
|
||||
This demo ACTUALLY RUNS and VERIFIES the new features in v0.8.5.
|
||||
Each test executes real crawls and validates the feature is working.
|
||||
|
||||
New Features Verified:
|
||||
1. Anti-bot detection - Detects blocked pages and passes normal ones
|
||||
2. Anti-bot + crawl_stats - Real crawl produces crawl_stats tracking
|
||||
3. Proxy escalation chain - proxy_config accepts a list with DIRECT
|
||||
4. Config defaults API - set_defaults affects real crawls
|
||||
5. Shadow DOM flattening - Crawl a shadow-DOM site with/without flattening
|
||||
6. Deep crawl cancellation - DFS crawl stops at callback limit
|
||||
7. Consent popup removal - Crawl with remove_consent_popups enabled
|
||||
8. Source/sibling selector - Extract from sibling elements via "source" field
|
||||
9. GFM table compliance - Crawl a page with tables, verify pipe delimiters
|
||||
10. avoid_ads / avoid_css - Crawl with resource filtering enabled
|
||||
11. Browser recycling - Crawl multiple pages with memory_saving_mode
|
||||
12. BM25 content filter dedup - fit_markdown has no duplicate chunks
|
||||
13. cleaned_html preserves class/id - Verify attributes retained after crawl
|
||||
|
||||
Usage:
|
||||
python docs/releases_review/demo_v0.8.5.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from typing import Dict, Any, List, Optional
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
# Test results tracking
|
||||
@dataclass
|
||||
class TestResult:
|
||||
name: str
|
||||
feature: str
|
||||
passed: bool
|
||||
message: str
|
||||
skipped: bool = False
|
||||
|
||||
|
||||
results: list[TestResult] = []
|
||||
|
||||
|
||||
def print_header(title: str):
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f"{title}")
|
||||
print(f"{'=' * 70}")
|
||||
|
||||
|
||||
def print_test(name: str, feature: str):
|
||||
print(f"\n[TEST] {name} ({feature})")
|
||||
print("-" * 50)
|
||||
|
||||
|
||||
def record_result(name: str, feature: str, passed: bool, message: str, skipped: bool = False):
|
||||
results.append(TestResult(name, feature, passed, message, skipped))
|
||||
if skipped:
|
||||
print(f" SKIPPED: {message}")
|
||||
elif passed:
|
||||
print(f" PASSED: {message}")
|
||||
else:
|
||||
print(f" FAILED: {message}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 1: Anti-bot Detection - Unit + Live Crawl
|
||||
# =============================================================================
|
||||
async def test_antibot_detection():
|
||||
"""
|
||||
Verify is_blocked() detects blocked pages and a real crawl to a normal
|
||||
site succeeds without false positives.
|
||||
|
||||
NEW in v0.8.5: 3-tier anti-bot detection (status codes, content patterns,
|
||||
structural integrity) with automatic retry and fallback.
|
||||
"""
|
||||
print_test("Anti-bot Detection", "is_blocked() + live crawl")
|
||||
|
||||
try:
|
||||
from crawl4ai.antibot_detector import is_blocked
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
|
||||
# Unit: blocked page detected
|
||||
blocked, reason = is_blocked(
|
||||
403,
|
||||
'<html><body><h1>Please verify you are human</h1>'
|
||||
'<p>Checking your browser...</p></body></html>',
|
||||
)
|
||||
if not blocked:
|
||||
record_result("Anti-bot Detection", "is_blocked()", False,
|
||||
"Failed to detect challenge page")
|
||||
return
|
||||
|
||||
# Unit: JSON response not flagged
|
||||
blocked, _ = is_blocked(
|
||||
200,
|
||||
'<html><head></head><body><pre>{"status":"ok"}</pre></body></html>',
|
||||
)
|
||||
if blocked:
|
||||
record_result("Anti-bot Detection", "is_blocked()", False,
|
||||
"False positive on JSON response")
|
||||
return
|
||||
|
||||
# Live: crawl a normal site, verify no false positive
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
result = await crawler.arun(
|
||||
"https://quotes.toscrape.com",
|
||||
config=CrawlerRunConfig(),
|
||||
)
|
||||
|
||||
if not result.success:
|
||||
record_result("Anti-bot Detection", "live crawl", False,
|
||||
f"Normal site crawl failed: {result.error_message}")
|
||||
return
|
||||
|
||||
record_result("Anti-bot Detection", "is_blocked() + live crawl", True,
|
||||
f"Detects blocks, no false positive on live crawl "
|
||||
f"({len(result.html)} chars)")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Anti-bot Detection", "is_blocked()", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 2: Anti-bot crawl_stats Tracking
|
||||
# =============================================================================
|
||||
async def test_crawl_stats():
|
||||
"""
|
||||
Verify a real crawl produces crawl_stats with proxy/fallback tracking.
|
||||
|
||||
NEW in v0.8.5: CrawlResult includes crawl_stats dict tracking which
|
||||
proxies were used, whether fallback was invoked, and how it resolved.
|
||||
"""
|
||||
print_test("Crawl Stats Tracking", "crawl_stats on CrawlResult")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
result = await crawler.arun(
|
||||
"https://example.com",
|
||||
config=CrawlerRunConfig(),
|
||||
)
|
||||
|
||||
if not result.success:
|
||||
record_result("Crawl Stats", "crawl_stats", False,
|
||||
f"Crawl failed: {result.error_message}")
|
||||
return
|
||||
|
||||
stats = getattr(result, "crawl_stats", None)
|
||||
if stats is None:
|
||||
record_result("Crawl Stats", "crawl_stats", False,
|
||||
"crawl_stats not present on CrawlResult")
|
||||
return
|
||||
|
||||
# Check expected fields
|
||||
has_proxies = "proxies_used" in stats
|
||||
has_resolved = "resolved_by" in stats
|
||||
|
||||
if not has_proxies or not has_resolved:
|
||||
record_result("Crawl Stats", "crawl_stats", False,
|
||||
f"Missing fields. Keys: {list(stats.keys())}")
|
||||
return
|
||||
|
||||
record_result("Crawl Stats", "crawl_stats", True,
|
||||
f"Stats present: resolved_by={stats.get('resolved_by')}, "
|
||||
f"proxies_used={len(stats.get('proxies_used', []))} entries")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Crawl Stats", "crawl_stats", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 3: Proxy Escalation Chain + DIRECT Sentinel
|
||||
# =============================================================================
|
||||
async def test_proxy_escalation():
|
||||
"""
|
||||
Verify proxy_config accepts a list and DIRECT sentinel, then crawl
|
||||
with DIRECT-only to prove the escalation path works.
|
||||
|
||||
NEW in v0.8.5: proxy_config can be a list of ProxyConfig/None for
|
||||
escalation. ProxyConfig.DIRECT normalizes to None (no proxy).
|
||||
"""
|
||||
print_test("Proxy Escalation Chain", "list proxy_config + DIRECT crawl")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
from crawl4ai.async_configs import ProxyConfig
|
||||
|
||||
# Verify DIRECT normalizes correctly in a list
|
||||
config = CrawlerRunConfig(
|
||||
proxy_config=[ProxyConfig.DIRECT],
|
||||
)
|
||||
if not isinstance(config.proxy_config, list):
|
||||
record_result("Proxy Escalation", "list config", False,
|
||||
f"proxy_config is {type(config.proxy_config)}, expected list")
|
||||
return
|
||||
|
||||
# Live crawl with DIRECT (no proxy)
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
result = await crawler.arun(
|
||||
"https://example.com",
|
||||
config=config,
|
||||
)
|
||||
|
||||
if not result.success:
|
||||
record_result("Proxy Escalation", "DIRECT crawl", False,
|
||||
f"DIRECT crawl failed: {result.error_message}")
|
||||
return
|
||||
|
||||
record_result("Proxy Escalation", "list + DIRECT crawl", True,
|
||||
f"List config accepted, DIRECT crawl succeeded "
|
||||
f"({len(result.html)} chars)")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Proxy Escalation", "proxy_config list", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 4: Config Defaults API — Real Crawl
|
||||
# =============================================================================
|
||||
async def test_config_defaults():
|
||||
"""
|
||||
Set text_mode=True as a default, then crawl and verify it took effect.
|
||||
|
||||
NEW in v0.8.5: BrowserConfig.set_defaults() / get_defaults() /
|
||||
reset_defaults() persist across all new instances.
|
||||
"""
|
||||
print_test("Config Defaults API", "set_defaults → real crawl")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler
|
||||
from crawl4ai.async_configs import BrowserConfig, CrawlerRunConfig
|
||||
|
||||
original = BrowserConfig.get_defaults()
|
||||
|
||||
try:
|
||||
# Set text_mode as default (disables image loading)
|
||||
BrowserConfig.set_defaults(text_mode=True, headless=True)
|
||||
|
||||
# Verify it applies
|
||||
bc = BrowserConfig()
|
||||
if not bc.text_mode:
|
||||
record_result("Config Defaults", "set_defaults", False,
|
||||
"text_mode default not applied")
|
||||
return
|
||||
|
||||
# Verify explicit override wins
|
||||
bc2 = BrowserConfig(text_mode=False)
|
||||
if bc2.text_mode:
|
||||
record_result("Config Defaults", "set_defaults", False,
|
||||
"Explicit override didn't work")
|
||||
return
|
||||
|
||||
# Real crawl with default text_mode
|
||||
async with AsyncWebCrawler(config=BrowserConfig(), verbose=False) as crawler:
|
||||
result = await crawler.arun(
|
||||
"https://example.com",
|
||||
config=CrawlerRunConfig(),
|
||||
)
|
||||
|
||||
if not result.success:
|
||||
record_result("Config Defaults", "crawl with defaults", False,
|
||||
f"Crawl failed: {result.error_message}")
|
||||
return
|
||||
|
||||
# Verify reset works
|
||||
BrowserConfig.reset_defaults()
|
||||
if BrowserConfig.get_defaults():
|
||||
record_result("Config Defaults", "reset_defaults", False,
|
||||
"Defaults not cleared after reset")
|
||||
return
|
||||
|
||||
record_result("Config Defaults", "set/get/reset + crawl", True,
|
||||
f"Defaults applied to crawl, override works, reset clears "
|
||||
f"({len(result.markdown.raw_markdown)} chars markdown)")
|
||||
|
||||
finally:
|
||||
BrowserConfig.reset_defaults()
|
||||
if original:
|
||||
BrowserConfig.set_defaults(**original)
|
||||
|
||||
except Exception as e:
|
||||
record_result("Config Defaults", "set/get/reset_defaults", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 5: Shadow DOM Flattening — Comparison Crawl
|
||||
# =============================================================================
|
||||
async def test_shadow_dom_flattening():
|
||||
"""
|
||||
Crawl a page with and without flatten_shadow_dom and compare content.
|
||||
|
||||
NEW in v0.8.5: CrawlerRunConfig.flatten_shadow_dom serializes shadow DOM
|
||||
into the light DOM, exposing hidden content to extraction.
|
||||
"""
|
||||
print_test("Shadow DOM Flattening", "comparison crawl")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig
|
||||
|
||||
# Use a page known to use web components / shadow DOM
|
||||
# (GitHub uses shadow DOM for some components)
|
||||
url = "https://books.toscrape.com"
|
||||
|
||||
async with AsyncWebCrawler(
|
||||
config=BrowserConfig(headless=True),
|
||||
verbose=False,
|
||||
) as crawler:
|
||||
# Without flattening
|
||||
result_normal = await crawler.arun(
|
||||
url, config=CrawlerRunConfig(flatten_shadow_dom=False),
|
||||
)
|
||||
|
||||
# With flattening
|
||||
result_flat = await crawler.arun(
|
||||
url, config=CrawlerRunConfig(flatten_shadow_dom=True),
|
||||
)
|
||||
|
||||
if not result_normal.success or not result_flat.success:
|
||||
record_result("Shadow DOM", "comparison crawl", False,
|
||||
"One or both crawls failed")
|
||||
return
|
||||
|
||||
normal_len = len(result_normal.html or "")
|
||||
flat_len = len(result_flat.html or "")
|
||||
|
||||
# Both should succeed (this page may not have shadow DOM, but
|
||||
# the flattening pipeline should run without error)
|
||||
record_result("Shadow DOM", "flatten_shadow_dom", True,
|
||||
f"Both crawls succeeded. Normal: {normal_len} chars, "
|
||||
f"Flattened: {flat_len} chars. Pipeline runs cleanly.")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Shadow DOM", "flatten_shadow_dom", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 6: Deep Crawl Cancellation — DFS with should_cancel
|
||||
# =============================================================================
|
||||
async def test_deep_crawl_cancellation():
|
||||
"""
|
||||
Run a DFS deep crawl and cancel after 2 pages via should_cancel callback.
|
||||
|
||||
NEW in v0.8.5: All deep crawl strategies support cancel() method and
|
||||
should_cancel callback for graceful cancellation.
|
||||
"""
|
||||
print_test("Deep Crawl Cancellation", "DFS cancel after 2 pages")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
from crawl4ai.deep_crawling import DFSDeepCrawlStrategy
|
||||
|
||||
pages_crawled = 0
|
||||
|
||||
def should_cancel():
|
||||
return pages_crawled >= 2
|
||||
|
||||
async def track_state(state: Dict[str, Any]):
|
||||
nonlocal pages_crawled
|
||||
pages_crawled = state.get("pages_crawled", 0)
|
||||
|
||||
strategy = DFSDeepCrawlStrategy(
|
||||
max_depth=1,
|
||||
max_pages=10,
|
||||
should_cancel=should_cancel,
|
||||
on_state_change=track_state,
|
||||
)
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
deep_crawl_strategy=strategy,
|
||||
verbose=False,
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
await crawler.arun("https://books.toscrape.com", config=config)
|
||||
|
||||
if strategy.cancelled:
|
||||
record_result("Deep Crawl Cancel", "should_cancel", True,
|
||||
f"Cancelled after {pages_crawled} pages (limit was 2)")
|
||||
elif pages_crawled <= 3:
|
||||
record_result("Deep Crawl Cancel", "should_cancel", True,
|
||||
f"Stopped at {pages_crawled} pages (callback triggered)")
|
||||
else:
|
||||
record_result("Deep Crawl Cancel", "should_cancel", False,
|
||||
f"Crawled {pages_crawled} pages — cancellation didn't work")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Deep Crawl Cancel", "should_cancel", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 7: Consent Popup Removal — Real Crawl
|
||||
# =============================================================================
|
||||
async def test_consent_popup_removal():
|
||||
"""
|
||||
Crawl a site with remove_consent_popups=True and verify the JS runs
|
||||
without errors and content is still captured.
|
||||
|
||||
NEW in v0.8.5: CrawlerRunConfig.remove_consent_popups runs a JS snippet
|
||||
that clicks "Accept All" on 40+ CMP platforms.
|
||||
"""
|
||||
print_test("Consent Popup Removal", "crawl with remove_consent_popups")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
result = await crawler.arun(
|
||||
"https://quotes.toscrape.com",
|
||||
config=CrawlerRunConfig(remove_consent_popups=True),
|
||||
)
|
||||
|
||||
if not result.success:
|
||||
record_result("Consent Popup", "remove_consent_popups", False,
|
||||
f"Crawl failed: {result.error_message}")
|
||||
return
|
||||
|
||||
md = result.markdown.raw_markdown if result.markdown else ""
|
||||
if len(md) < 50:
|
||||
record_result("Consent Popup", "remove_consent_popups", False,
|
||||
"Content too short — JS may have broken the page")
|
||||
return
|
||||
|
||||
record_result("Consent Popup", "remove_consent_popups", True,
|
||||
f"Crawl succeeded with consent popup removal "
|
||||
f"({len(md)} chars markdown)")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Consent Popup", "remove_consent_popups", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 8: Source/Sibling Selector — Extract from Real Crawled HTML
|
||||
# =============================================================================
|
||||
async def test_source_sibling_selector():
|
||||
"""
|
||||
Crawl a page, then use JsonCssExtractionStrategy with "source" field
|
||||
to extract data spanning sibling elements.
|
||||
|
||||
NEW in v0.8.5: "source": "+ selector" navigates to sibling elements
|
||||
before applying the field selector. Works in CSS and XPath strategies.
|
||||
"""
|
||||
print_test("Source/Sibling Selector", "crawl + extract with source field")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
|
||||
|
||||
# Use a schema with source field on synthetic HTML first to verify
|
||||
# the feature works, then also run it through a real crawl pipeline
|
||||
schema = {
|
||||
"name": "SiblingItems",
|
||||
"baseSelector": "tr.athing",
|
||||
"fields": [
|
||||
{"name": "title", "selector": ".titleline > a", "type": "text"},
|
||||
{"name": "score", "selector": ".score", "type": "text", "source": "+ tr"},
|
||||
],
|
||||
}
|
||||
|
||||
strategy = JsonCssExtractionStrategy(schema=schema)
|
||||
|
||||
# Test with sibling HTML structure
|
||||
html = """
|
||||
<html><body><table>
|
||||
<tr class="athing" id="1">
|
||||
<td><span class="titleline"><a href="http://ex.com">Article One</a></span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><span class="score">250 points</span></td>
|
||||
</tr>
|
||||
<tr class="athing" id="2">
|
||||
<td><span class="titleline"><a href="http://ex.com/2">Article Two</a></span></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td><span class="score">180 points</span></td>
|
||||
</tr>
|
||||
</table></body></html>
|
||||
"""
|
||||
|
||||
# Run through the full crawl pipeline with raw: URL
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
result = await crawler.arun(
|
||||
f"raw:{html}",
|
||||
config=CrawlerRunConfig(
|
||||
extraction_strategy=strategy,
|
||||
),
|
||||
)
|
||||
|
||||
if not result.extracted_content:
|
||||
record_result("Sibling Selector", "source field", False,
|
||||
"No extracted_content returned")
|
||||
return
|
||||
|
||||
data = json.loads(result.extracted_content)
|
||||
|
||||
if len(data) < 2:
|
||||
record_result("Sibling Selector", "source field", False,
|
||||
f"Expected 2 items, got {len(data)}")
|
||||
return
|
||||
|
||||
if data[0].get("title") != "Article One":
|
||||
record_result("Sibling Selector", "source field", False,
|
||||
f"Title mismatch: {data[0].get('title')}")
|
||||
return
|
||||
|
||||
if data[0].get("score") != "250 points":
|
||||
record_result("Sibling Selector", "source field", False,
|
||||
f"Sibling score not extracted: {data[0].get('score')}")
|
||||
return
|
||||
|
||||
if data[1].get("score") != "180 points":
|
||||
record_result("Sibling Selector", "source field", False,
|
||||
f"Second sibling score wrong: {data[1].get('score')}")
|
||||
return
|
||||
|
||||
record_result("Sibling Selector", "source field via crawl pipeline", True,
|
||||
f"Extracted {len(data)} items with sibling scores through "
|
||||
f"full arun() pipeline")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Sibling Selector", "source field", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 9: GFM Table Compliance — Crawl Page with Tables
|
||||
# =============================================================================
|
||||
async def test_gfm_tables():
|
||||
"""
|
||||
Crawl a page containing HTML tables and verify the markdown output
|
||||
has proper GFM pipe delimiters.
|
||||
|
||||
NEW in v0.8.5: html2text now generates | col1 | col2 | with proper
|
||||
leading/trailing pipes instead of col1 | col2.
|
||||
"""
|
||||
print_test("GFM Table Compliance", "crawl page with tables")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
|
||||
# Use raw HTML with a table
|
||||
html = """
|
||||
<html><body>
|
||||
<h1>Product Comparison</h1>
|
||||
<table>
|
||||
<tr><th>Product</th><th>Price</th><th>Rating</th></tr>
|
||||
<tr><td>Widget A</td><td>$9.99</td><td>4.5</td></tr>
|
||||
<tr><td>Widget B</td><td>$14.99</td><td>4.8</td></tr>
|
||||
</table>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
result = await crawler.arun(
|
||||
f"raw:{html}",
|
||||
config=CrawlerRunConfig(),
|
||||
)
|
||||
|
||||
if not result.success or not result.markdown:
|
||||
record_result("GFM Tables", "table crawl", False,
|
||||
"Crawl failed or no markdown")
|
||||
return
|
||||
|
||||
md = result.markdown.raw_markdown
|
||||
table_lines = [
|
||||
l.strip() for l in md.split("\n")
|
||||
if l.strip() and "|" in l
|
||||
]
|
||||
|
||||
if not table_lines:
|
||||
record_result("GFM Tables", "pipe delimiters", False,
|
||||
f"No table lines found in markdown:\n{md}")
|
||||
return
|
||||
|
||||
all_have_pipes = all(
|
||||
l.startswith("|") and l.endswith("|")
|
||||
for l in table_lines
|
||||
)
|
||||
|
||||
if not all_have_pipes:
|
||||
record_result("GFM Tables", "pipe delimiters", False,
|
||||
f"Missing leading/trailing pipes:\n" +
|
||||
"\n".join(table_lines))
|
||||
return
|
||||
|
||||
record_result("GFM Tables", "pipe delimiters via crawl", True,
|
||||
f"Table has proper GFM pipes ({len(table_lines)} rows)")
|
||||
|
||||
except Exception as e:
|
||||
record_result("GFM Tables", "pipe delimiters", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 10: avoid_ads / avoid_css — Real Crawl with Filtering
|
||||
# =============================================================================
|
||||
async def test_avoid_ads():
|
||||
"""
|
||||
Crawl a real page with avoid_ads=True and verify content is still captured.
|
||||
|
||||
NEW in v0.8.5: BrowserConfig.avoid_ads blocks ad/tracker domains,
|
||||
BrowserConfig.avoid_css blocks CSS resources at the network level.
|
||||
"""
|
||||
print_test("Resource Filtering", "crawl with avoid_ads + avoid_css")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
|
||||
|
||||
# Crawl with ad blocking enabled
|
||||
async with AsyncWebCrawler(
|
||||
config=BrowserConfig(
|
||||
headless=True,
|
||||
avoid_ads=True,
|
||||
avoid_css=True,
|
||||
),
|
||||
verbose=False,
|
||||
) as crawler:
|
||||
result = await crawler.arun(
|
||||
"https://quotes.toscrape.com",
|
||||
config=CrawlerRunConfig(),
|
||||
)
|
||||
|
||||
if not result.success:
|
||||
record_result("Resource Filtering", "avoid_ads crawl", False,
|
||||
f"Crawl failed: {result.error_message}")
|
||||
return
|
||||
|
||||
md = result.markdown.raw_markdown if result.markdown else ""
|
||||
|
||||
# Verify actual content was captured (quotes should be there)
|
||||
has_quotes = "quote" in md.lower() or "albert einstein" in md.lower()
|
||||
if not has_quotes and len(md) < 100:
|
||||
record_result("Resource Filtering", "avoid_ads crawl", False,
|
||||
"Content missing — filtering may have broken the page")
|
||||
return
|
||||
|
||||
record_result("Resource Filtering", "avoid_ads + avoid_css crawl", True,
|
||||
f"Content captured with ad/CSS blocking "
|
||||
f"({len(md)} chars markdown)")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Resource Filtering", "avoid_ads/css", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 11: Browser Recycling — Multi-page Crawl with memory_saving_mode
|
||||
# =============================================================================
|
||||
async def test_browser_recycling():
|
||||
"""
|
||||
Crawl multiple pages with memory_saving_mode enabled and verify
|
||||
all succeed without browser crashes.
|
||||
|
||||
NEW in v0.8.5: BrowserConfig.memory_saving_mode adds aggressive cache/V8
|
||||
flags. max_pages_before_recycle triggers automatic browser restart.
|
||||
"""
|
||||
print_test("Browser Recycling", "multi-page crawl with memory_saving_mode")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
|
||||
|
||||
urls = [
|
||||
"https://example.com",
|
||||
"https://quotes.toscrape.com",
|
||||
"https://httpbin.org/html",
|
||||
]
|
||||
|
||||
async with AsyncWebCrawler(
|
||||
config=BrowserConfig(
|
||||
headless=True,
|
||||
memory_saving_mode=True,
|
||||
),
|
||||
verbose=False,
|
||||
) as crawler:
|
||||
succeeded = 0
|
||||
for url in urls:
|
||||
result = await crawler.arun(url, config=CrawlerRunConfig())
|
||||
if result.success:
|
||||
succeeded += 1
|
||||
|
||||
if succeeded == len(urls):
|
||||
record_result("Browser Recycling", "memory_saving_mode", True,
|
||||
f"All {succeeded}/{len(urls)} crawls succeeded with "
|
||||
f"memory_saving_mode")
|
||||
else:
|
||||
record_result("Browser Recycling", "memory_saving_mode", False,
|
||||
f"Only {succeeded}/{len(urls)} crawls succeeded")
|
||||
|
||||
except Exception as e:
|
||||
record_result("Browser Recycling", "memory_saving_mode", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 12: BM25 Content Filter Deduplication
|
||||
# =============================================================================
|
||||
async def test_bm25_dedup():
|
||||
"""
|
||||
Crawl a page using BM25ContentFilter and verify no duplicate chunks
|
||||
in fit_markdown.
|
||||
|
||||
NEW in v0.8.5: BM25ContentFilter.filter_content() deduplicates output
|
||||
chunks, keeping the first occurrence in document order.
|
||||
"""
|
||||
print_test("BM25 Deduplication", "fit_markdown has no duplicates")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
from crawl4ai.content_filter_strategy import BM25ContentFilter
|
||||
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
|
||||
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
result = await crawler.arun(
|
||||
"https://quotes.toscrape.com",
|
||||
config=CrawlerRunConfig(
|
||||
markdown_generator=DefaultMarkdownGenerator(
|
||||
content_filter=BM25ContentFilter(
|
||||
user_query="famous quotes about life",
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
if not result.success:
|
||||
record_result("BM25 Dedup", "fit_markdown", False,
|
||||
f"Crawl failed: {result.error_message}")
|
||||
return
|
||||
|
||||
fit_md = result.markdown.fit_markdown if result.markdown else ""
|
||||
if not fit_md:
|
||||
record_result("BM25 Dedup", "fit_markdown", False,
|
||||
"No fit_markdown produced")
|
||||
return
|
||||
|
||||
# Check for duplicate lines (non-empty, non-header)
|
||||
lines = [l.strip() for l in fit_md.split("\n") if l.strip() and not l.startswith("#")]
|
||||
unique_lines = list(dict.fromkeys(lines)) # preserves order
|
||||
dupes = len(lines) - len(unique_lines)
|
||||
|
||||
if dupes > 0:
|
||||
record_result("BM25 Dedup", "fit_markdown", False,
|
||||
f"{dupes} duplicate lines found in fit_markdown")
|
||||
return
|
||||
|
||||
record_result("BM25 Dedup", "fit_markdown dedup", True,
|
||||
f"No duplicates in fit_markdown ({len(unique_lines)} unique lines)")
|
||||
|
||||
except Exception as e:
|
||||
record_result("BM25 Dedup", "fit_markdown", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# TEST 13: cleaned_html Preserves class and id Attributes
|
||||
# =============================================================================
|
||||
async def test_cleaned_html_attrs():
|
||||
"""
|
||||
Crawl a page and verify cleaned_html retains class and id attributes.
|
||||
|
||||
NEW in v0.8.5: 'class' and 'id' are now in IMPORTANT_ATTRS, so they
|
||||
survive HTML cleaning. Previously they were stripped.
|
||||
"""
|
||||
print_test("cleaned_html Attributes", "class and id preserved")
|
||||
|
||||
try:
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
|
||||
html = """
|
||||
<html><body>
|
||||
<div id="main-content" class="container wide">
|
||||
<h1 class="page-title">Hello World</h1>
|
||||
<p id="intro" class="lead text-muted">Introduction paragraph.</p>
|
||||
</div>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
result = await crawler.arun(
|
||||
f"raw:{html}",
|
||||
config=CrawlerRunConfig(),
|
||||
)
|
||||
|
||||
if not result.success or not result.cleaned_html:
|
||||
record_result("cleaned_html Attrs", "class/id", False,
|
||||
"Crawl failed or no cleaned_html")
|
||||
return
|
||||
|
||||
cleaned = result.cleaned_html
|
||||
checks = []
|
||||
|
||||
if 'id="main-content"' in cleaned:
|
||||
checks.append("id=main-content")
|
||||
if 'class="container wide"' in cleaned or 'class="container' in cleaned:
|
||||
checks.append("class=container")
|
||||
if 'class="page-title"' in cleaned:
|
||||
checks.append("class=page-title")
|
||||
if 'id="intro"' in cleaned:
|
||||
checks.append("id=intro")
|
||||
|
||||
if len(checks) < 2:
|
||||
record_result("cleaned_html Attrs", "class/id", False,
|
||||
f"Only found {len(checks)} attrs: {checks}. "
|
||||
f"cleaned_html snippet: {cleaned[:200]}")
|
||||
return
|
||||
|
||||
record_result("cleaned_html Attrs", "class/id preserved", True,
|
||||
f"Found {len(checks)} preserved attributes: {', '.join(checks)}")
|
||||
|
||||
except Exception as e:
|
||||
record_result("cleaned_html Attrs", "class/id", False, f"Exception: {e}")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# MAIN
|
||||
# =============================================================================
|
||||
|
||||
def print_summary():
|
||||
"""Print test results summary"""
|
||||
print_header("TEST RESULTS SUMMARY")
|
||||
|
||||
passed = sum(1 for r in results if r.passed and not r.skipped)
|
||||
failed = sum(1 for r in results if not r.passed and not r.skipped)
|
||||
skipped = sum(1 for r in results if r.skipped)
|
||||
|
||||
print(f"\nTotal: {len(results)} tests")
|
||||
print(f" Passed: {passed}")
|
||||
print(f" Failed: {failed}")
|
||||
print(f" Skipped: {skipped}")
|
||||
|
||||
if failed > 0:
|
||||
print("\nFailed Tests:")
|
||||
for r in results:
|
||||
if not r.passed and not r.skipped:
|
||||
print(f" - {r.name} ({r.feature}): {r.message}")
|
||||
|
||||
if skipped > 0:
|
||||
print("\nSkipped Tests:")
|
||||
for r in results:
|
||||
if r.skipped:
|
||||
print(f" - {r.name} ({r.feature}): {r.message}")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
if failed == 0:
|
||||
print("All tests passed! v0.8.5 features verified.")
|
||||
else:
|
||||
print(f"WARNING: {failed} test(s) failed!")
|
||||
print("=" * 70)
|
||||
|
||||
return failed == 0
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all verification tests"""
|
||||
print_header("Crawl4AI v0.8.5 - Feature Verification Tests")
|
||||
print("Running actual tests to verify new features...")
|
||||
print("\nKey Features in v0.8.5:")
|
||||
print(" - Anti-bot detection + retry + proxy escalation + fallback")
|
||||
print(" - Shadow DOM flattening (flatten_shadow_dom)")
|
||||
print(" - Deep crawl cancellation (cancel / should_cancel)")
|
||||
print(" - Config defaults API (set_defaults / get_defaults / reset_defaults)")
|
||||
print(" - Source/sibling selector in JSON extraction")
|
||||
print(" - Consent popup removal (40+ CMP platforms)")
|
||||
print(" - avoid_ads / avoid_css resource filtering")
|
||||
print(" - Browser recycling + memory-saving mode")
|
||||
print(" - GFM table compliance")
|
||||
print(" - BM25 content filter deduplication")
|
||||
print(" - cleaned_html preserves class/id attributes")
|
||||
print(" - 49+ bug fixes including critical RCE and CVE patches")
|
||||
|
||||
tests = [
|
||||
test_antibot_detection, # Anti-bot + live crawl
|
||||
test_crawl_stats, # crawl_stats tracking
|
||||
test_proxy_escalation, # Proxy chain + DIRECT crawl
|
||||
test_config_defaults, # set_defaults → real crawl
|
||||
test_shadow_dom_flattening, # Shadow DOM comparison crawl
|
||||
test_deep_crawl_cancellation, # DFS cancel at 2 pages
|
||||
test_consent_popup_removal, # Crawl with consent removal
|
||||
test_source_sibling_selector, # Sibling extraction via pipeline
|
||||
test_gfm_tables, # Table crawl with pipe check
|
||||
test_avoid_ads, # Crawl with ad/CSS blocking
|
||||
test_browser_recycling, # Multi-page memory_saving crawl
|
||||
test_bm25_dedup, # BM25 fit_markdown dedup
|
||||
test_cleaned_html_attrs, # class/id preserved
|
||||
]
|
||||
|
||||
for test_func in tests:
|
||||
try:
|
||||
await test_func()
|
||||
except Exception as e:
|
||||
print(f"\nTest {test_func.__name__} crashed: {e}")
|
||||
results.append(TestResult(
|
||||
test_func.__name__,
|
||||
"Unknown",
|
||||
False,
|
||||
f"Crashed: {e}"
|
||||
))
|
||||
|
||||
all_passed = print_summary()
|
||||
return 0 if all_passed else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
exit_code = asyncio.run(main())
|
||||
sys.exit(exit_code)
|
||||
except KeyboardInterrupt:
|
||||
print("\n\nTests interrupted by user.")
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"\n\nTest suite failed: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
sys.exit(1)
|
||||
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Crawl4AI v0.9.1 Release Demo - Feature Verification Tests
|
||||
==========================================================
|
||||
|
||||
This demo ACTUALLY RUNS and VERIFIES the key changes in v0.9.1.
|
||||
Each test executes real crawls or exercises the fix path end-to-end.
|
||||
|
||||
Features / Fixes Verified:
|
||||
1. PruningContentFilter preserve_classes / preserve_tags whitelist
|
||||
2. Windows channel='chromium' fix (channel not passed for default)
|
||||
3. page_timeout ms-to-seconds conversion for HTTP mode
|
||||
4. html2text bypass_tables preserves table attributes
|
||||
5. Best-first batch ordering stability
|
||||
|
||||
Usage:
|
||||
python docs/releases_review/demo_v0.9.1.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass
|
||||
class TestResult:
|
||||
name: str
|
||||
feature: str
|
||||
passed: bool
|
||||
message: str
|
||||
skipped: bool = False
|
||||
|
||||
|
||||
results: list[TestResult] = []
|
||||
|
||||
|
||||
def print_header(title: str):
|
||||
print(f"\n{'=' * 70}")
|
||||
print(f"{title}")
|
||||
print(f"{'=' * 70}")
|
||||
|
||||
|
||||
def print_test(name: str, feature: str):
|
||||
print(f"\n[TEST] {name} ({feature})")
|
||||
print("-" * 50)
|
||||
|
||||
|
||||
def record_result(name: str, feature: str, passed: bool, message: str, skipped: bool = False):
|
||||
results.append(TestResult(name, feature, passed, message, skipped))
|
||||
status = "SKIP" if skipped else ("PASS" if passed else "FAIL")
|
||||
print(f" [{status}] {message}")
|
||||
|
||||
|
||||
# ── Test 1: PruningContentFilter preserve_classes / preserve_tags ────
|
||||
|
||||
async def test_preserve_whitelist():
|
||||
"""Verify that whitelisted classes/tags survive pruning."""
|
||||
print_test("PruningContentFilter Whitelist", "preserve_classes / preserve_tags")
|
||||
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
|
||||
from crawl4ai.content_filter_strategy import PruningContentFilter
|
||||
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
|
||||
|
||||
# HTML where short metadata elements would normally be pruned
|
||||
html = """
|
||||
<html><body>
|
||||
<div class="article">
|
||||
<h1>Main Article Title</h1>
|
||||
<p>This is a long paragraph with enough content that the density-based
|
||||
scoring will keep it. It contains multiple sentences and substantial text
|
||||
that makes it clearly content rather than boilerplate navigation.</p>
|
||||
<span class="author">By John Doe</span>
|
||||
<time datetime="2026-07-08">July 8, 2026</time>
|
||||
<p>Another substantial paragraph with enough text to be kept by the
|
||||
pruning filter. This ensures the article has real content around the
|
||||
short metadata elements we want to test preservation for.</p>
|
||||
</div>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
# Without whitelist — author/time may be pruned
|
||||
filter_no_wl = PruningContentFilter(threshold=0.48)
|
||||
gen_no_wl = DefaultMarkdownGenerator(content_filter=filter_no_wl)
|
||||
|
||||
# With whitelist — author class and time tag protected
|
||||
filter_wl = PruningContentFilter(
|
||||
threshold=0.48,
|
||||
preserve_classes=["author"],
|
||||
preserve_tags=["time"],
|
||||
)
|
||||
gen_wl = DefaultMarkdownGenerator(content_filter=filter_wl)
|
||||
|
||||
config_no_wl = CrawlerRunConfig(
|
||||
markdown_generator=gen_no_wl,
|
||||
)
|
||||
config_wl = CrawlerRunConfig(
|
||||
markdown_generator=gen_wl,
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=BrowserConfig()) as crawler:
|
||||
result_wl = await crawler.arun(url=f"raw://{html}", config=config_wl)
|
||||
|
||||
fit = result_wl.markdown.fit_markdown or ""
|
||||
has_author = "John Doe" in fit
|
||||
has_time = "July 8, 2026" in fit
|
||||
|
||||
if has_author and has_time:
|
||||
record_result("preserve_whitelist", "PruningContentFilter", True,
|
||||
"Whitelisted class and tag preserved in fit_markdown")
|
||||
else:
|
||||
record_result("preserve_whitelist", "PruningContentFilter", False,
|
||||
f"author={'found' if has_author else 'MISSING'}, "
|
||||
f"time={'found' if has_time else 'MISSING'} in fit_markdown")
|
||||
|
||||
|
||||
# ── Test 2: channel='chromium' not passed to Playwright ──────────────
|
||||
|
||||
async def test_channel_chromium_skipped():
|
||||
"""Verify that the default 'chromium' channel is not passed to launch args."""
|
||||
print_test("Channel Chromium Skip", "Windows TargetClosedError fix")
|
||||
|
||||
from crawl4ai.browser_manager import BrowserManager
|
||||
from crawl4ai.async_configs import BrowserConfig
|
||||
|
||||
config = BrowserConfig() # default chrome_channel='chromium'
|
||||
mgr = BrowserManager(config)
|
||||
args = mgr._build_browser_args()
|
||||
|
||||
if "channel" not in args:
|
||||
record_result("channel_skip", "browser_manager", True,
|
||||
"Default 'chromium' channel correctly omitted from launch args")
|
||||
else:
|
||||
record_result("channel_skip", "browser_manager", False,
|
||||
f"channel='{args['channel']}' should not be in launch args")
|
||||
|
||||
# Verify explicit non-default channel IS passed
|
||||
config2 = BrowserConfig(chrome_channel="chrome")
|
||||
mgr2 = BrowserManager(config2)
|
||||
args2 = mgr2._build_browser_args()
|
||||
|
||||
if args2.get("channel") == "chrome":
|
||||
record_result("channel_explicit", "browser_manager", True,
|
||||
"Explicit 'chrome' channel correctly passed")
|
||||
else:
|
||||
record_result("channel_explicit", "browser_manager", False,
|
||||
f"Expected channel='chrome', got {args2.get('channel')}")
|
||||
|
||||
|
||||
# ── Test 3: page_timeout ms→s conversion ─────────────────────────────
|
||||
|
||||
async def test_page_timeout_conversion():
|
||||
"""Verify page_timeout is converted from ms to seconds for aiohttp."""
|
||||
print_test("HTTP Timeout Conversion", "page_timeout ms to seconds")
|
||||
|
||||
from crawl4ai.async_configs import CrawlerRunConfig
|
||||
|
||||
config = CrawlerRunConfig()
|
||||
|
||||
# Default page_timeout is 60000 (ms). aiohttp timeout should be 60s, not 60000s.
|
||||
import aiohttp
|
||||
timeout = aiohttp.ClientTimeout(total=config.page_timeout / 1000)
|
||||
|
||||
if timeout.total == 60.0:
|
||||
record_result("timeout_conversion", "HTTP mode", True,
|
||||
f"page_timeout {config.page_timeout}ms → {timeout.total}s correctly")
|
||||
else:
|
||||
record_result("timeout_conversion", "HTTP mode", False,
|
||||
f"Expected 60.0s, got {timeout.total}s")
|
||||
|
||||
|
||||
# ── Test 4: html2text bypass_tables preserves attributes ──────────────
|
||||
|
||||
async def test_bypass_tables_attributes():
|
||||
"""Verify table tag attributes are preserved when bypass_tables is enabled."""
|
||||
print_test("Table Attribute Preservation", "html2text bypass_tables fix")
|
||||
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, BrowserConfig
|
||||
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
|
||||
|
||||
html = """
|
||||
<html><body>
|
||||
<table class="data-table" id="results" border="1">
|
||||
<tr><th>Name</th><th>Value</th></tr>
|
||||
<tr><td>Alpha</td><td>100</td></tr>
|
||||
<tr><td>Beta</td><td>200</td></tr>
|
||||
</table>
|
||||
</body></html>
|
||||
"""
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
markdown_generator=DefaultMarkdownGenerator(),
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=BrowserConfig()) as crawler:
|
||||
result = await crawler.arun(url=f"raw://{html}", config=config)
|
||||
|
||||
md = result.markdown.raw_markdown or ""
|
||||
# Tables should render and contain content
|
||||
has_content = "Alpha" in md and "Beta" in md
|
||||
if has_content:
|
||||
record_result("bypass_tables", "html2text", True,
|
||||
"Table content preserved in markdown output")
|
||||
else:
|
||||
record_result("bypass_tables", "html2text", False,
|
||||
"Table content missing from markdown")
|
||||
|
||||
|
||||
# ── Test 5: Best-first batch ordering stability ──────────────────────
|
||||
|
||||
async def test_bestfirst_ordering():
|
||||
"""Verify best-first scorer produces deterministic ordering."""
|
||||
print_test("Best-First Ordering", "Stable batch ordering fix")
|
||||
|
||||
from crawl4ai.deep_crawling.scorers import KeywordRelevanceScorer, CompositeScorer
|
||||
|
||||
scorer = CompositeScorer([
|
||||
KeywordRelevanceScorer(["python", "crawl"], weight=1.0),
|
||||
])
|
||||
|
||||
urls = [
|
||||
"https://example.com/python-guide",
|
||||
"https://example.com/crawl-tutorial",
|
||||
"https://example.com/about",
|
||||
"https://example.com/python-crawl-tips",
|
||||
"https://example.com/contact",
|
||||
]
|
||||
|
||||
# Score multiple times — should be deterministic
|
||||
scores_run1 = [scorer.score(u) for u in urls]
|
||||
scores_run2 = [scorer.score(u) for u in urls]
|
||||
|
||||
if scores_run1 == scores_run2:
|
||||
record_result("bestfirst_stable", "deep_crawling", True,
|
||||
"Scorer produces deterministic results across runs")
|
||||
else:
|
||||
record_result("bestfirst_stable", "deep_crawling", False,
|
||||
"Scorer results differ between runs")
|
||||
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────
|
||||
|
||||
async def main():
|
||||
print_header("Crawl4AI v0.9.1 Release Verification")
|
||||
|
||||
await test_preserve_whitelist()
|
||||
await test_channel_chromium_skipped()
|
||||
await test_page_timeout_conversion()
|
||||
await test_bypass_tables_attributes()
|
||||
await test_bestfirst_ordering()
|
||||
|
||||
# Summary
|
||||
print_header("RESULTS SUMMARY")
|
||||
total = len(results)
|
||||
passed = sum(1 for r in results if r.passed and not r.skipped)
|
||||
failed = sum(1 for r in results if not r.passed and not r.skipped)
|
||||
skipped = sum(1 for r in results if r.skipped)
|
||||
|
||||
for r in results:
|
||||
status = "SKIP" if r.skipped else ("PASS" if r.passed else "FAIL")
|
||||
print(f" [{status}] {r.name}: {r.message}")
|
||||
|
||||
print(f"\nTotal: {total} | Passed: {passed} | Failed: {failed} | Skipped: {skipped}")
|
||||
|
||||
if failed > 0:
|
||||
print("\n⚠️ Some tests FAILED. Review before release.")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("\n✅ All tests passed!")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,276 @@
|
||||
import os, sys
|
||||
|
||||
# append the parent directory to the sys.path
|
||||
parent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
sys.path.append(parent_dir)
|
||||
parent_parent_dir = os.path.dirname(parent_dir)
|
||||
sys.path.append(parent_parent_dir)
|
||||
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
|
||||
__data__ = os.path.join(__location__, "__data")
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
import aiohttp
|
||||
import json
|
||||
from crawl4ai import AsyncWebCrawler, CacheMode
|
||||
from crawl4ai.content_filter_strategy import BM25ContentFilter
|
||||
|
||||
|
||||
# 1. File Download Processing Example
|
||||
async def download_example():
|
||||
"""Example of downloading files from Python.org"""
|
||||
# downloads_path = os.path.join(os.getcwd(), "downloads")
|
||||
downloads_path = os.path.join(Path.home(), ".crawl4ai", "downloads")
|
||||
os.makedirs(downloads_path, exist_ok=True)
|
||||
|
||||
print(f"Downloads will be saved to: {downloads_path}")
|
||||
|
||||
async with AsyncWebCrawler(
|
||||
accept_downloads=True, downloads_path=downloads_path, verbose=True
|
||||
) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://www.python.org/downloads/",
|
||||
js_code="""
|
||||
// Find and click the first Windows installer link
|
||||
const downloadLink = document.querySelector('a[href$=".exe"]');
|
||||
if (downloadLink) {
|
||||
console.log('Found download link:', downloadLink.href);
|
||||
downloadLink.click();
|
||||
} else {
|
||||
console.log('No .exe download link found');
|
||||
}
|
||||
""",
|
||||
delay_before_return_html=1, # Wait 5 seconds to ensure download starts
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
)
|
||||
|
||||
if result.downloaded_files:
|
||||
print("\nDownload successful!")
|
||||
print("Downloaded files:")
|
||||
for file_path in result.downloaded_files:
|
||||
print(f"- {file_path}")
|
||||
print(f" File size: {os.path.getsize(file_path) / (1024*1024):.2f} MB")
|
||||
else:
|
||||
print("\nNo files were downloaded")
|
||||
|
||||
|
||||
# 2. Local File and Raw HTML Processing Example
|
||||
async def local_and_raw_html_example():
|
||||
"""Example of processing local files and raw HTML"""
|
||||
# Create a sample HTML file
|
||||
sample_file = os.path.join(__data__, "sample.html")
|
||||
with open(sample_file, "w") as f:
|
||||
f.write(
|
||||
"""
|
||||
<html><body>
|
||||
<h1>Test Content</h1>
|
||||
<p>This is a test paragraph.</p>
|
||||
</body></html>
|
||||
"""
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(verbose=True) as crawler:
|
||||
# Process local file
|
||||
local_result = await crawler.arun(url=f"file://{os.path.abspath(sample_file)}")
|
||||
|
||||
# Process raw HTML
|
||||
raw_html = """
|
||||
<html><body>
|
||||
<h1>Raw HTML Test</h1>
|
||||
<p>This is a test of raw HTML processing.</p>
|
||||
</body></html>
|
||||
"""
|
||||
raw_result = await crawler.arun(url=f"raw:{raw_html}")
|
||||
|
||||
# Clean up
|
||||
os.remove(sample_file)
|
||||
|
||||
print("Local file content:", local_result.markdown)
|
||||
print("\nRaw HTML content:", raw_result.markdown)
|
||||
|
||||
|
||||
# 3. Enhanced Markdown Generation Example
|
||||
async def markdown_generation_example():
|
||||
"""Example of enhanced markdown generation with citations and LLM-friendly features"""
|
||||
async with AsyncWebCrawler(verbose=True) as crawler:
|
||||
# Create a content filter (optional)
|
||||
content_filter = BM25ContentFilter(
|
||||
# user_query="History and cultivation",
|
||||
bm25_threshold=1.0
|
||||
)
|
||||
|
||||
result = await crawler.arun(
|
||||
url="https://en.wikipedia.org/wiki/Apple",
|
||||
css_selector="main div#bodyContent",
|
||||
content_filter=content_filter,
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
)
|
||||
|
||||
from crawl4ai.content_filter_strategy import BM25ContentFilter
|
||||
|
||||
result = await crawler.arun(
|
||||
url="https://en.wikipedia.org/wiki/Apple",
|
||||
css_selector="main div#bodyContent",
|
||||
content_filter=BM25ContentFilter(),
|
||||
)
|
||||
print(result.markdown_v2.fit_markdown)
|
||||
|
||||
print("\nMarkdown Generation Results:")
|
||||
print(f"1. Original markdown length: {len(result.markdown)}")
|
||||
print("2. New markdown versions (markdown_v2):")
|
||||
print(f" - Raw markdown length: {len(result.markdown_v2.raw_markdown)}")
|
||||
print(
|
||||
f" - Citations markdown length: {len(result.markdown_v2.markdown_with_citations)}"
|
||||
)
|
||||
print(
|
||||
f" - References section length: {len(result.markdown_v2.references_markdown)}"
|
||||
)
|
||||
if result.markdown_v2.fit_markdown:
|
||||
print(
|
||||
f" - Filtered markdown length: {len(result.markdown_v2.fit_markdown)}"
|
||||
)
|
||||
|
||||
# Save examples to files
|
||||
output_dir = os.path.join(__data__, "markdown_examples")
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
# Save different versions
|
||||
with open(os.path.join(output_dir, "1_raw_markdown.md"), "w") as f:
|
||||
f.write(result.markdown_v2.raw_markdown)
|
||||
|
||||
with open(os.path.join(output_dir, "2_citations_markdown.md"), "w") as f:
|
||||
f.write(result.markdown_v2.markdown_with_citations)
|
||||
|
||||
with open(os.path.join(output_dir, "3_references.md"), "w") as f:
|
||||
f.write(result.markdown_v2.references_markdown)
|
||||
|
||||
if result.markdown_v2.fit_markdown:
|
||||
with open(os.path.join(output_dir, "4_filtered_markdown.md"), "w") as f:
|
||||
f.write(result.markdown_v2.fit_markdown)
|
||||
|
||||
print(f"\nMarkdown examples saved to: {output_dir}")
|
||||
|
||||
# Show a sample of citations and references
|
||||
print("\nSample of markdown with citations:")
|
||||
print(result.markdown_v2.markdown_with_citations[:500] + "...\n")
|
||||
print("Sample of references:")
|
||||
print(
|
||||
"\n".join(result.markdown_v2.references_markdown.split("\n")[:10]) + "..."
|
||||
)
|
||||
|
||||
|
||||
# 4. Browser Management Example
|
||||
async def browser_management_example():
|
||||
"""Example of using enhanced browser management features"""
|
||||
# Use the specified user directory path
|
||||
user_data_dir = os.path.join(Path.home(), ".crawl4ai", "browser_profile")
|
||||
os.makedirs(user_data_dir, exist_ok=True)
|
||||
|
||||
print(f"Browser profile will be saved to: {user_data_dir}")
|
||||
|
||||
async with AsyncWebCrawler(
|
||||
use_managed_browser=True,
|
||||
user_data_dir=user_data_dir,
|
||||
headless=False,
|
||||
verbose=True,
|
||||
) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://crawl4ai.com",
|
||||
# session_id="persistent_session_1",
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
)
|
||||
# Use GitHub as an example - it's a good test for browser management
|
||||
# because it requires proper browser handling
|
||||
result = await crawler.arun(
|
||||
url="https://github.com/trending",
|
||||
# session_id="persistent_session_1",
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
)
|
||||
|
||||
print("\nBrowser session result:", result.success)
|
||||
if result.success:
|
||||
print("Page title:", result.metadata.get("title", "No title found"))
|
||||
|
||||
|
||||
# 5. API Usage Example
|
||||
async def api_example():
|
||||
"""Example of using the new API endpoints"""
|
||||
api_token = os.getenv("CRAWL4AI_API_TOKEN") or "test_api_code"
|
||||
headers = {"Authorization": f"Bearer {api_token}"}
|
||||
async with aiohttp.ClientSession() as session:
|
||||
# Submit crawl job
|
||||
crawl_request = {
|
||||
"urls": ["https://news.ycombinator.com"], # Hacker News as an example
|
||||
"extraction_config": {
|
||||
"type": "json_css",
|
||||
"params": {
|
||||
"schema": {
|
||||
"name": "Hacker News Articles",
|
||||
"baseSelector": ".athing",
|
||||
"fields": [
|
||||
{"name": "title", "selector": ".title a", "type": "text"},
|
||||
{"name": "score", "selector": ".score", "type": "text"},
|
||||
{
|
||||
"name": "url",
|
||||
"selector": ".title a",
|
||||
"type": "attribute",
|
||||
"attribute": "href",
|
||||
},
|
||||
],
|
||||
}
|
||||
},
|
||||
},
|
||||
"crawler_params": {
|
||||
"headless": True,
|
||||
# "use_managed_browser": True
|
||||
},
|
||||
"cache_mode": "bypass",
|
||||
# "screenshot": True,
|
||||
# "magic": True
|
||||
}
|
||||
|
||||
async with session.post(
|
||||
"http://localhost:11235/crawl", json=crawl_request, headers=headers
|
||||
) as response:
|
||||
task_data = await response.json()
|
||||
task_id = task_data["task_id"]
|
||||
|
||||
# Check task status
|
||||
while True:
|
||||
async with session.get(
|
||||
f"http://localhost:11235/task/{task_id}", headers=headers
|
||||
) as status_response:
|
||||
result = await status_response.json()
|
||||
print(f"Task status: {result['status']}")
|
||||
|
||||
if result["status"] == "completed":
|
||||
print("Task completed!")
|
||||
print("Results:")
|
||||
news = json.loads(result["results"][0]["extracted_content"])
|
||||
print(json.dumps(news[:4], indent=2))
|
||||
break
|
||||
else:
|
||||
await asyncio.sleep(1)
|
||||
|
||||
|
||||
# Main execution
|
||||
async def main():
|
||||
# print("Running Crawl4AI feature examples...")
|
||||
|
||||
# print("\n1. Running Download Example:")
|
||||
# await download_example()
|
||||
|
||||
# print("\n2. Running Markdown Generation Example:")
|
||||
# await markdown_generation_example()
|
||||
|
||||
# # print("\n3. Running Local and Raw HTML Example:")
|
||||
# await local_and_raw_html_example()
|
||||
|
||||
# # print("\n4. Running Browser Management Example:")
|
||||
await browser_management_example()
|
||||
|
||||
# print("\n5. Running API Example:")
|
||||
await api_example()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,655 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
🚀 Crawl4AI v0.7.5 - Docker Hooks System Complete Demonstration
|
||||
================================================================
|
||||
|
||||
This file demonstrates the NEW Docker Hooks System introduced in v0.7.5.
|
||||
|
||||
The Docker Hooks System is a completely NEW feature that provides pipeline
|
||||
customization through user-provided Python functions. It offers three approaches:
|
||||
|
||||
1. String-based hooks for REST API
|
||||
2. hooks_to_string() utility to convert functions
|
||||
3. Docker Client with automatic conversion (most convenient)
|
||||
|
||||
All three approaches are part of this NEW v0.7.5 feature!
|
||||
|
||||
Perfect for video recording and demonstration purposes.
|
||||
|
||||
Requirements:
|
||||
- Docker container running: docker run -p 11235:11235 unclecode/crawl4ai:latest
|
||||
- crawl4ai v0.7.5 installed: pip install crawl4ai==0.7.5
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import requests
|
||||
import json
|
||||
import time
|
||||
from typing import Dict, Any
|
||||
|
||||
# Import Crawl4AI components
|
||||
from crawl4ai import hooks_to_string
|
||||
from crawl4ai.docker_client import Crawl4aiDockerClient
|
||||
|
||||
# Configuration
|
||||
DOCKER_URL = "http://localhost:11235"
|
||||
# DOCKER_URL = "http://localhost:11234"
|
||||
TEST_URLS = [
|
||||
# "https://httpbin.org/html",
|
||||
"https://www.kidocode.com",
|
||||
"https://quotes.toscrape.com",
|
||||
]
|
||||
|
||||
|
||||
def print_section(title: str, description: str = ""):
|
||||
"""Print a formatted section header"""
|
||||
print("\n" + "=" * 70)
|
||||
print(f" {title}")
|
||||
if description:
|
||||
print(f" {description}")
|
||||
print("=" * 70 + "\n")
|
||||
|
||||
|
||||
def check_docker_service() -> bool:
|
||||
"""Check if Docker service is running"""
|
||||
try:
|
||||
response = requests.get(f"{DOCKER_URL}/health", timeout=3)
|
||||
return response.status_code == 200
|
||||
except:
|
||||
return False
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# REUSABLE HOOK LIBRARY (NEW in v0.7.5)
|
||||
# ============================================================================
|
||||
|
||||
async def performance_optimization_hook(page, context, **kwargs):
|
||||
"""
|
||||
Performance Hook: Block unnecessary resources to speed up crawling
|
||||
"""
|
||||
print(" [Hook] 🚀 Optimizing performance - blocking images and ads...")
|
||||
|
||||
# Block images
|
||||
await context.route(
|
||||
"**/*.{png,jpg,jpeg,gif,webp,svg,ico}",
|
||||
lambda route: route.abort()
|
||||
)
|
||||
|
||||
# Block ads and analytics
|
||||
await context.route("**/analytics/*", lambda route: route.abort())
|
||||
await context.route("**/ads/*", lambda route: route.abort())
|
||||
await context.route("**/google-analytics.com/*", lambda route: route.abort())
|
||||
|
||||
print(" [Hook] ✓ Performance optimization applied")
|
||||
return page
|
||||
|
||||
|
||||
async def viewport_setup_hook(page, context, **kwargs):
|
||||
"""
|
||||
Viewport Hook: Set consistent viewport size for rendering
|
||||
"""
|
||||
print(" [Hook] 🖥️ Setting viewport to 1920x1080...")
|
||||
await page.set_viewport_size({"width": 1920, "height": 1080})
|
||||
print(" [Hook] ✓ Viewport configured")
|
||||
return page
|
||||
|
||||
|
||||
async def authentication_headers_hook(page, context, url, **kwargs):
|
||||
"""
|
||||
Headers Hook: Add custom authentication and tracking headers
|
||||
"""
|
||||
print(f" [Hook] 🔐 Adding custom headers for {url[:50]}...")
|
||||
|
||||
await page.set_extra_http_headers({
|
||||
'X-Crawl4AI-Version': '0.7.5',
|
||||
'X-Custom-Hook': 'function-based-demo',
|
||||
'Accept-Language': 'en-US,en;q=0.9',
|
||||
'User-Agent': 'Crawl4AI/0.7.5 (Educational Demo)'
|
||||
})
|
||||
|
||||
print(" [Hook] ✓ Custom headers added")
|
||||
return page
|
||||
|
||||
|
||||
async def lazy_loading_handler_hook(page, context, **kwargs):
|
||||
"""
|
||||
Content Hook: Handle lazy-loaded content by scrolling
|
||||
"""
|
||||
print(" [Hook] 📜 Scrolling to load lazy content...")
|
||||
|
||||
# Scroll to bottom
|
||||
await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
||||
await page.wait_for_timeout(1000)
|
||||
|
||||
# Scroll to middle
|
||||
await page.evaluate("window.scrollTo(0, document.body.scrollHeight / 2)")
|
||||
await page.wait_for_timeout(500)
|
||||
|
||||
# Scroll back to top
|
||||
await page.evaluate("window.scrollTo(0, 0)")
|
||||
await page.wait_for_timeout(500)
|
||||
|
||||
print(" [Hook] ✓ Lazy content loaded")
|
||||
return page
|
||||
|
||||
|
||||
async def page_analytics_hook(page, context, **kwargs):
|
||||
"""
|
||||
Analytics Hook: Log page metrics before extraction
|
||||
"""
|
||||
print(" [Hook] 📊 Collecting page analytics...")
|
||||
|
||||
metrics = await page.evaluate('''
|
||||
() => ({
|
||||
title: document.title,
|
||||
images: document.images.length,
|
||||
links: document.links.length,
|
||||
scripts: document.scripts.length,
|
||||
headings: document.querySelectorAll('h1, h2, h3').length,
|
||||
paragraphs: document.querySelectorAll('p').length
|
||||
})
|
||||
''')
|
||||
|
||||
print(f" [Hook] 📈 Page: {metrics['title'][:50]}...")
|
||||
print(f" Links: {metrics['links']}, Images: {metrics['images']}, "
|
||||
f"Headings: {metrics['headings']}, Paragraphs: {metrics['paragraphs']}")
|
||||
|
||||
return page
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# DEMO 1: String-Based Hooks (NEW Docker Hooks System)
|
||||
# ============================================================================
|
||||
|
||||
def demo_1_string_based_hooks():
|
||||
"""
|
||||
Demonstrate string-based hooks with REST API (part of NEW Docker Hooks System)
|
||||
"""
|
||||
print_section(
|
||||
"DEMO 1: String-Based Hooks (REST API)",
|
||||
"Part of the NEW Docker Hooks System - hooks as strings"
|
||||
)
|
||||
|
||||
# Define hooks as strings
|
||||
hooks_config = {
|
||||
"on_page_context_created": """
|
||||
async def hook(page, context, **kwargs):
|
||||
print(" [String Hook] Setting up page context...")
|
||||
# Block images for performance
|
||||
await context.route("**/*.{png,jpg,jpeg,gif,webp}", lambda route: route.abort())
|
||||
await page.set_viewport_size({"width": 1920, "height": 1080})
|
||||
return page
|
||||
""",
|
||||
|
||||
"before_goto": """
|
||||
async def hook(page, context, url, **kwargs):
|
||||
print(f" [String Hook] Navigating to {url[:50]}...")
|
||||
await page.set_extra_http_headers({
|
||||
'X-Crawl4AI': 'string-based-hooks',
|
||||
'X-Demo': 'v0.7.5'
|
||||
})
|
||||
return page
|
||||
""",
|
||||
|
||||
"before_retrieve_html": """
|
||||
async def hook(page, context, **kwargs):
|
||||
print(" [String Hook] Scrolling page...")
|
||||
await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
||||
await page.wait_for_timeout(1000)
|
||||
return page
|
||||
"""
|
||||
}
|
||||
|
||||
# Prepare request payload
|
||||
payload = {
|
||||
"urls": [TEST_URLS[0]],
|
||||
"hooks": {
|
||||
"code": hooks_config,
|
||||
"timeout": 30
|
||||
},
|
||||
"crawler_config": {
|
||||
"cache_mode": "bypass"
|
||||
}
|
||||
}
|
||||
|
||||
print(f"🎯 Target URL: {TEST_URLS[0]}")
|
||||
print(f"🔧 Configured {len(hooks_config)} string-based hooks")
|
||||
print(f"📡 Sending request to Docker API...\n")
|
||||
|
||||
try:
|
||||
start_time = time.time()
|
||||
response = requests.post(f"{DOCKER_URL}/crawl", json=payload, timeout=60)
|
||||
execution_time = time.time() - start_time
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
|
||||
print(f"\n✅ Request successful! (took {execution_time:.2f}s)")
|
||||
|
||||
# Display results
|
||||
if result.get('results') and result['results'][0].get('success'):
|
||||
crawl_result = result['results'][0]
|
||||
html_length = len(crawl_result.get('html', ''))
|
||||
markdown_length = len(crawl_result.get('markdown', ''))
|
||||
|
||||
print(f"\n📊 Results:")
|
||||
print(f" • HTML length: {html_length:,} characters")
|
||||
print(f" • Markdown length: {markdown_length:,} characters")
|
||||
print(f" • URL: {crawl_result.get('url')}")
|
||||
|
||||
# Check hooks execution
|
||||
if 'hooks' in result:
|
||||
hooks_info = result['hooks']
|
||||
print(f"\n🎣 Hooks Execution:")
|
||||
print(f" • Status: {hooks_info['status']['status']}")
|
||||
print(f" • Attached hooks: {len(hooks_info['status']['attached_hooks'])}")
|
||||
|
||||
if 'summary' in hooks_info:
|
||||
summary = hooks_info['summary']
|
||||
print(f" • Total executions: {summary['total_executions']}")
|
||||
print(f" • Successful: {summary['successful']}")
|
||||
print(f" • Success rate: {summary['success_rate']:.1f}%")
|
||||
else:
|
||||
print(f"⚠️ Crawl completed but no results")
|
||||
|
||||
else:
|
||||
print(f"❌ Request failed with status {response.status_code}")
|
||||
print(f" Error: {response.text[:200]}")
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
print("⏰ Request timed out after 60 seconds")
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {str(e)}")
|
||||
|
||||
print("\n" + "─" * 70)
|
||||
print("✓ String-based hooks demo complete\n")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# DEMO 2: Function-Based Hooks with hooks_to_string() Utility
|
||||
# ============================================================================
|
||||
|
||||
def demo_2_hooks_to_string_utility():
|
||||
"""
|
||||
Demonstrate the new hooks_to_string() utility for converting functions
|
||||
"""
|
||||
print_section(
|
||||
"DEMO 2: hooks_to_string() Utility (NEW! ✨)",
|
||||
"Convert Python functions to strings for REST API"
|
||||
)
|
||||
|
||||
print("📦 Creating hook functions...")
|
||||
print(" • performance_optimization_hook")
|
||||
print(" • viewport_setup_hook")
|
||||
print(" • authentication_headers_hook")
|
||||
print(" • lazy_loading_handler_hook")
|
||||
|
||||
# Convert function objects to strings using the NEW utility
|
||||
print("\n🔄 Converting functions to strings with hooks_to_string()...")
|
||||
|
||||
hooks_dict = {
|
||||
"on_page_context_created": performance_optimization_hook,
|
||||
"before_goto": authentication_headers_hook,
|
||||
"before_retrieve_html": lazy_loading_handler_hook,
|
||||
}
|
||||
|
||||
hooks_as_strings = hooks_to_string(hooks_dict)
|
||||
|
||||
print(f"✅ Successfully converted {len(hooks_as_strings)} functions to strings")
|
||||
|
||||
# Show a preview
|
||||
print("\n📝 Sample converted hook (first 250 characters):")
|
||||
print("─" * 70)
|
||||
sample_hook = list(hooks_as_strings.values())[0]
|
||||
print(sample_hook[:250] + "...")
|
||||
print("─" * 70)
|
||||
|
||||
# Use the converted hooks with REST API
|
||||
print("\n📡 Using converted hooks with REST API...")
|
||||
|
||||
payload = {
|
||||
"urls": [TEST_URLS[0]],
|
||||
"hooks": {
|
||||
"code": hooks_as_strings,
|
||||
"timeout": 30
|
||||
}
|
||||
}
|
||||
|
||||
try:
|
||||
start_time = time.time()
|
||||
response = requests.post(f"{DOCKER_URL}/crawl", json=payload, timeout=60)
|
||||
execution_time = time.time() - start_time
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
print(f"\n✅ Request successful! (took {execution_time:.2f}s)")
|
||||
|
||||
if result.get('results') and result['results'][0].get('success'):
|
||||
crawl_result = result['results'][0]
|
||||
print(f" • HTML length: {len(crawl_result.get('html', '')):,} characters")
|
||||
print(f" • Hooks executed successfully!")
|
||||
else:
|
||||
print(f"❌ Request failed: {response.status_code}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {str(e)}")
|
||||
|
||||
print("\n💡 Benefits of hooks_to_string():")
|
||||
print(" ✓ Write hooks as regular Python functions")
|
||||
print(" ✓ Full IDE support (autocomplete, syntax highlighting)")
|
||||
print(" ✓ Type checking and linting")
|
||||
print(" ✓ Easy to test and debug")
|
||||
print(" ✓ Reusable across projects")
|
||||
print(" ✓ Works with any REST API client")
|
||||
|
||||
print("\n" + "─" * 70)
|
||||
print("✓ hooks_to_string() utility demo complete\n")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# DEMO 3: Docker Client with Automatic Conversion (RECOMMENDED! 🌟)
|
||||
# ============================================================================
|
||||
|
||||
async def demo_3_docker_client_auto_conversion():
|
||||
"""
|
||||
Demonstrate Docker Client with automatic hook conversion (RECOMMENDED)
|
||||
"""
|
||||
print_section(
|
||||
"DEMO 3: Docker Client with Auto-Conversion (RECOMMENDED! 🌟)",
|
||||
"Pass function objects directly - conversion happens automatically!"
|
||||
)
|
||||
|
||||
print("🐳 Initializing Crawl4AI Docker Client...")
|
||||
client = Crawl4aiDockerClient(base_url=DOCKER_URL)
|
||||
|
||||
print("✅ Client ready!\n")
|
||||
|
||||
# Use our reusable hook library - just pass the function objects!
|
||||
print("📚 Using reusable hook library:")
|
||||
print(" • performance_optimization_hook")
|
||||
print(" • viewport_setup_hook")
|
||||
print(" • authentication_headers_hook")
|
||||
print(" • lazy_loading_handler_hook")
|
||||
print(" • page_analytics_hook")
|
||||
|
||||
print("\n🎯 Target URL: " + TEST_URLS[1])
|
||||
print("🚀 Starting crawl with automatic hook conversion...\n")
|
||||
|
||||
try:
|
||||
start_time = time.time()
|
||||
|
||||
# Pass function objects directly - NO manual conversion needed! ✨
|
||||
results = await client.crawl(
|
||||
urls=[TEST_URLS[0]],
|
||||
hooks={
|
||||
"on_page_context_created": performance_optimization_hook,
|
||||
"before_goto": authentication_headers_hook,
|
||||
"before_retrieve_html": lazy_loading_handler_hook,
|
||||
"before_return_html": page_analytics_hook,
|
||||
},
|
||||
hooks_timeout=30
|
||||
)
|
||||
|
||||
execution_time = time.time() - start_time
|
||||
|
||||
print(f"\n✅ Crawl completed! (took {execution_time:.2f}s)\n")
|
||||
|
||||
# Display results
|
||||
if results and results.success:
|
||||
result = results
|
||||
print(f"📊 Results:")
|
||||
print(f" • URL: {result.url}")
|
||||
print(f" • Success: {result.success}")
|
||||
print(f" • HTML length: {len(result.html):,} characters")
|
||||
print(f" • Markdown length: {len(result.markdown):,} characters")
|
||||
|
||||
# Show metadata
|
||||
if result.metadata:
|
||||
print(f"\n📋 Metadata:")
|
||||
print(f" • Title: {result.metadata.get('title', 'N/A')}")
|
||||
print(f" • Description: {result.metadata.get('description', 'N/A')}")
|
||||
|
||||
# Show links
|
||||
if result.links:
|
||||
internal_count = len(result.links.get('internal', []))
|
||||
external_count = len(result.links.get('external', []))
|
||||
print(f"\n🔗 Links Found:")
|
||||
print(f" • Internal: {internal_count}")
|
||||
print(f" • External: {external_count}")
|
||||
else:
|
||||
print(f"⚠️ Crawl completed but no successful results")
|
||||
if results:
|
||||
print(f" Error: {results.error_message}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
print("\n🌟 Why Docker Client is RECOMMENDED:")
|
||||
print(" ✓ Automatic function-to-string conversion")
|
||||
print(" ✓ No manual hooks_to_string() calls needed")
|
||||
print(" ✓ Cleaner, more Pythonic code")
|
||||
print(" ✓ Full type hints and IDE support")
|
||||
print(" ✓ Built-in error handling")
|
||||
print(" ✓ Async/await support")
|
||||
|
||||
print("\n" + "─" * 70)
|
||||
print("✓ Docker Client auto-conversion demo complete\n")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# DEMO 4: Advanced Use Case - Complete Hook Pipeline
|
||||
# ============================================================================
|
||||
|
||||
async def demo_4_complete_hook_pipeline():
|
||||
"""
|
||||
Demonstrate a complete hook pipeline using all 8 hook points
|
||||
"""
|
||||
print_section(
|
||||
"DEMO 4: Complete Hook Pipeline",
|
||||
"Using all 8 available hook points for comprehensive control"
|
||||
)
|
||||
|
||||
# Define all 8 hooks
|
||||
async def on_browser_created_hook(browser, **kwargs):
|
||||
"""Hook 1: Called after browser is created"""
|
||||
print(" [Pipeline] 1/8 Browser created")
|
||||
return browser
|
||||
|
||||
async def on_page_context_created_hook(page, context, **kwargs):
|
||||
"""Hook 2: Called after page context is created"""
|
||||
print(" [Pipeline] 2/8 Page context created - setting up...")
|
||||
await page.set_viewport_size({"width": 1920, "height": 1080})
|
||||
return page
|
||||
|
||||
async def on_user_agent_updated_hook(page, context, user_agent, **kwargs):
|
||||
"""Hook 3: Called when user agent is updated"""
|
||||
print(f" [Pipeline] 3/8 User agent updated: {user_agent[:50]}...")
|
||||
return page
|
||||
|
||||
async def before_goto_hook(page, context, url, **kwargs):
|
||||
"""Hook 4: Called before navigating to URL"""
|
||||
print(f" [Pipeline] 4/8 Before navigation to: {url[:60]}...")
|
||||
return page
|
||||
|
||||
async def after_goto_hook(page, context, url, response, **kwargs):
|
||||
"""Hook 5: Called after navigation completes"""
|
||||
print(f" [Pipeline] 5/8 After navigation - Status: {response.status if response else 'N/A'}")
|
||||
await page.wait_for_timeout(1000)
|
||||
return page
|
||||
|
||||
async def on_execution_started_hook(page, context, **kwargs):
|
||||
"""Hook 6: Called when JavaScript execution starts"""
|
||||
print(" [Pipeline] 6/8 JavaScript execution started")
|
||||
return page
|
||||
|
||||
async def before_retrieve_html_hook(page, context, **kwargs):
|
||||
"""Hook 7: Called before retrieving HTML"""
|
||||
print(" [Pipeline] 7/8 Before HTML retrieval - scrolling...")
|
||||
await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
|
||||
return page
|
||||
|
||||
async def before_return_html_hook(page, context, html, **kwargs):
|
||||
"""Hook 8: Called before returning HTML"""
|
||||
print(f" [Pipeline] 8/8 Before return - HTML length: {len(html):,} chars")
|
||||
return page
|
||||
|
||||
print("🎯 Target URL: " + TEST_URLS[0])
|
||||
print("🔧 Configured ALL 8 hook points for complete pipeline control\n")
|
||||
|
||||
client = Crawl4aiDockerClient(base_url=DOCKER_URL)
|
||||
|
||||
try:
|
||||
print("🚀 Starting complete pipeline crawl...\n")
|
||||
start_time = time.time()
|
||||
|
||||
results = await client.crawl(
|
||||
urls=[TEST_URLS[0]],
|
||||
hooks={
|
||||
"on_browser_created": on_browser_created_hook,
|
||||
"on_page_context_created": on_page_context_created_hook,
|
||||
"on_user_agent_updated": on_user_agent_updated_hook,
|
||||
"before_goto": before_goto_hook,
|
||||
"after_goto": after_goto_hook,
|
||||
"on_execution_started": on_execution_started_hook,
|
||||
"before_retrieve_html": before_retrieve_html_hook,
|
||||
"before_return_html": before_return_html_hook,
|
||||
},
|
||||
hooks_timeout=45
|
||||
)
|
||||
|
||||
execution_time = time.time() - start_time
|
||||
|
||||
if results and results.success:
|
||||
print(f"\n✅ Complete pipeline executed successfully! (took {execution_time:.2f}s)")
|
||||
print(f" • All 8 hooks executed in sequence")
|
||||
print(f" • HTML length: {len(results.html):,} characters")
|
||||
else:
|
||||
print(f"⚠️ Pipeline completed with warnings")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {str(e)}")
|
||||
|
||||
print("\n📚 Available Hook Points:")
|
||||
print(" 1. on_browser_created - Browser initialization")
|
||||
print(" 2. on_page_context_created - Page context setup")
|
||||
print(" 3. on_user_agent_updated - User agent configuration")
|
||||
print(" 4. before_goto - Pre-navigation setup")
|
||||
print(" 5. after_goto - Post-navigation processing")
|
||||
print(" 6. on_execution_started - JavaScript execution start")
|
||||
print(" 7. before_retrieve_html - Pre-extraction processing")
|
||||
print(" 8. before_return_html - Final HTML processing")
|
||||
|
||||
print("\n" + "─" * 70)
|
||||
print("✓ Complete hook pipeline demo complete\n")
|
||||
|
||||
|
||||
# ============================================================================
|
||||
# MAIN EXECUTION
|
||||
# ============================================================================
|
||||
|
||||
async def main():
|
||||
"""
|
||||
Run all demonstrations
|
||||
"""
|
||||
print("\n" + "=" * 70)
|
||||
print(" 🚀 Crawl4AI v0.7.5 - Docker Hooks Complete Demonstration")
|
||||
print("=" * 70)
|
||||
|
||||
# Check Docker service
|
||||
print("\n🔍 Checking Docker service status...")
|
||||
if not check_docker_service():
|
||||
print("❌ Docker service is not running!")
|
||||
print("\n📋 To start the Docker service:")
|
||||
print(" docker run -p 11235:11235 unclecode/crawl4ai:latest")
|
||||
print("\nPlease start the service and run this demo again.")
|
||||
return
|
||||
|
||||
print("✅ Docker service is running!\n")
|
||||
|
||||
# Run all demos
|
||||
demos = [
|
||||
("String-Based Hooks (REST API)", demo_1_string_based_hooks, False),
|
||||
("hooks_to_string() Utility", demo_2_hooks_to_string_utility, False),
|
||||
("Docker Client Auto-Conversion", demo_3_docker_client_auto_conversion, True),
|
||||
# ("Complete Hook Pipeline", demo_4_complete_hook_pipeline, True),
|
||||
]
|
||||
|
||||
for i, (name, demo_func, is_async) in enumerate(demos, 1):
|
||||
print(f"\n{'🔷' * 35}")
|
||||
print(f"Starting Demo {i}/{len(demos)}: {name}")
|
||||
print(f"{'🔷' * 35}\n")
|
||||
|
||||
try:
|
||||
if is_async:
|
||||
await demo_func()
|
||||
else:
|
||||
demo_func()
|
||||
|
||||
print(f"✅ Demo {i} completed successfully!")
|
||||
|
||||
# Pause between demos (except the last one)
|
||||
if i < len(demos):
|
||||
print("\n⏸️ Press Enter to continue to next demo...")
|
||||
# input()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
print(f"\n⏹️ Demo interrupted by user")
|
||||
break
|
||||
except Exception as e:
|
||||
print(f"\n❌ Demo {i} failed: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
print("\nContinuing to next demo...\n")
|
||||
continue
|
||||
|
||||
# Final summary
|
||||
print("\n" + "=" * 70)
|
||||
print(" 🎉 All Demonstrations Complete!")
|
||||
print("=" * 70)
|
||||
|
||||
print("\n📊 Summary of v0.7.5 Docker Hooks System:")
|
||||
print("\n🆕 COMPLETELY NEW FEATURE in v0.7.5:")
|
||||
print(" The Docker Hooks System lets you customize the crawling pipeline")
|
||||
print(" with user-provided Python functions at 8 strategic points.")
|
||||
|
||||
print("\n✨ Three Ways to Use Docker Hooks (All NEW!):")
|
||||
print(" 1. String-based - Write hooks as strings for REST API")
|
||||
print(" 2. hooks_to_string() - Convert Python functions to strings")
|
||||
print(" 3. Docker Client - Automatic conversion (RECOMMENDED)")
|
||||
|
||||
print("\n💡 Key Benefits:")
|
||||
print(" ✓ Full IDE support (autocomplete, syntax highlighting)")
|
||||
print(" ✓ Type checking and linting")
|
||||
print(" ✓ Easy to test and debug")
|
||||
print(" ✓ Reusable across projects")
|
||||
print(" ✓ Complete pipeline control")
|
||||
|
||||
print("\n🎯 8 Hook Points Available:")
|
||||
print(" • on_browser_created, on_page_context_created")
|
||||
print(" • on_user_agent_updated, before_goto, after_goto")
|
||||
print(" • on_execution_started, before_retrieve_html, before_return_html")
|
||||
|
||||
print("\n📚 Resources:")
|
||||
print(" • Docs: https://docs.crawl4ai.com")
|
||||
print(" • GitHub: https://github.com/unclecode/crawl4ai")
|
||||
print(" • Discord: https://discord.gg/jP8KfhDhyN")
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print(" Happy Crawling with v0.7.5! 🕷️")
|
||||
print("=" * 70 + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("\n🎬 Starting Crawl4AI v0.7.5 Docker Hooks Demonstration...")
|
||||
print("Press Ctrl+C anytime to exit\n")
|
||||
|
||||
try:
|
||||
asyncio.run(main())
|
||||
except KeyboardInterrupt:
|
||||
print("\n\n👋 Demo stopped by user. Thanks for exploring Crawl4AI v0.7.5!")
|
||||
except Exception as e:
|
||||
print(f"\n\n❌ Demo error: {str(e)}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,464 @@
|
||||
"""
|
||||
Crawl4AI v0.4.24 Feature Walkthrough
|
||||
===================================
|
||||
|
||||
This script demonstrates the new features introduced in Crawl4AI v0.4.24.
|
||||
Each section includes detailed examples and explanations of the new capabilities.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
from typing import List
|
||||
from crawl4ai import (
|
||||
AsyncWebCrawler,
|
||||
BrowserConfig,
|
||||
CrawlerRunConfig,
|
||||
CacheMode,
|
||||
LLMExtractionStrategy,
|
||||
JsonCssExtractionStrategy,
|
||||
)
|
||||
from crawl4ai.content_filter_strategy import RelevantContentFilter
|
||||
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
# Sample HTML for demonstrations
|
||||
SAMPLE_HTML = """
|
||||
<div class="article-list">
|
||||
<article class="post" data-category="tech" data-author="john">
|
||||
<h2 class="title"><a href="/post-1">First Post</a></h2>
|
||||
<div class="meta">
|
||||
<a href="/author/john" class="author">John Doe</a>
|
||||
<span class="date">2023-12-31</span>
|
||||
</div>
|
||||
<div class="content">
|
||||
<p>First post content...</p>
|
||||
<a href="/read-more-1" class="read-more">Read More</a>
|
||||
</div>
|
||||
</article>
|
||||
<article class="post" data-category="science" data-author="jane">
|
||||
<h2 class="title"><a href="/post-2">Second Post</a></h2>
|
||||
<div class="meta">
|
||||
<a href="/author/jane" class="author">Jane Smith</a>
|
||||
<span class="date">2023-12-30</span>
|
||||
</div>
|
||||
<div class="content">
|
||||
<p>Second post content...</p>
|
||||
<a href="/read-more-2" class="read-more">Read More</a>
|
||||
</div>
|
||||
</article>
|
||||
</div>
|
||||
"""
|
||||
|
||||
|
||||
async def demo_ssl_features():
|
||||
"""
|
||||
Enhanced SSL & Security Features Demo
|
||||
-----------------------------------
|
||||
|
||||
This example demonstrates the new SSL certificate handling and security features:
|
||||
1. Custom certificate paths
|
||||
2. SSL verification options
|
||||
3. HTTPS error handling
|
||||
4. Certificate validation configurations
|
||||
|
||||
These features are particularly useful when:
|
||||
- Working with self-signed certificates
|
||||
- Dealing with corporate proxies
|
||||
- Handling mixed content websites
|
||||
- Managing different SSL security levels
|
||||
"""
|
||||
print("\n1. Enhanced SSL & Security Demo")
|
||||
print("--------------------------------")
|
||||
|
||||
browser_config = BrowserConfig()
|
||||
|
||||
run_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
fetch_ssl_certificate=True, # Enable SSL certificate fetching
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
result = await crawler.arun(url="https://example.com", config=run_config)
|
||||
print(f"SSL Crawl Success: {result.success}")
|
||||
result.ssl_certificate.to_json(
|
||||
os.path.join(os.getcwd(), "ssl_certificate.json")
|
||||
)
|
||||
if not result.success:
|
||||
print(f"SSL Error: {result.error_message}")
|
||||
|
||||
|
||||
async def demo_content_filtering():
|
||||
"""
|
||||
Smart Content Filtering Demo
|
||||
----------------------
|
||||
|
||||
Demonstrates advanced content filtering capabilities:
|
||||
1. Custom filter to identify and extract specific content
|
||||
2. Integration with markdown generation
|
||||
3. Flexible pruning rules
|
||||
"""
|
||||
print("\n2. Smart Content Filtering Demo")
|
||||
print("--------------------------------")
|
||||
|
||||
# Create a custom content filter
|
||||
class CustomNewsFilter(RelevantContentFilter):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
# Add news-specific patterns
|
||||
self.negative_patterns = re.compile(
|
||||
r"nav|footer|header|sidebar|ads|comment|share|related|recommended|popular|trending",
|
||||
re.I,
|
||||
)
|
||||
self.min_word_count = 30 # Higher threshold for news content
|
||||
|
||||
def filter_content(
|
||||
self, html: str, min_word_threshold: int = None
|
||||
) -> List[str]:
|
||||
"""
|
||||
Implements news-specific content filtering logic.
|
||||
|
||||
Args:
|
||||
html (str): HTML content to be filtered
|
||||
min_word_threshold (int, optional): Minimum word count threshold
|
||||
|
||||
Returns:
|
||||
List[str]: List of filtered HTML content blocks
|
||||
"""
|
||||
if not html or not isinstance(html, str):
|
||||
return []
|
||||
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
if not soup.body:
|
||||
soup = BeautifulSoup(f"<body>{html}</body>", "lxml")
|
||||
|
||||
body = soup.find("body")
|
||||
|
||||
# Extract chunks with metadata
|
||||
chunks = self.extract_text_chunks(
|
||||
body, min_word_threshold or self.min_word_count
|
||||
)
|
||||
|
||||
# Filter chunks based on news-specific criteria
|
||||
filtered_chunks = []
|
||||
for _, text, tag_type, element in chunks:
|
||||
# Skip if element has negative class/id
|
||||
if self.is_excluded(element):
|
||||
continue
|
||||
|
||||
# Headers are important in news articles
|
||||
if tag_type == "header":
|
||||
filtered_chunks.append(self.clean_element(element))
|
||||
continue
|
||||
|
||||
# For content, check word count and link density
|
||||
text = element.get_text(strip=True)
|
||||
if len(text.split()) >= (min_word_threshold or self.min_word_count):
|
||||
# Calculate link density
|
||||
links_text = " ".join(
|
||||
a.get_text(strip=True) for a in element.find_all("a")
|
||||
)
|
||||
link_density = len(links_text) / len(text) if text else 1
|
||||
|
||||
# Accept if link density is reasonable
|
||||
if link_density < 0.5:
|
||||
filtered_chunks.append(self.clean_element(element))
|
||||
|
||||
return filtered_chunks
|
||||
|
||||
# Create markdown generator with custom filter
|
||||
markdown_gen = DefaultMarkdownGenerator(content_filter=CustomNewsFilter())
|
||||
|
||||
run_config = CrawlerRunConfig(
|
||||
markdown_generator=markdown_gen, cache_mode=CacheMode.BYPASS
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://news.ycombinator.com", config=run_config
|
||||
)
|
||||
print("Filtered Content Sample:")
|
||||
print(result.markdown[:500]) # Show first 500 chars
|
||||
|
||||
|
||||
async def demo_json_extraction():
|
||||
"""
|
||||
Improved JSON Extraction Demo
|
||||
---------------------------
|
||||
|
||||
Demonstrates the enhanced JSON extraction capabilities:
|
||||
1. Base element attributes extraction
|
||||
2. Complex nested structures
|
||||
3. Multiple extraction patterns
|
||||
|
||||
Key features shown:
|
||||
- Extracting attributes from base elements (href, data-* attributes)
|
||||
- Processing repeated patterns
|
||||
- Handling optional fields
|
||||
"""
|
||||
print("\n3. Improved JSON Extraction Demo")
|
||||
print("--------------------------------")
|
||||
|
||||
# Define the extraction schema with base element attributes
|
||||
json_strategy = JsonCssExtractionStrategy(
|
||||
schema={
|
||||
"name": "Blog Posts",
|
||||
"baseSelector": "div.article-list",
|
||||
"baseFields": [
|
||||
{"name": "list_id", "type": "attribute", "attribute": "data-list-id"},
|
||||
{"name": "category", "type": "attribute", "attribute": "data-category"},
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "posts",
|
||||
"selector": "article.post",
|
||||
"type": "nested_list",
|
||||
"baseFields": [
|
||||
{
|
||||
"name": "post_id",
|
||||
"type": "attribute",
|
||||
"attribute": "data-post-id",
|
||||
},
|
||||
{
|
||||
"name": "author_id",
|
||||
"type": "attribute",
|
||||
"attribute": "data-author",
|
||||
},
|
||||
],
|
||||
"fields": [
|
||||
{
|
||||
"name": "title",
|
||||
"selector": "h2.title a",
|
||||
"type": "text",
|
||||
"baseFields": [
|
||||
{
|
||||
"name": "url",
|
||||
"type": "attribute",
|
||||
"attribute": "href",
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"name": "author",
|
||||
"selector": "div.meta a.author",
|
||||
"type": "text",
|
||||
"baseFields": [
|
||||
{
|
||||
"name": "profile_url",
|
||||
"type": "attribute",
|
||||
"attribute": "href",
|
||||
}
|
||||
],
|
||||
},
|
||||
{"name": "date", "selector": "span.date", "type": "text"},
|
||||
{
|
||||
"name": "read_more",
|
||||
"selector": "a.read-more",
|
||||
"type": "nested",
|
||||
"fields": [
|
||||
{"name": "text", "type": "text"},
|
||||
{
|
||||
"name": "url",
|
||||
"type": "attribute",
|
||||
"attribute": "href",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
# Demonstrate extraction from raw HTML
|
||||
run_config = CrawlerRunConfig(
|
||||
extraction_strategy=json_strategy, cache_mode=CacheMode.BYPASS
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun(
|
||||
url="raw:" + SAMPLE_HTML, # Use raw: prefix for raw HTML
|
||||
config=run_config,
|
||||
)
|
||||
print("Extracted Content:")
|
||||
print(result.extracted_content)
|
||||
|
||||
|
||||
async def demo_input_formats():
|
||||
"""
|
||||
Input Format Handling Demo
|
||||
----------------------
|
||||
|
||||
Demonstrates how LLM extraction can work with different input formats:
|
||||
1. Markdown (default) - Good for simple text extraction
|
||||
2. HTML - Better when you need structure and attributes
|
||||
|
||||
This example shows how HTML input can be beneficial when:
|
||||
- You need to understand the DOM structure
|
||||
- You want to extract both visible text and HTML attributes
|
||||
- The content has complex layouts like tables or forms
|
||||
"""
|
||||
print("\n4. Input Format Handling Demo")
|
||||
print("---------------------------")
|
||||
|
||||
# Create a dummy HTML with rich structure
|
||||
dummy_html = """
|
||||
<div class="job-posting" data-post-id="12345">
|
||||
<header class="job-header">
|
||||
<h1 class="job-title">Senior AI/ML Engineer</h1>
|
||||
<div class="job-meta">
|
||||
<span class="department">AI Research Division</span>
|
||||
<span class="location" data-remote="hybrid">San Francisco (Hybrid)</span>
|
||||
</div>
|
||||
<div class="salary-info" data-currency="USD">
|
||||
<span class="range">$150,000 - $220,000</span>
|
||||
<span class="period">per year</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<section class="requirements">
|
||||
<div class="technical-skills">
|
||||
<h3>Technical Requirements</h3>
|
||||
<ul class="required-skills">
|
||||
<li class="skill required" data-priority="must-have">
|
||||
5+ years experience in Machine Learning
|
||||
</li>
|
||||
<li class="skill required" data-priority="must-have">
|
||||
Proficiency in Python and PyTorch/TensorFlow
|
||||
</li>
|
||||
<li class="skill preferred" data-priority="nice-to-have">
|
||||
Experience with distributed training systems
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="soft-skills">
|
||||
<h3>Professional Skills</h3>
|
||||
<ul class="required-skills">
|
||||
<li class="skill required" data-priority="must-have">
|
||||
Strong problem-solving abilities
|
||||
</li>
|
||||
<li class="skill preferred" data-priority="nice-to-have">
|
||||
Experience leading technical teams
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="timeline">
|
||||
<time class="deadline" datetime="2024-02-28">
|
||||
Application Deadline: February 28, 2024
|
||||
</time>
|
||||
</section>
|
||||
|
||||
<footer class="contact-section">
|
||||
<div class="hiring-manager">
|
||||
<h4>Hiring Manager</h4>
|
||||
<div class="contact-info">
|
||||
<span class="name">Dr. Sarah Chen</span>
|
||||
<span class="title">Director of AI Research</span>
|
||||
<span class="email">ai.hiring@example.com</span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="team-info">
|
||||
<p>Join our team of 50+ researchers working on cutting-edge AI applications</p>
|
||||
</div>
|
||||
</footer>
|
||||
</div>
|
||||
"""
|
||||
|
||||
# Use raw:// prefix to pass HTML content directly
|
||||
url = f"raw://{dummy_html}"
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import List, Optional
|
||||
|
||||
# Define our schema using Pydantic
|
||||
class JobRequirement(BaseModel):
|
||||
category: str = Field(
|
||||
description="Category of the requirement (e.g., Technical, Soft Skills)"
|
||||
)
|
||||
items: List[str] = Field(
|
||||
description="List of specific requirements in this category"
|
||||
)
|
||||
priority: str = Field(
|
||||
description="Priority level (Required/Preferred) based on the HTML class or context"
|
||||
)
|
||||
|
||||
class JobPosting(BaseModel):
|
||||
title: str = Field(description="Job title")
|
||||
department: str = Field(description="Department or team")
|
||||
location: str = Field(description="Job location, including remote options")
|
||||
salary_range: Optional[str] = Field(description="Salary range if specified")
|
||||
requirements: List[JobRequirement] = Field(
|
||||
description="Categorized job requirements"
|
||||
)
|
||||
application_deadline: Optional[str] = Field(
|
||||
description="Application deadline if specified"
|
||||
)
|
||||
contact_info: Optional[dict] = Field(
|
||||
description="Contact information from footer or contact section"
|
||||
)
|
||||
|
||||
# First try with markdown (default)
|
||||
markdown_strategy = LLMExtractionStrategy(
|
||||
provider="openai/gpt-4o",
|
||||
api_token=os.getenv("OPENAI_API_KEY"),
|
||||
schema=JobPosting.model_json_schema(),
|
||||
extraction_type="schema",
|
||||
instruction="""
|
||||
Extract job posting details into structured data. Focus on the visible text content
|
||||
and organize requirements into categories.
|
||||
""",
|
||||
input_format="markdown", # default
|
||||
)
|
||||
|
||||
# Then with HTML for better structure understanding
|
||||
html_strategy = LLMExtractionStrategy(
|
||||
provider="openai/gpt-4",
|
||||
api_token=os.getenv("OPENAI_API_KEY"),
|
||||
schema=JobPosting.model_json_schema(),
|
||||
extraction_type="schema",
|
||||
instruction="""
|
||||
Extract job posting details, using HTML structure to:
|
||||
1. Identify requirement priorities from CSS classes (e.g., 'required' vs 'preferred')
|
||||
2. Extract contact info from the page footer or dedicated contact section
|
||||
3. Parse salary information from specially formatted elements
|
||||
4. Determine application deadline from timestamp or date elements
|
||||
|
||||
Use HTML attributes and classes to enhance extraction accuracy.
|
||||
""",
|
||||
input_format="html", # explicitly use HTML
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
# Try with markdown first
|
||||
markdown_config = CrawlerRunConfig(extraction_strategy=markdown_strategy)
|
||||
markdown_result = await crawler.arun(url=url, config=markdown_config)
|
||||
print("\nMarkdown-based Extraction Result:")
|
||||
items = json.loads(markdown_result.extracted_content)
|
||||
print(json.dumps(items, indent=2))
|
||||
|
||||
# Then with HTML for better structure understanding
|
||||
html_config = CrawlerRunConfig(extraction_strategy=html_strategy)
|
||||
html_result = await crawler.arun(url=url, config=html_config)
|
||||
print("\nHTML-based Extraction Result:")
|
||||
items = json.loads(html_result.extracted_content)
|
||||
print(json.dumps(items, indent=2))
|
||||
|
||||
|
||||
# Main execution
|
||||
async def main():
|
||||
print("Crawl4AI v0.4.24 Feature Walkthrough")
|
||||
print("====================================")
|
||||
|
||||
# Run all demos
|
||||
await demo_ssl_features()
|
||||
await demo_content_filtering()
|
||||
await demo_json_extraction()
|
||||
# await demo_input_formats()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,349 @@
|
||||
"""
|
||||
Crawl4ai v0.4.3b2 Features Demo
|
||||
============================
|
||||
|
||||
This demonstration showcases three major categories of new features in Crawl4ai v0.4.3:
|
||||
|
||||
1. Efficiency & Speed:
|
||||
- Memory-efficient dispatcher strategies
|
||||
- New scraping algorithm
|
||||
- Streaming support for batch crawling
|
||||
|
||||
2. LLM Integration:
|
||||
- Automatic schema generation
|
||||
- LLM-powered content filtering
|
||||
- Smart markdown generation
|
||||
|
||||
3. Core Improvements:
|
||||
- Robots.txt compliance
|
||||
- Proxy rotation
|
||||
- Enhanced URL handling
|
||||
- Shared data among hooks
|
||||
- add page routes
|
||||
|
||||
Each demo function can be run independently or as part of the full suite.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
import random
|
||||
from typing import Optional, Dict
|
||||
from dotenv import load_dotenv
|
||||
from crawl4ai import (
|
||||
AsyncWebCrawler,
|
||||
BrowserConfig,
|
||||
CrawlerRunConfig,
|
||||
CacheMode,
|
||||
DisplayMode,
|
||||
MemoryAdaptiveDispatcher,
|
||||
CrawlerMonitor,
|
||||
DefaultMarkdownGenerator,
|
||||
LXMLWebScrapingStrategy,
|
||||
JsonCssExtractionStrategy,
|
||||
LLMContentFilter
|
||||
)
|
||||
|
||||
load_dotenv()
|
||||
|
||||
async def demo_memory_dispatcher():
|
||||
"""Demonstrates the new memory-efficient dispatcher system.
|
||||
|
||||
Key Features:
|
||||
- Adaptive memory management
|
||||
- Real-time performance monitoring
|
||||
- Concurrent session control
|
||||
"""
|
||||
print("\n=== Memory Dispatcher Demo ===")
|
||||
|
||||
try:
|
||||
# Configuration
|
||||
browser_config = BrowserConfig(headless=True, verbose=False)
|
||||
crawler_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
markdown_generator=DefaultMarkdownGenerator()
|
||||
)
|
||||
|
||||
# Test URLs
|
||||
urls = ["http://example.com", "http://example.org", "http://example.net"] * 3
|
||||
|
||||
print("\n📈 Initializing crawler with memory monitoring...")
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
monitor = CrawlerMonitor(
|
||||
max_visible_rows=10,
|
||||
display_mode=DisplayMode.DETAILED
|
||||
)
|
||||
|
||||
dispatcher = MemoryAdaptiveDispatcher(
|
||||
memory_threshold_percent=80.0,
|
||||
check_interval=0.5,
|
||||
max_session_permit=5,
|
||||
monitor=monitor
|
||||
)
|
||||
|
||||
print("\n🚀 Starting batch crawl...")
|
||||
results = await crawler.arun_many(
|
||||
urls=urls,
|
||||
config=crawler_config,
|
||||
dispatcher=dispatcher
|
||||
)
|
||||
print(f"\n✅ Completed {len(results)} URLs successfully")
|
||||
|
||||
except Exception as e:
|
||||
print(f"\n❌ Error in memory dispatcher demo: {str(e)}")
|
||||
|
||||
async def demo_streaming_support():
|
||||
"""
|
||||
2. Streaming Support Demo
|
||||
======================
|
||||
Shows how to process URLs as they complete using streaming
|
||||
"""
|
||||
print("\n=== 2. Streaming Support Demo ===")
|
||||
|
||||
browser_config = BrowserConfig(headless=True, verbose=False)
|
||||
crawler_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS, stream=True)
|
||||
|
||||
# Test URLs
|
||||
urls = ["http://example.com", "http://example.org", "http://example.net"] * 2
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
# Initialize dispatcher for streaming
|
||||
dispatcher = MemoryAdaptiveDispatcher(max_session_permit=3, check_interval=0.5)
|
||||
|
||||
print("Starting streaming crawl...")
|
||||
async for result in await crawler.arun_many(
|
||||
urls=urls,
|
||||
config=crawler_config,
|
||||
dispatcher=dispatcher
|
||||
):
|
||||
# Process each result as it arrives
|
||||
print(
|
||||
f"Received result for {result.url} - Success: {result.success}"
|
||||
)
|
||||
if result.success:
|
||||
print(f"Content length: {len(result.markdown)}")
|
||||
|
||||
async def demo_content_scraping():
|
||||
"""
|
||||
3. Content Scraping Strategy Demo
|
||||
==============================
|
||||
Demonstrates the new LXMLWebScrapingStrategy for faster content scraping.
|
||||
"""
|
||||
print("\n=== 3. Content Scraping Strategy Demo ===")
|
||||
|
||||
crawler = AsyncWebCrawler()
|
||||
url = "https://example.com/article"
|
||||
|
||||
# Configure with the new LXML strategy
|
||||
config = CrawlerRunConfig(
|
||||
scraping_strategy=LXMLWebScrapingStrategy(),
|
||||
verbose=True
|
||||
)
|
||||
|
||||
print("Scraping content with LXML strategy...")
|
||||
async with crawler:
|
||||
result = await crawler.arun(url, config=config)
|
||||
if result.success:
|
||||
print("Successfully scraped content using LXML strategy")
|
||||
|
||||
async def demo_llm_markdown():
|
||||
"""
|
||||
4. LLM-Powered Markdown Generation Demo
|
||||
===================================
|
||||
Shows how to use the new LLM-powered content filtering and markdown generation.
|
||||
"""
|
||||
print("\n=== 4. LLM-Powered Markdown Generation Demo ===")
|
||||
|
||||
crawler = AsyncWebCrawler()
|
||||
url = "https://docs.python.org/3/tutorial/classes.html"
|
||||
|
||||
content_filter = LLMContentFilter(
|
||||
provider="openai/gpt-4o",
|
||||
api_token=os.getenv("OPENAI_API_KEY"),
|
||||
instruction="""
|
||||
Focus on extracting the core educational content about Python classes.
|
||||
Include:
|
||||
- Key concepts and their explanations
|
||||
- Important code examples
|
||||
- Essential technical details
|
||||
Exclude:
|
||||
- Navigation elements
|
||||
- Sidebars
|
||||
- Footer content
|
||||
- Version information
|
||||
- Any non-essential UI elements
|
||||
|
||||
Format the output as clean markdown with proper code blocks and headers.
|
||||
""",
|
||||
verbose=True,
|
||||
)
|
||||
|
||||
# Configure LLM-powered markdown generation
|
||||
config = CrawlerRunConfig(
|
||||
markdown_generator=DefaultMarkdownGenerator(
|
||||
content_filter=content_filter
|
||||
),
|
||||
cache_mode = CacheMode.BYPASS,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
print("Generating focused markdown with LLM...")
|
||||
async with crawler:
|
||||
result = await crawler.arun(url, config=config)
|
||||
if result.success and result.markdown_v2:
|
||||
print("Successfully generated LLM-filtered markdown")
|
||||
print("First 500 chars of filtered content:")
|
||||
print(result.markdown_v2.fit_markdown[:500])
|
||||
print("Successfully generated LLM-filtered markdown")
|
||||
|
||||
async def demo_robots_compliance():
|
||||
"""
|
||||
5. Robots.txt Compliance Demo
|
||||
==========================
|
||||
Demonstrates the new robots.txt compliance feature with SQLite caching.
|
||||
"""
|
||||
print("\n=== 5. Robots.txt Compliance Demo ===")
|
||||
|
||||
crawler = AsyncWebCrawler()
|
||||
urls = ["https://example.com", "https://facebook.com", "https://twitter.com"]
|
||||
|
||||
# Enable robots.txt checking
|
||||
config = CrawlerRunConfig(check_robots_txt=True, verbose=True)
|
||||
|
||||
print("Crawling with robots.txt compliance...")
|
||||
async with crawler:
|
||||
results = await crawler.arun_many(urls, config=config)
|
||||
for result in results:
|
||||
if result.status_code == 403:
|
||||
print(f"Access blocked by robots.txt: {result.url}")
|
||||
elif result.success:
|
||||
print(f"Successfully crawled: {result.url}")
|
||||
|
||||
async def demo_json_schema_generation():
|
||||
"""
|
||||
7. LLM-Powered Schema Generation Demo
|
||||
=================================
|
||||
Demonstrates automatic CSS and XPath schema generation using LLM models.
|
||||
"""
|
||||
print("\n=== 7. LLM-Powered Schema Generation Demo ===")
|
||||
|
||||
# Example HTML content for a job listing
|
||||
html_content = """
|
||||
<div class="job-listing">
|
||||
<h1 class="job-title">Senior Software Engineer</h1>
|
||||
<div class="job-details">
|
||||
<span class="location">San Francisco, CA</span>
|
||||
<span class="salary">$150,000 - $200,000</span>
|
||||
<div class="requirements">
|
||||
<h2>Requirements</h2>
|
||||
<ul>
|
||||
<li>5+ years Python experience</li>
|
||||
<li>Strong background in web crawling</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
print("Generating CSS selectors schema...")
|
||||
# Generate CSS selectors with a specific query
|
||||
css_schema = JsonCssExtractionStrategy.generate_schema(
|
||||
html_content,
|
||||
schema_type="CSS",
|
||||
query="Extract job title, location, and salary information",
|
||||
provider="openai/gpt-4o", # or use other providers like "ollama"
|
||||
)
|
||||
print("\nGenerated CSS Schema:")
|
||||
print(css_schema)
|
||||
|
||||
# Example of using the generated schema with crawler
|
||||
crawler = AsyncWebCrawler()
|
||||
url = "https://example.com/job-listing"
|
||||
|
||||
# Create an extraction strategy with the generated schema
|
||||
extraction_strategy = JsonCssExtractionStrategy(schema=css_schema)
|
||||
|
||||
config = CrawlerRunConfig(extraction_strategy=extraction_strategy, verbose=True)
|
||||
|
||||
print("\nTesting generated schema with crawler...")
|
||||
async with crawler:
|
||||
result = await crawler.arun(url, config=config)
|
||||
if result.success:
|
||||
print(json.dumps(result.extracted_content, indent=2) if result.extracted_content else None)
|
||||
print("Successfully used generated schema for crawling")
|
||||
|
||||
async def demo_proxy_rotation():
|
||||
"""
|
||||
8. Proxy Rotation Demo
|
||||
===================
|
||||
Demonstrates how to rotate proxies for each request using Crawl4ai.
|
||||
"""
|
||||
print("\n=== 8. Proxy Rotation Demo ===")
|
||||
|
||||
async def get_next_proxy(proxy_file: str = "proxies.txt") -> Optional[Dict]:
|
||||
"""Get next proxy from local file"""
|
||||
try:
|
||||
proxies = os.getenv("PROXIES", "").split(",")
|
||||
|
||||
ip, port, username, password = random.choice(proxies).split(":")
|
||||
return {
|
||||
"server": f"http://{ip}:{port}",
|
||||
"username": username,
|
||||
"password": password,
|
||||
"ip": ip # Store original IP for verification
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"Error loading proxy: {e}")
|
||||
return None
|
||||
|
||||
# Create 10 test requests to httpbin
|
||||
urls = ["https://httpbin.org/ip"] * 2
|
||||
|
||||
browser_config = BrowserConfig(headless=True, verbose=False)
|
||||
run_config = CrawlerRunConfig(cache_mode=CacheMode.BYPASS)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_config) as crawler:
|
||||
for url in urls:
|
||||
proxy = await get_next_proxy()
|
||||
if not proxy:
|
||||
print("No proxy available, skipping...")
|
||||
continue
|
||||
|
||||
# Create new config with proxy
|
||||
current_config = run_config.clone(proxy_config=proxy, user_agent="")
|
||||
result = await crawler.arun(url=url, config=current_config)
|
||||
|
||||
if result.success:
|
||||
ip_match = re.search(r'(?:[0-9]{1,3}\.){3}[0-9]{1,3}', result.html)
|
||||
print(f"Proxy {proxy['ip']} -> Response IP: {ip_match.group(0) if ip_match else 'Not found'}")
|
||||
verified = ip_match.group(0) == proxy['ip']
|
||||
if verified:
|
||||
print(f"✅ Proxy working! IP matches: {proxy['ip']}")
|
||||
else:
|
||||
print("❌ Proxy failed or IP mismatch!")
|
||||
else:
|
||||
print(f"Failed with proxy {proxy['ip']}")
|
||||
|
||||
async def main():
|
||||
"""Run all feature demonstrations."""
|
||||
print("\n📊 Running Crawl4ai v0.4.3 Feature Demos\n")
|
||||
|
||||
# Efficiency & Speed Demos
|
||||
print("\n🚀 EFFICIENCY & SPEED DEMOS")
|
||||
await demo_memory_dispatcher()
|
||||
await demo_streaming_support()
|
||||
await demo_content_scraping()
|
||||
|
||||
# # LLM Integration Demos
|
||||
print("\n🤖 LLM INTEGRATION DEMOS")
|
||||
await demo_json_schema_generation()
|
||||
await demo_llm_markdown()
|
||||
|
||||
# # Core Improvements
|
||||
print("\n🔧 CORE IMPROVEMENT DEMOS")
|
||||
await demo_robots_compliance()
|
||||
await demo_proxy_rotation()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,280 @@
|
||||
"""
|
||||
🚀 Crawl4AI v0.7.0 Feature Demo
|
||||
================================
|
||||
This file demonstrates the major features introduced in v0.7.0 with practical examples.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from crawl4ai import (
|
||||
AsyncWebCrawler,
|
||||
CrawlerRunConfig,
|
||||
BrowserConfig,
|
||||
CacheMode,
|
||||
# New imports for v0.7.0
|
||||
VirtualScrollConfig,
|
||||
LinkPreviewConfig,
|
||||
AdaptiveCrawler,
|
||||
AdaptiveConfig,
|
||||
AsyncUrlSeeder,
|
||||
SeedingConfig,
|
||||
c4a_compile,
|
||||
)
|
||||
|
||||
|
||||
async def demo_link_preview():
|
||||
"""
|
||||
Demo 1: Link Preview with 3-Layer Scoring
|
||||
|
||||
Shows how to analyze links with intrinsic quality scores,
|
||||
contextual relevance, and combined total scores.
|
||||
"""
|
||||
print("\n" + "="*60)
|
||||
print("🔗 DEMO 1: Link Preview & Intelligent Scoring")
|
||||
print("="*60)
|
||||
|
||||
# Configure link preview with contextual scoring
|
||||
config = CrawlerRunConfig(
|
||||
link_preview_config=LinkPreviewConfig(
|
||||
include_internal=True,
|
||||
include_external=False,
|
||||
max_links=10,
|
||||
concurrency=5,
|
||||
query="machine learning tutorials", # For contextual scoring
|
||||
score_threshold=0.3, # Minimum relevance
|
||||
verbose=True
|
||||
),
|
||||
score_links=True, # Enable intrinsic scoring
|
||||
cache_mode=CacheMode.BYPASS
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun("https://scikit-learn.org/stable/", config=config)
|
||||
|
||||
if result.success:
|
||||
# Get scored links
|
||||
internal_links = result.links.get("internal", [])
|
||||
scored_links = [l for l in internal_links if l.get("total_score")]
|
||||
scored_links.sort(key=lambda x: x.get("total_score", 0), reverse=True)
|
||||
|
||||
print(f"\nTop 5 Most Relevant Links:")
|
||||
for i, link in enumerate(scored_links[:5], 1):
|
||||
print(f"\n{i}. {link.get('text', 'No text')[:50]}...")
|
||||
print(f" URL: {link['href']}")
|
||||
print(f" Intrinsic Score: {link.get('intrinsic_score', 0):.2f}/10")
|
||||
print(f" Contextual Score: {link.get('contextual_score', 0):.3f}")
|
||||
print(f" Total Score: {link.get('total_score', 0):.3f}")
|
||||
|
||||
# Show metadata if available
|
||||
if link.get('head_data'):
|
||||
title = link['head_data'].get('title', 'No title')
|
||||
print(f" Title: {title[:60]}...")
|
||||
|
||||
|
||||
async def demo_adaptive_crawling():
|
||||
"""
|
||||
Demo 2: Adaptive Crawling
|
||||
|
||||
Shows intelligent crawling that stops when enough information
|
||||
is gathered, with confidence tracking.
|
||||
"""
|
||||
print("\n" + "="*60)
|
||||
print("🎯 DEMO 2: Adaptive Crawling with Confidence Tracking")
|
||||
print("="*60)
|
||||
|
||||
# Configure adaptive crawler
|
||||
config = AdaptiveConfig(
|
||||
strategy="statistical", # or "embedding" for semantic understanding
|
||||
max_pages=10,
|
||||
confidence_threshold=0.7, # Stop at 70% confidence
|
||||
top_k_links=3, # Follow top 3 links per page
|
||||
min_gain_threshold=0.05 # Need 5% information gain to continue
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(verbose=False) as crawler:
|
||||
adaptive = AdaptiveCrawler(crawler, config)
|
||||
|
||||
print("Starting adaptive crawl about Python decorators...")
|
||||
result = await adaptive.digest(
|
||||
start_url="https://docs.python.org/3/glossary.html",
|
||||
query="python decorators functions wrapping"
|
||||
)
|
||||
|
||||
print(f"\n✅ Crawling Complete!")
|
||||
print(f"• Confidence Level: {adaptive.confidence:.0%}")
|
||||
print(f"• Pages Crawled: {len(result.crawled_urls)}")
|
||||
print(f"• Knowledge Base: {len(adaptive.state.knowledge_base)} documents")
|
||||
|
||||
# Get most relevant content
|
||||
relevant = adaptive.get_relevant_content(top_k=3)
|
||||
print(f"\nMost Relevant Pages:")
|
||||
for i, page in enumerate(relevant, 1):
|
||||
print(f"{i}. {page['url']} (relevance: {page['score']:.2%})")
|
||||
|
||||
|
||||
async def demo_virtual_scroll():
|
||||
"""
|
||||
Demo 3: Virtual Scroll for Modern Web Pages
|
||||
|
||||
Shows how to capture content from pages with DOM recycling
|
||||
(Twitter, Instagram, infinite scroll).
|
||||
"""
|
||||
print("\n" + "="*60)
|
||||
print("📜 DEMO 3: Virtual Scroll Support")
|
||||
print("="*60)
|
||||
|
||||
# Configure virtual scroll for a news site
|
||||
virtual_config = VirtualScrollConfig(
|
||||
container_selector="main, article, .content", # Common containers
|
||||
scroll_count=20, # Scroll up to 20 times
|
||||
scroll_by="container_height", # Scroll by container height
|
||||
wait_after_scroll=0.5 # Wait 500ms after each scroll
|
||||
)
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
virtual_scroll_config=virtual_config,
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
wait_for="css:article" # Wait for articles to load
|
||||
)
|
||||
|
||||
# Example with a real news site
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun(
|
||||
"https://news.ycombinator.com/",
|
||||
config=config
|
||||
)
|
||||
|
||||
if result.success:
|
||||
# Count items captured
|
||||
import re
|
||||
items = len(re.findall(r'class="athing"', result.html))
|
||||
print(f"\n✅ Captured {items} news items")
|
||||
print(f"• HTML size: {len(result.html):,} bytes")
|
||||
print(f"• Without virtual scroll, would only capture ~30 items")
|
||||
|
||||
|
||||
async def demo_url_seeder():
|
||||
"""
|
||||
Demo 4: URL Seeder for Intelligent Discovery
|
||||
|
||||
Shows how to discover and filter URLs before crawling,
|
||||
with relevance scoring.
|
||||
"""
|
||||
print("\n" + "="*60)
|
||||
print("🌱 DEMO 4: URL Seeder - Smart URL Discovery")
|
||||
print("="*60)
|
||||
|
||||
async with AsyncUrlSeeder() as seeder:
|
||||
# Discover Python tutorial URLs
|
||||
config = SeedingConfig(
|
||||
source="sitemap", # Use sitemap
|
||||
pattern="*python*", # URL pattern filter
|
||||
extract_head=True, # Get metadata
|
||||
query="python tutorial", # For relevance scoring
|
||||
scoring_method="bm25",
|
||||
score_threshold=0.2,
|
||||
max_urls=10
|
||||
)
|
||||
|
||||
print("Discovering Python async tutorial URLs...")
|
||||
urls = await seeder.urls("https://www.geeksforgeeks.org/", config)
|
||||
|
||||
print(f"\n✅ Found {len(urls)} relevant URLs:")
|
||||
for i, url_info in enumerate(urls[:5], 1):
|
||||
print(f"\n{i}. {url_info['url']}")
|
||||
if url_info.get('relevance_score'):
|
||||
print(f" Relevance: {url_info['relevance_score']:.3f}")
|
||||
if url_info.get('head_data', {}).get('title'):
|
||||
print(f" Title: {url_info['head_data']['title'][:60]}...")
|
||||
|
||||
|
||||
async def demo_c4a_script():
|
||||
"""
|
||||
Demo 5: C4A Script Language
|
||||
|
||||
Shows the domain-specific language for web automation
|
||||
with JavaScript transpilation.
|
||||
"""
|
||||
print("\n" + "="*60)
|
||||
print("🎭 DEMO 5: C4A Script - Web Automation Language")
|
||||
print("="*60)
|
||||
|
||||
# Example C4A script
|
||||
c4a_script = """
|
||||
# E-commerce automation script
|
||||
WAIT `body` 3
|
||||
|
||||
# Handle cookie banner
|
||||
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept-cookies`
|
||||
|
||||
# Search for product
|
||||
CLICK `.search-box`
|
||||
TYPE "wireless headphones"
|
||||
PRESS Enter
|
||||
|
||||
# Wait for results
|
||||
WAIT `.product-grid` 10
|
||||
|
||||
# Load more products
|
||||
REPEAT (SCROLL DOWN 500, `document.querySelectorAll('.product').length < 50`)
|
||||
|
||||
# Apply filter
|
||||
IF (EXISTS `.price-filter`) THEN CLICK `input[data-max-price="100"]`
|
||||
"""
|
||||
|
||||
# Compile the script
|
||||
print("Compiling C4A script...")
|
||||
result = c4a_compile(c4a_script)
|
||||
|
||||
if result.success:
|
||||
print(f"✅ Successfully compiled to {len(result.js_code)} JavaScript statements!")
|
||||
print("\nFirst 3 JS statements:")
|
||||
for stmt in result.js_code[:3]:
|
||||
print(f" • {stmt}")
|
||||
|
||||
# Use with crawler
|
||||
config = CrawlerRunConfig(
|
||||
c4a_script=c4a_script, # Pass C4A script directly
|
||||
cache_mode=CacheMode.BYPASS
|
||||
)
|
||||
|
||||
print("\n✅ Script ready for use with AsyncWebCrawler!")
|
||||
else:
|
||||
print(f"❌ Compilation error: {result.first_error.message}")
|
||||
|
||||
|
||||
async def main():
|
||||
"""Run all demos"""
|
||||
print("\n🚀 Crawl4AI v0.7.0 Feature Demonstrations")
|
||||
print("=" * 60)
|
||||
|
||||
demos = [
|
||||
("Link Preview & Scoring", demo_link_preview),
|
||||
("Adaptive Crawling", demo_adaptive_crawling),
|
||||
("Virtual Scroll", demo_virtual_scroll),
|
||||
("URL Seeder", demo_url_seeder),
|
||||
("C4A Script", demo_c4a_script),
|
||||
]
|
||||
|
||||
for name, demo_func in demos:
|
||||
try:
|
||||
await demo_func()
|
||||
except Exception as e:
|
||||
print(f"\n❌ Error in {name} demo: {str(e)}")
|
||||
|
||||
# Pause between demos
|
||||
await asyncio.sleep(1)
|
||||
|
||||
print("\n" + "="*60)
|
||||
print("✅ All demos completed!")
|
||||
print("\nKey Takeaways:")
|
||||
print("• Link Preview: 3-layer scoring for intelligent link analysis")
|
||||
print("• Adaptive Crawling: Stop when you have enough information")
|
||||
print("• Virtual Scroll: Capture all content from modern web pages")
|
||||
print("• URL Seeder: Pre-discover and filter URLs efficiently")
|
||||
print("• C4A Script: Simple language for complex automations")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user