chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,807 @@
|
||||
"""
|
||||
BBC Sport Research Assistant Pipeline
|
||||
=====================================
|
||||
|
||||
This example demonstrates how URLSeeder helps create an efficient research pipeline:
|
||||
1. Discover all available URLs without crawling
|
||||
2. Filter and rank them based on relevance
|
||||
3. Crawl only the most relevant content
|
||||
4. Generate comprehensive research insights
|
||||
|
||||
Pipeline Steps:
|
||||
1. Get user query
|
||||
2. Optionally enhance query using LLM
|
||||
3. Use URLSeeder to discover and rank URLs
|
||||
4. Crawl top K URLs with BM25 filtering
|
||||
5. Generate detailed response with citations
|
||||
|
||||
Requirements:
|
||||
- pip install crawl4ai
|
||||
- pip install litellm
|
||||
- export GEMINI_API_KEY="your-api-key"
|
||||
|
||||
Usage:
|
||||
- Run normally: python bbc_sport_research_assistant.py
|
||||
- Run test mode: python bbc_sport_research_assistant.py test
|
||||
|
||||
Note: AsyncUrlSeeder now uses context manager for automatic cleanup.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import hashlib
|
||||
import pickle
|
||||
from typing import List, Dict, Optional, Tuple
|
||||
from dataclasses import dataclass, asdict
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
# Rich for colored output
|
||||
from rich.console import Console
|
||||
from rich.text import Text
|
||||
from rich.panel import Panel
|
||||
from rich.table import Table
|
||||
from rich.progress import Progress, SpinnerColumn, TextColumn
|
||||
|
||||
# Crawl4AI imports
|
||||
from crawl4ai import (
|
||||
AsyncWebCrawler,
|
||||
BrowserConfig,
|
||||
CrawlerRunConfig,
|
||||
AsyncUrlSeeder,
|
||||
SeedingConfig,
|
||||
AsyncLogger
|
||||
)
|
||||
from crawl4ai.content_filter_strategy import PruningContentFilter
|
||||
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
|
||||
|
||||
# LiteLLM for AI communication
|
||||
import litellm
|
||||
|
||||
# Initialize Rich console
|
||||
console = Console()
|
||||
|
||||
# Get the current directory where this script is located
|
||||
SCRIPT_DIR = Path(__file__).parent.resolve()
|
||||
|
||||
# Cache configuration - relative to script directory
|
||||
CACHE_DIR = SCRIPT_DIR / "temp_cache"
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Testing limits
|
||||
TESTING_MODE = True
|
||||
MAX_URLS_DISCOVERY = 100 if TESTING_MODE else 1000
|
||||
MAX_URLS_TO_CRAWL = 5 if TESTING_MODE else 10
|
||||
|
||||
|
||||
def get_cache_key(prefix: str, *args) -> str:
|
||||
"""Generate cache key from prefix and arguments"""
|
||||
content = f"{prefix}:{'|'.join(str(arg) for arg in args)}"
|
||||
return hashlib.md5(content.encode()).hexdigest()
|
||||
|
||||
|
||||
def load_from_cache(cache_key: str) -> Optional[any]:
|
||||
"""Load data from cache if exists"""
|
||||
cache_path = CACHE_DIR / f"{cache_key}.pkl"
|
||||
if cache_path.exists():
|
||||
with open(cache_path, 'rb') as f:
|
||||
return pickle.load(f)
|
||||
return None
|
||||
|
||||
|
||||
def save_to_cache(cache_key: str, data: any) -> None:
|
||||
"""Save data to cache"""
|
||||
cache_path = CACHE_DIR / f"{cache_key}.pkl"
|
||||
with open(cache_path, 'wb') as f:
|
||||
pickle.dump(data, f)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResearchConfig:
|
||||
"""Configuration for research pipeline"""
|
||||
# Core settings
|
||||
domain: str = "www.bbc.com/sport"
|
||||
max_urls_discovery: int = 100
|
||||
max_urls_to_crawl: int = 10
|
||||
top_k_urls: int = 10
|
||||
|
||||
# Scoring and filtering
|
||||
score_threshold: float = 0.1
|
||||
scoring_method: str = "bm25"
|
||||
|
||||
# Processing options
|
||||
use_llm_enhancement: bool = True
|
||||
extract_head_metadata: bool = True
|
||||
live_check: bool = True
|
||||
force_refresh: bool = False
|
||||
|
||||
# Crawler settings
|
||||
max_concurrent_crawls: int = 5
|
||||
timeout: int = 30000
|
||||
headless: bool = True
|
||||
|
||||
# Output settings
|
||||
save_json: bool = True
|
||||
save_markdown: bool = True
|
||||
output_dir: str = None # Will be set in __post_init__
|
||||
|
||||
# Development settings
|
||||
test_mode: bool = False
|
||||
interactive_mode: bool = False
|
||||
verbose: bool = True
|
||||
|
||||
def __post_init__(self):
|
||||
"""Adjust settings based on test mode"""
|
||||
if self.test_mode:
|
||||
self.max_urls_discovery = 50
|
||||
self.max_urls_to_crawl = 3
|
||||
self.top_k_urls = 5
|
||||
|
||||
# Set default output directory relative to script location
|
||||
if self.output_dir is None:
|
||||
self.output_dir = str(SCRIPT_DIR / "research_results")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResearchQuery:
|
||||
"""Container for research query and metadata"""
|
||||
original_query: str
|
||||
enhanced_query: Optional[str] = None
|
||||
search_patterns: List[str] = None
|
||||
timestamp: str = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResearchResult:
|
||||
"""Container for research results"""
|
||||
query: ResearchQuery
|
||||
discovered_urls: List[Dict]
|
||||
crawled_content: List[Dict]
|
||||
synthesis: str
|
||||
citations: List[Dict]
|
||||
metadata: Dict
|
||||
|
||||
|
||||
async def get_user_query() -> str:
|
||||
"""
|
||||
Get research query from user input
|
||||
"""
|
||||
query = input("\n🔍 Enter your research query: ")
|
||||
return query.strip()
|
||||
|
||||
|
||||
async def enhance_query_with_llm(query: str) -> ResearchQuery:
|
||||
"""
|
||||
Use LLM to enhance the research query:
|
||||
- Extract key terms
|
||||
- Generate search patterns
|
||||
- Identify related topics
|
||||
"""
|
||||
# Check cache
|
||||
cache_key = get_cache_key("enhanced_query", query)
|
||||
cached_result = load_from_cache(cache_key)
|
||||
if cached_result:
|
||||
console.print("[dim cyan]📦 Using cached enhanced query[/dim cyan]")
|
||||
return cached_result
|
||||
|
||||
try:
|
||||
response = await litellm.acompletion(
|
||||
model="gemini/gemini-2.5-flash-preview-04-17",
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": f"""Given this research query: "{query}"
|
||||
|
||||
Extract:
|
||||
1. Key terms and concepts (as a list)
|
||||
2. Related search terms
|
||||
3. A more specific/enhanced version of the query
|
||||
|
||||
Return as JSON:
|
||||
{{
|
||||
"key_terms": ["term1", "term2"],
|
||||
"related_terms": ["related1", "related2"],
|
||||
"enhanced_query": "enhanced version of query"
|
||||
}}"""
|
||||
}],
|
||||
# reasoning_effort="low",
|
||||
temperature=0.3,
|
||||
response_format={"type": "json_object"}
|
||||
)
|
||||
|
||||
data = json.loads(response.choices[0].message.content)
|
||||
|
||||
# Create search patterns
|
||||
all_terms = data["key_terms"] + data["related_terms"]
|
||||
patterns = [f"*{term.lower()}*" for term in all_terms]
|
||||
|
||||
result = ResearchQuery(
|
||||
original_query=query,
|
||||
enhanced_query=data["enhanced_query"],
|
||||
search_patterns=patterns[:10], # Limit patterns
|
||||
timestamp=datetime.now().isoformat()
|
||||
)
|
||||
|
||||
# Cache the result
|
||||
save_to_cache(cache_key, result)
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[yellow]⚠️ LLM enhancement failed: {e}[/yellow]")
|
||||
# Fallback to simple tokenization
|
||||
return ResearchQuery(
|
||||
original_query=query,
|
||||
enhanced_query=query,
|
||||
search_patterns=tokenize_query_to_patterns(query),
|
||||
timestamp=datetime.now().isoformat()
|
||||
)
|
||||
|
||||
|
||||
def tokenize_query_to_patterns(query: str) -> List[str]:
|
||||
"""
|
||||
Convert query into URL patterns for URLSeeder
|
||||
Example: "AI startups funding" -> ["*ai*", "*startup*", "*funding*"]
|
||||
"""
|
||||
# Simple tokenization - split and create patterns
|
||||
words = query.lower().split()
|
||||
# Filter out common words
|
||||
stop_words = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'that'}
|
||||
keywords = [w for w in words if w not in stop_words and len(w) > 2]
|
||||
|
||||
# Create patterns
|
||||
patterns = [f"*{keyword}*" for keyword in keywords]
|
||||
return patterns[:8] # Limit to 8 patterns
|
||||
|
||||
|
||||
async def discover_urls(domain: str, query: str, config: ResearchConfig) -> List[Dict]:
|
||||
"""
|
||||
Use URLSeeder to discover and rank URLs:
|
||||
1. Fetch all URLs from domain
|
||||
2. Filter by patterns
|
||||
3. Extract metadata (titles, descriptions)
|
||||
4. Rank by BM25 relevance score
|
||||
5. Return top K URLs
|
||||
"""
|
||||
# Check cache
|
||||
cache_key = get_cache_key("discovered_urls", domain, query, config.top_k_urls)
|
||||
cached_result = load_from_cache(cache_key)
|
||||
if cached_result and not config.force_refresh:
|
||||
console.print("[dim cyan]📦 Using cached URL discovery[/dim cyan]")
|
||||
return cached_result
|
||||
|
||||
console.print(f"\n[cyan]🔍 Discovering URLs from {domain}...[/cyan]")
|
||||
|
||||
# Initialize URL seeder with context manager
|
||||
async with AsyncUrlSeeder(logger=AsyncLogger(verbose=config.verbose)) as seeder:
|
||||
# Configure seeding
|
||||
seeding_config = SeedingConfig(
|
||||
source="sitemap+cc", # Use both sitemap and Common Crawl
|
||||
extract_head=config.extract_head_metadata,
|
||||
query=query,
|
||||
scoring_method=config.scoring_method,
|
||||
score_threshold=config.score_threshold,
|
||||
max_urls=config.max_urls_discovery,
|
||||
live_check=config.live_check,
|
||||
force=config.force_refresh
|
||||
)
|
||||
|
||||
try:
|
||||
# Discover URLs
|
||||
urls = await seeder.urls(domain, seeding_config)
|
||||
|
||||
# Sort by relevance score (descending)
|
||||
sorted_urls = sorted(
|
||||
urls,
|
||||
key=lambda x: x.get('relevance_score', 0),
|
||||
reverse=True
|
||||
)
|
||||
|
||||
# Take top K
|
||||
top_urls = sorted_urls[:config.top_k_urls]
|
||||
|
||||
console.print(f"[green]✅ Discovered {len(urls)} URLs, selected top {len(top_urls)}[/green]")
|
||||
|
||||
# Cache the result
|
||||
save_to_cache(cache_key, top_urls)
|
||||
return top_urls
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]❌ URL discovery failed: {e}[/red]")
|
||||
return []
|
||||
|
||||
|
||||
async def crawl_selected_urls(urls: List[str], query: str, config: ResearchConfig) -> List[Dict]:
|
||||
"""
|
||||
Crawl selected URLs with content filtering:
|
||||
- Use AsyncWebCrawler.arun_many()
|
||||
- Apply content filter
|
||||
- Generate clean markdown
|
||||
"""
|
||||
# Extract just URLs from the discovery results
|
||||
url_list = [u['url'] for u in urls if 'url' in u][:config.max_urls_to_crawl]
|
||||
|
||||
if not url_list:
|
||||
console.print("[red]❌ No URLs to crawl[/red]")
|
||||
return []
|
||||
|
||||
console.print(f"\n[cyan]🕷️ Crawling {len(url_list)} URLs...[/cyan]")
|
||||
|
||||
# Check cache for each URL
|
||||
crawled_results = []
|
||||
urls_to_crawl = []
|
||||
|
||||
for url in url_list:
|
||||
cache_key = get_cache_key("crawled_content", url, query)
|
||||
cached_content = load_from_cache(cache_key)
|
||||
if cached_content and not config.force_refresh:
|
||||
crawled_results.append(cached_content)
|
||||
else:
|
||||
urls_to_crawl.append(url)
|
||||
|
||||
if urls_to_crawl:
|
||||
console.print(f"[cyan]📥 Crawling {len(urls_to_crawl)} new URLs (cached: {len(crawled_results)})[/cyan]")
|
||||
|
||||
# Configure markdown generator with content filter
|
||||
md_generator = DefaultMarkdownGenerator(
|
||||
content_filter=PruningContentFilter(
|
||||
threshold=0.48,
|
||||
threshold_type="dynamic",
|
||||
min_word_threshold=10
|
||||
),
|
||||
)
|
||||
|
||||
# Configure crawler
|
||||
crawler_config = CrawlerRunConfig(
|
||||
markdown_generator=md_generator,
|
||||
exclude_external_links=True,
|
||||
excluded_tags=['nav', 'header', 'footer', 'aside'],
|
||||
)
|
||||
|
||||
# Create crawler with browser config
|
||||
async with AsyncWebCrawler(
|
||||
config=BrowserConfig(
|
||||
headless=config.headless,
|
||||
verbose=config.verbose
|
||||
)
|
||||
) as crawler:
|
||||
# Crawl URLs
|
||||
results = await crawler.arun_many(
|
||||
urls_to_crawl,
|
||||
config=crawler_config,
|
||||
max_concurrent=config.max_concurrent_crawls
|
||||
)
|
||||
|
||||
# Process results
|
||||
for url, result in zip(urls_to_crawl, results):
|
||||
if result.success:
|
||||
content_data = {
|
||||
'url': url,
|
||||
'title': result.metadata.get('title', ''),
|
||||
'markdown': result.markdown.fit_markdown or result.markdown.raw_markdown,
|
||||
'raw_length': len(result.markdown.raw_markdown),
|
||||
'fit_length': len(result.markdown.fit_markdown) if result.markdown.fit_markdown else len(result.markdown.raw_markdown),
|
||||
'metadata': result.metadata
|
||||
}
|
||||
crawled_results.append(content_data)
|
||||
|
||||
# Cache the result
|
||||
cache_key = get_cache_key("crawled_content", url, query)
|
||||
save_to_cache(cache_key, content_data)
|
||||
else:
|
||||
console.print(f" [red]❌ Failed: {url[:50]}... - {result.error}[/red]")
|
||||
|
||||
console.print(f"[green]✅ Successfully crawled {len(crawled_results)} URLs[/green]")
|
||||
return crawled_results
|
||||
|
||||
|
||||
async def generate_research_synthesis(
|
||||
query: str,
|
||||
crawled_content: List[Dict]
|
||||
) -> Tuple[str, List[Dict]]:
|
||||
"""
|
||||
Use LLM to synthesize research findings:
|
||||
- Analyze all crawled content
|
||||
- Generate comprehensive answer
|
||||
- Extract citations and references
|
||||
"""
|
||||
if not crawled_content:
|
||||
return "No content available for synthesis.", []
|
||||
|
||||
console.print("\n[cyan]🤖 Generating research synthesis...[/cyan]")
|
||||
|
||||
# Prepare content for LLM
|
||||
content_sections = []
|
||||
for i, content in enumerate(crawled_content, 1):
|
||||
section = f"""
|
||||
SOURCE {i}:
|
||||
Title: {content['title']}
|
||||
URL: {content['url']}
|
||||
Content Preview:
|
||||
{content['markdown'][:1500]}...
|
||||
"""
|
||||
content_sections.append(section)
|
||||
|
||||
combined_content = "\n---\n".join(content_sections)
|
||||
|
||||
try:
|
||||
response = await litellm.acompletion(
|
||||
model="gemini/gemini-2.5-flash-preview-04-17",
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": f"""Research Query: "{query}"
|
||||
|
||||
Based on the following sources, provide a comprehensive research synthesis.
|
||||
|
||||
{combined_content}
|
||||
|
||||
Please provide:
|
||||
1. An executive summary (2-3 sentences)
|
||||
2. Key findings (3-5 bullet points)
|
||||
3. Detailed analysis (2-3 paragraphs)
|
||||
4. Future implications or trends
|
||||
|
||||
Format your response with clear sections and cite sources using [Source N] notation.
|
||||
Keep the total response under 800 words."""
|
||||
}],
|
||||
# reasoning_effort="medium",
|
||||
temperature=0.7
|
||||
)
|
||||
|
||||
synthesis = response.choices[0].message.content
|
||||
|
||||
# Extract citations from the synthesis
|
||||
citations = []
|
||||
for i, content in enumerate(crawled_content, 1):
|
||||
if f"[Source {i}]" in synthesis or f"Source {i}" in synthesis:
|
||||
citations.append({
|
||||
'source_id': i,
|
||||
'title': content['title'],
|
||||
'url': content['url']
|
||||
})
|
||||
|
||||
return synthesis, citations
|
||||
|
||||
except Exception as e:
|
||||
console.print(f"[red]❌ Synthesis generation failed: {e}[/red]")
|
||||
# Fallback to simple summary
|
||||
summary = f"Research on '{query}' found {len(crawled_content)} relevant articles:\n\n"
|
||||
for content in crawled_content[:3]:
|
||||
summary += f"- {content['title']}\n {content['url']}\n\n"
|
||||
return summary, []
|
||||
|
||||
|
||||
def format_research_output(result: ResearchResult) -> str:
|
||||
"""
|
||||
Format the final research output with:
|
||||
- Executive summary
|
||||
- Key findings
|
||||
- Detailed analysis
|
||||
- Citations and sources
|
||||
"""
|
||||
output = []
|
||||
output.append("\n" + "=" * 60)
|
||||
output.append("🔬 RESEARCH RESULTS")
|
||||
output.append("=" * 60)
|
||||
|
||||
# Query info
|
||||
output.append(f"\n📋 Query: {result.query.original_query}")
|
||||
if result.query.enhanced_query != result.query.original_query:
|
||||
output.append(f" Enhanced: {result.query.enhanced_query}")
|
||||
|
||||
# Discovery stats
|
||||
output.append(f"\n📊 Statistics:")
|
||||
output.append(f" - URLs discovered: {len(result.discovered_urls)}")
|
||||
output.append(f" - URLs crawled: {len(result.crawled_content)}")
|
||||
output.append(f" - Processing time: {result.metadata.get('duration', 'N/A')}")
|
||||
|
||||
# Synthesis
|
||||
output.append(f"\n📝 SYNTHESIS")
|
||||
output.append("-" * 60)
|
||||
output.append(result.synthesis)
|
||||
|
||||
# Citations
|
||||
if result.citations:
|
||||
output.append(f"\n📚 SOURCES")
|
||||
output.append("-" * 60)
|
||||
for citation in result.citations:
|
||||
output.append(f"[{citation['source_id']}] {citation['title']}")
|
||||
output.append(f" {citation['url']}")
|
||||
|
||||
return "\n".join(output)
|
||||
|
||||
|
||||
async def save_research_results(result: ResearchResult, config: ResearchConfig) -> Tuple[str, str]:
|
||||
"""
|
||||
Save research results in JSON and Markdown formats
|
||||
|
||||
Returns:
|
||||
Tuple of (json_path, markdown_path)
|
||||
"""
|
||||
# Create output directory
|
||||
output_dir = Path(config.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Generate filename based on query and timestamp
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
query_slug = result.query.original_query[:50].replace(" ", "_").replace("/", "_")
|
||||
base_filename = f"{timestamp}_{query_slug}"
|
||||
|
||||
json_path = None
|
||||
md_path = None
|
||||
|
||||
# Save JSON
|
||||
if config.save_json:
|
||||
json_path = output_dir / f"{base_filename}.json"
|
||||
with open(json_path, 'w') as f:
|
||||
json.dump(asdict(result), f, indent=2, default=str)
|
||||
console.print(f"\n[green]💾 JSON saved: {json_path}[/green]")
|
||||
|
||||
# Save Markdown
|
||||
if config.save_markdown:
|
||||
md_path = output_dir / f"{base_filename}.md"
|
||||
|
||||
# Create formatted markdown
|
||||
md_content = [
|
||||
f"# Research Report: {result.query.original_query}",
|
||||
f"\n**Generated on:** {result.metadata.get('timestamp', 'N/A')}",
|
||||
f"\n**Domain:** {result.metadata.get('domain', 'N/A')}",
|
||||
f"\n**Processing time:** {result.metadata.get('duration', 'N/A')}",
|
||||
"\n---\n",
|
||||
"## Query Information",
|
||||
f"- **Original Query:** {result.query.original_query}",
|
||||
f"- **Enhanced Query:** {result.query.enhanced_query or 'N/A'}",
|
||||
f"- **Search Patterns:** {', '.join(result.query.search_patterns or [])}",
|
||||
"\n## Statistics",
|
||||
f"- **URLs Discovered:** {len(result.discovered_urls)}",
|
||||
f"- **URLs Crawled:** {len(result.crawled_content)}",
|
||||
f"- **Sources Cited:** {len(result.citations)}",
|
||||
"\n## Research Synthesis\n",
|
||||
result.synthesis,
|
||||
"\n## Sources\n"
|
||||
]
|
||||
|
||||
# Add citations
|
||||
for citation in result.citations:
|
||||
md_content.append(f"### [{citation['source_id']}] {citation['title']}")
|
||||
md_content.append(f"- **URL:** [{citation['url']}]({citation['url']})")
|
||||
md_content.append("")
|
||||
|
||||
# Add discovered URLs summary
|
||||
md_content.extend([
|
||||
"\n## Discovered URLs (Top 10)\n",
|
||||
"| Score | URL | Title |",
|
||||
"|-------|-----|-------|"
|
||||
])
|
||||
|
||||
for url_data in result.discovered_urls[:10]:
|
||||
score = url_data.get('relevance_score', 0)
|
||||
url = url_data.get('url', '')
|
||||
title = 'N/A'
|
||||
if 'head_data' in url_data and url_data['head_data']:
|
||||
title = url_data['head_data'].get('title', 'N/A')[:60] + '...'
|
||||
md_content.append(f"| {score:.3f} | {url[:50]}... | {title} |")
|
||||
|
||||
# Write markdown
|
||||
with open(md_path, 'w') as f:
|
||||
f.write('\n'.join(md_content))
|
||||
|
||||
console.print(f"[green]📄 Markdown saved: {md_path}[/green]")
|
||||
|
||||
return str(json_path) if json_path else None, str(md_path) if md_path else None
|
||||
|
||||
|
||||
async def wait_for_user(message: str = "\nPress Enter to continue..."):
|
||||
"""Wait for user input in interactive mode"""
|
||||
input(message)
|
||||
|
||||
|
||||
async def research_pipeline(
|
||||
query: str,
|
||||
config: ResearchConfig
|
||||
) -> ResearchResult:
|
||||
"""
|
||||
Main research pipeline orchestrator with configurable settings
|
||||
"""
|
||||
start_time = datetime.now()
|
||||
|
||||
# Display pipeline header
|
||||
header = Panel(
|
||||
f"[bold cyan]Research Pipeline[/bold cyan]\n\n"
|
||||
f"[dim]Domain:[/dim] {config.domain}\n"
|
||||
f"[dim]Mode:[/dim] {'Test' if config.test_mode else 'Production'}\n"
|
||||
f"[dim]Interactive:[/dim] {'Yes' if config.interactive_mode else 'No'}",
|
||||
title="🚀 Starting",
|
||||
border_style="cyan"
|
||||
)
|
||||
console.print(header)
|
||||
|
||||
# Step 1: Enhance query (optional)
|
||||
console.print(f"\n[bold cyan]📝 Step 1: Query Processing[/bold cyan]")
|
||||
if config.interactive_mode:
|
||||
await wait_for_user()
|
||||
|
||||
if config.use_llm_enhancement:
|
||||
research_query = await enhance_query_with_llm(query)
|
||||
else:
|
||||
research_query = ResearchQuery(
|
||||
original_query=query,
|
||||
enhanced_query=query,
|
||||
search_patterns=tokenize_query_to_patterns(query),
|
||||
timestamp=datetime.now().isoformat()
|
||||
)
|
||||
|
||||
console.print(f" [green]✅ Query ready:[/green] {research_query.enhanced_query or query}")
|
||||
|
||||
# Step 2: Discover URLs
|
||||
console.print(f"\n[bold cyan]🔍 Step 2: URL Discovery[/bold cyan]")
|
||||
if config.interactive_mode:
|
||||
await wait_for_user()
|
||||
|
||||
discovered_urls = await discover_urls(
|
||||
domain=config.domain,
|
||||
query=research_query.enhanced_query or query,
|
||||
config=config
|
||||
)
|
||||
|
||||
if not discovered_urls:
|
||||
return ResearchResult(
|
||||
query=research_query,
|
||||
discovered_urls=[],
|
||||
crawled_content=[],
|
||||
synthesis="No relevant URLs found for the given query.",
|
||||
citations=[],
|
||||
metadata={'duration': str(datetime.now() - start_time)}
|
||||
)
|
||||
|
||||
console.print(f" [green]✅ Found {len(discovered_urls)} relevant URLs[/green]")
|
||||
|
||||
# Step 3: Crawl selected URLs
|
||||
console.print(f"\n[bold cyan]🕷️ Step 3: Content Crawling[/bold cyan]")
|
||||
if config.interactive_mode:
|
||||
await wait_for_user()
|
||||
|
||||
crawled_content = await crawl_selected_urls(
|
||||
urls=discovered_urls,
|
||||
query=research_query.enhanced_query or query,
|
||||
config=config
|
||||
)
|
||||
|
||||
console.print(f" [green]✅ Successfully crawled {len(crawled_content)} pages[/green]")
|
||||
|
||||
# Step 4: Generate synthesis
|
||||
console.print(f"\n[bold cyan]🤖 Step 4: Synthesis Generation[/bold cyan]")
|
||||
if config.interactive_mode:
|
||||
await wait_for_user()
|
||||
|
||||
synthesis, citations = await generate_research_synthesis(
|
||||
query=research_query.enhanced_query or query,
|
||||
crawled_content=crawled_content
|
||||
)
|
||||
|
||||
console.print(f" [green]✅ Generated synthesis with {len(citations)} citations[/green]")
|
||||
|
||||
# Step 5: Create result
|
||||
result = ResearchResult(
|
||||
query=research_query,
|
||||
discovered_urls=discovered_urls,
|
||||
crawled_content=crawled_content,
|
||||
synthesis=synthesis,
|
||||
citations=citations,
|
||||
metadata={
|
||||
'duration': str(datetime.now() - start_time),
|
||||
'domain': config.domain,
|
||||
'timestamp': datetime.now().isoformat(),
|
||||
'config': asdict(config)
|
||||
}
|
||||
)
|
||||
|
||||
duration = datetime.now() - start_time
|
||||
console.print(f"\n[bold green]✅ Research completed in {duration}[/bold green]")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
async def main():
|
||||
"""
|
||||
Main entry point for the BBC Sport Research Assistant
|
||||
"""
|
||||
# Example queries
|
||||
example_queries = [
|
||||
"Premier League transfer news and rumors",
|
||||
"Champions League match results and analysis",
|
||||
"World Cup qualifying updates",
|
||||
"Football injury reports and return dates",
|
||||
"Tennis grand slam tournament results"
|
||||
]
|
||||
|
||||
# Display header
|
||||
console.print(Panel.fit(
|
||||
"[bold cyan]BBC Sport Research Assistant[/bold cyan]\n\n"
|
||||
"This tool demonstrates efficient research using URLSeeder:\n"
|
||||
"[dim]• Discover all URLs without crawling\n"
|
||||
"• Filter and rank by relevance\n"
|
||||
"• Crawl only the most relevant content\n"
|
||||
"• Generate AI-powered insights with citations[/dim]\n\n"
|
||||
f"[dim]📁 Working directory: {SCRIPT_DIR}[/dim]",
|
||||
title="🔬 Welcome",
|
||||
border_style="cyan"
|
||||
))
|
||||
|
||||
# Configuration options table
|
||||
config_table = Table(title="\n⚙️ Configuration Options", show_header=False, box=None)
|
||||
config_table.add_column(style="bold cyan", width=3)
|
||||
config_table.add_column()
|
||||
|
||||
config_table.add_row("1", "Quick Test Mode (3 URLs, fast)")
|
||||
config_table.add_row("2", "Standard Mode (10 URLs, balanced)")
|
||||
config_table.add_row("3", "Comprehensive Mode (20 URLs, thorough)")
|
||||
config_table.add_row("4", "Custom Configuration")
|
||||
|
||||
console.print(config_table)
|
||||
|
||||
config_choice = input("\nSelect configuration (1-4): ").strip()
|
||||
|
||||
# Create config based on choice
|
||||
if config_choice == "1":
|
||||
config = ResearchConfig(test_mode=True, interactive_mode=False)
|
||||
elif config_choice == "2":
|
||||
config = ResearchConfig(max_urls_to_crawl=10, top_k_urls=10)
|
||||
elif config_choice == "3":
|
||||
config = ResearchConfig(max_urls_to_crawl=20, top_k_urls=20, max_urls_discovery=200)
|
||||
else:
|
||||
# Custom configuration
|
||||
config = ResearchConfig()
|
||||
config.test_mode = input("\nTest mode? (y/n): ").lower() == 'y'
|
||||
config.interactive_mode = input("Interactive mode (pause between steps)? (y/n): ").lower() == 'y'
|
||||
config.use_llm_enhancement = input("Use AI to enhance queries? (y/n): ").lower() == 'y'
|
||||
|
||||
if not config.test_mode:
|
||||
try:
|
||||
config.max_urls_to_crawl = int(input("Max URLs to crawl (default 10): ") or "10")
|
||||
config.top_k_urls = int(input("Top K URLs to select (default 10): ") or "10")
|
||||
except ValueError:
|
||||
console.print("[yellow]Using default values[/yellow]")
|
||||
|
||||
# Display example queries
|
||||
query_table = Table(title="\n📋 Example Queries", show_header=False, box=None)
|
||||
query_table.add_column(style="bold cyan", width=3)
|
||||
query_table.add_column()
|
||||
|
||||
for i, q in enumerate(example_queries, 1):
|
||||
query_table.add_row(str(i), q)
|
||||
|
||||
console.print(query_table)
|
||||
|
||||
query_input = input("\nSelect a query (1-5) or enter your own: ").strip()
|
||||
|
||||
if query_input.isdigit() and 1 <= int(query_input) <= len(example_queries):
|
||||
query = example_queries[int(query_input) - 1]
|
||||
else:
|
||||
query = query_input if query_input else example_queries[0]
|
||||
|
||||
console.print(f"\n[bold cyan]📝 Selected Query:[/bold cyan] {query}")
|
||||
|
||||
# Run the research pipeline
|
||||
result = await research_pipeline(query=query, config=config)
|
||||
|
||||
# Display results
|
||||
formatted_output = format_research_output(result)
|
||||
# print(formatted_output)
|
||||
console.print(Panel.fit(
|
||||
formatted_output,
|
||||
title="🔬 Research Results",
|
||||
border_style="green"
|
||||
))
|
||||
|
||||
# Save results
|
||||
if config.save_json or config.save_markdown:
|
||||
json_path, md_path = await save_research_results(result, config)
|
||||
# print(f"\n✅ Results saved successfully!")
|
||||
if json_path:
|
||||
console.print(f"[green]JSON saved at:[/green] {json_path}")
|
||||
if md_path:
|
||||
console.print(f"[green]Markdown saved at:[/green] {md_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Convert Crawl4AI URL Seeder tutorial markdown to Colab notebook format
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def parse_markdown_to_cells(markdown_content):
|
||||
"""Parse markdown content and convert to notebook cells"""
|
||||
cells = []
|
||||
|
||||
# Split content by cell markers
|
||||
lines = markdown_content.split('\n')
|
||||
|
||||
# Extract the header content before first cell marker
|
||||
header_lines = []
|
||||
i = 0
|
||||
while i < len(lines) and not lines[i].startswith('# cell'):
|
||||
header_lines.append(lines[i])
|
||||
i += 1
|
||||
|
||||
# Add header as markdown cell if it exists
|
||||
if header_lines:
|
||||
header_content = '\n'.join(header_lines).strip()
|
||||
if header_content:
|
||||
cells.append({
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": header_content.split('\n')
|
||||
})
|
||||
|
||||
# Process cells marked with # cell X type:Y
|
||||
current_cell_content = []
|
||||
current_cell_type = None
|
||||
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
|
||||
# Check for cell marker
|
||||
cell_match = re.match(r'^# cell (\d+) type:(markdown|code)$', line)
|
||||
|
||||
if cell_match:
|
||||
# Save previous cell if exists
|
||||
if current_cell_content and current_cell_type:
|
||||
content = '\n'.join(current_cell_content).strip()
|
||||
if content:
|
||||
if current_cell_type == 'code':
|
||||
cells.append({
|
||||
"cell_type": "code",
|
||||
"execution_count": None,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": content.split('\n')
|
||||
})
|
||||
else:
|
||||
cells.append({
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": content.split('\n')
|
||||
})
|
||||
|
||||
# Start new cell
|
||||
current_cell_type = cell_match.group(2)
|
||||
current_cell_content = []
|
||||
else:
|
||||
# Add line to current cell
|
||||
current_cell_content.append(line)
|
||||
|
||||
i += 1
|
||||
|
||||
# Add last cell if exists
|
||||
if current_cell_content and current_cell_type:
|
||||
content = '\n'.join(current_cell_content).strip()
|
||||
if content:
|
||||
if current_cell_type == 'code':
|
||||
cells.append({
|
||||
"cell_type": "code",
|
||||
"execution_count": None,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": content.split('\n')
|
||||
})
|
||||
else:
|
||||
cells.append({
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": content.split('\n')
|
||||
})
|
||||
|
||||
return cells
|
||||
|
||||
|
||||
def create_colab_notebook(cells):
|
||||
"""Create a Colab notebook structure"""
|
||||
notebook = {
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 0,
|
||||
"metadata": {
|
||||
"colab": {
|
||||
"name": "Crawl4AI_URL_Seeder_Tutorial.ipynb",
|
||||
"provenance": [],
|
||||
"collapsed_sections": [],
|
||||
"toc_visible": True
|
||||
},
|
||||
"kernelspec": {
|
||||
"name": "python3",
|
||||
"display_name": "Python 3"
|
||||
},
|
||||
"language_info": {
|
||||
"name": "python"
|
||||
}
|
||||
},
|
||||
"cells": cells
|
||||
}
|
||||
|
||||
return notebook
|
||||
|
||||
|
||||
def main():
|
||||
# Read the markdown file
|
||||
md_path = Path("tutorial_url_seeder.md")
|
||||
|
||||
if not md_path.exists():
|
||||
print(f"Error: {md_path} not found!")
|
||||
return
|
||||
|
||||
print(f"Reading {md_path}...")
|
||||
with open(md_path, 'r', encoding='utf-8') as f:
|
||||
markdown_content = f.read()
|
||||
|
||||
# Parse markdown to cells
|
||||
print("Parsing markdown content...")
|
||||
cells = parse_markdown_to_cells(markdown_content)
|
||||
print(f"Created {len(cells)} cells")
|
||||
|
||||
# Create notebook
|
||||
print("Creating Colab notebook...")
|
||||
notebook = create_colab_notebook(cells)
|
||||
|
||||
# Save notebook
|
||||
output_path = Path("Crawl4AI_URL_Seeder_Tutorial.ipynb")
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(notebook, f, indent=2, ensure_ascii=False)
|
||||
|
||||
print(f"✅ Successfully created {output_path}")
|
||||
print(f" - Total cells: {len(cells)}")
|
||||
print(f" - Markdown cells: {sum(1 for c in cells if c['cell_type'] == 'markdown')}")
|
||||
print(f" - Code cells: {sum(1 for c in cells if c['cell_type'] == 'code')}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,263 @@
|
||||
"""
|
||||
URL Seeder Demo - Interactive showcase of Crawl4AI's URL discovery capabilities
|
||||
|
||||
This demo shows:
|
||||
1. Basic URL discovery from sitemaps and Common Crawl
|
||||
2. Cache management and forced refresh
|
||||
3. Live URL validation and metadata extraction
|
||||
4. BM25 relevance scoring for intelligent filtering
|
||||
5. Integration with AsyncWebCrawler for the complete pipeline
|
||||
6. Multi-domain discovery across multiple sites
|
||||
|
||||
Note: The AsyncUrlSeeder now supports context manager protocol for automatic cleanup.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from datetime import datetime
|
||||
from rich.console import Console
|
||||
from rich.table import Table
|
||||
from rich.panel import Panel
|
||||
from rich.progress import Progress, SpinnerColumn, BarColumn, TimeElapsedColumn
|
||||
from rich.prompt import Prompt, Confirm
|
||||
from crawl4ai import (
|
||||
AsyncWebCrawler,
|
||||
CrawlerRunConfig,
|
||||
AsyncUrlSeeder,
|
||||
SeedingConfig
|
||||
)
|
||||
|
||||
console = Console()
|
||||
|
||||
console.rule("[bold green]🌐 Crawl4AI URL Seeder: Interactive Demo")
|
||||
|
||||
DOMAIN = "crawl4ai.com"
|
||||
|
||||
# Utils
|
||||
|
||||
def print_head_info(head_data):
|
||||
table = Table(title="<head> Metadata", expand=True)
|
||||
table.add_column("Key", style="cyan", no_wrap=True)
|
||||
table.add_column("Value", style="magenta")
|
||||
|
||||
if not head_data:
|
||||
console.print("[yellow]No head data found.")
|
||||
return
|
||||
|
||||
if head_data.get("title"):
|
||||
table.add_row("title", head_data["title"])
|
||||
if head_data.get("charset"):
|
||||
table.add_row("charset", head_data["charset"])
|
||||
for k, v in head_data.get("meta", {}).items():
|
||||
table.add_row(f"meta:{k}", v)
|
||||
for rel, items in head_data.get("link", {}).items():
|
||||
for item in items:
|
||||
table.add_row(f"link:{rel}", item.get("href", ""))
|
||||
console.print(table)
|
||||
|
||||
|
||||
async def section_1_basic_exploration(seed: AsyncUrlSeeder):
|
||||
console.rule("[bold cyan]1. Basic Seeding")
|
||||
cfg = SeedingConfig(source="cc+sitemap", pattern="*", verbose=True)
|
||||
|
||||
start_time = time.time()
|
||||
with Progress(SpinnerColumn(), "[progress.description]{task.description}") as p:
|
||||
p.add_task(description="Fetching from Common Crawl + Sitemap...", total=None)
|
||||
urls = await seed.urls(DOMAIN, cfg)
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
console.print(f"[green]✓ Fetched {len(urls)} URLs in {elapsed:.2f} seconds")
|
||||
console.print(f"[dim] Speed: {len(urls)/elapsed:.0f} URLs/second[/dim]\n")
|
||||
|
||||
console.print("[bold]Sample URLs:[/bold]")
|
||||
for u in urls[:5]:
|
||||
console.print(f" • {u['url']}")
|
||||
|
||||
|
||||
async def section_2_cache_demo(seed: AsyncUrlSeeder):
|
||||
console.rule("[bold cyan]2. Caching Demonstration")
|
||||
console.print("[yellow]Using `force=True` to bypass cache and fetch fresh data.[/yellow]")
|
||||
cfg = SeedingConfig(source="cc", pattern="*crawl4ai.com/core/*", verbose=False, force = True)
|
||||
await seed.urls(DOMAIN, cfg)
|
||||
|
||||
async def section_3_live_head(seed: AsyncUrlSeeder):
|
||||
console.rule("[bold cyan]3. Live Check + Head Extraction")
|
||||
cfg = SeedingConfig(
|
||||
extract_head=True,
|
||||
concurrency=10,
|
||||
hits_per_sec=5,
|
||||
pattern="*crawl4ai.com/*",
|
||||
max_urls=10,
|
||||
verbose=False,
|
||||
)
|
||||
urls = await seed.urls(DOMAIN, cfg)
|
||||
|
||||
valid = [u for u in urls if u["status"] == "valid"]
|
||||
console.print(f"[green]Valid: {len(valid)} / {len(urls)}")
|
||||
if valid:
|
||||
print_head_info(valid[0]["head_data"])
|
||||
|
||||
|
||||
async def section_4_bm25_scoring(seed: AsyncUrlSeeder):
|
||||
console.rule("[bold cyan]4. BM25 Relevance Scoring")
|
||||
console.print("[yellow]Using AI-powered relevance scoring to find the most relevant content[/yellow]")
|
||||
|
||||
query = "markdown generation extraction strategies"
|
||||
cfg = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query=query,
|
||||
scoring_method="bm25",
|
||||
score_threshold=0.3, # Only URLs with >30% relevance
|
||||
max_urls=20,
|
||||
verbose=False
|
||||
)
|
||||
|
||||
with Progress(SpinnerColumn(), "[progress.description]{task.description}") as p:
|
||||
p.add_task(description=f"Searching for: '{query}'", total=None)
|
||||
urls = await seed.urls(DOMAIN, cfg)
|
||||
|
||||
console.print(f"[green]Found {len(urls)} relevant URLs (score > 0.3)")
|
||||
|
||||
# Show top results with scores
|
||||
table = Table(title="Top 5 Most Relevant Pages", expand=True)
|
||||
table.add_column("Score", style="cyan", width=8)
|
||||
table.add_column("Title", style="magenta")
|
||||
table.add_column("URL", style="blue", overflow="fold")
|
||||
|
||||
for url in urls[:5]:
|
||||
score = f"{url['relevance_score']:.2f}"
|
||||
title = url['head_data'].get('title', 'No title')[:60] + "..."
|
||||
table.add_row(score, title, url['url'])
|
||||
|
||||
console.print(table)
|
||||
|
||||
async def section_5_keyword_filter_to_agent(seed: AsyncUrlSeeder):
|
||||
console.rule("[bold cyan]5. Complete Pipeline: Discover → Filter → Crawl")
|
||||
cfg = SeedingConfig(
|
||||
extract_head=True,
|
||||
concurrency=20,
|
||||
hits_per_sec=10,
|
||||
max_urls=10,
|
||||
pattern="*crawl4ai.com/*",
|
||||
force=True,
|
||||
)
|
||||
urls = await seed.urls(DOMAIN, cfg)
|
||||
|
||||
keywords = ["deep crawling", "markdown", "llm"]
|
||||
selected = [u for u in urls if any(k in str(u["head_data"]).lower() for k in keywords)]
|
||||
|
||||
console.print(f"[cyan]Selected {len(selected)} URLs with relevant keywords:")
|
||||
for u in selected[:10]:
|
||||
console.print("•", u["url"])
|
||||
|
||||
console.print("\n[yellow]Passing above URLs to arun_many() LLM agent for crawling...")
|
||||
async with AsyncWebCrawler(verbose=True) as crawler:
|
||||
crawl_run_config = CrawlerRunConfig(
|
||||
# Example crawl settings for these URLs:
|
||||
only_text=True, # Just get text content
|
||||
screenshot=False,
|
||||
pdf=False,
|
||||
word_count_threshold=50, # Only process pages with at least 50 words
|
||||
stream=True,
|
||||
verbose=False # Keep logs clean for arun_many in this demo
|
||||
)
|
||||
|
||||
# Extract just the URLs from the selected results
|
||||
urls_to_crawl = [u["url"] for u in selected]
|
||||
|
||||
# We'll stream results for large lists, but collect them here for demonstration
|
||||
crawled_results_stream = await crawler.arun_many(urls_to_crawl, config=crawl_run_config)
|
||||
final_crawled_data = []
|
||||
async for result in crawled_results_stream:
|
||||
final_crawled_data.append(result)
|
||||
if len(final_crawled_data) % 5 == 0:
|
||||
print(f" Processed {len(final_crawled_data)}/{len(urls_to_crawl)} URLs...")
|
||||
|
||||
print(f"\n Successfully crawled {len(final_crawled_data)} URLs.")
|
||||
if final_crawled_data:
|
||||
print("\n Example of a crawled result's URL and Markdown (first successful one):")
|
||||
for result in final_crawled_data:
|
||||
if result.success and result.markdown.raw_markdown:
|
||||
print(f" URL: {result.url}")
|
||||
print(f" Markdown snippet: {result.markdown.raw_markdown[:200]}...")
|
||||
break
|
||||
else:
|
||||
print(" No successful crawls with markdown found.")
|
||||
else:
|
||||
print(" No successful crawls found.")
|
||||
|
||||
|
||||
async def section_6_multi_domain(seed: AsyncUrlSeeder):
|
||||
console.rule("[bold cyan]6. Multi-Domain Discovery")
|
||||
console.print("[yellow]Discovering Python tutorials across multiple educational sites[/yellow]\n")
|
||||
|
||||
domains = ["docs.python.org", "realpython.com", "docs.crawl4ai.com"]
|
||||
cfg = SeedingConfig(
|
||||
source="sitemap",
|
||||
extract_head=True,
|
||||
query="python tutorial guide",
|
||||
scoring_method="bm25",
|
||||
score_threshold=0.2,
|
||||
max_urls=5 # Per domain
|
||||
)
|
||||
|
||||
start_time = time.time()
|
||||
with Progress(SpinnerColumn(), "[progress.description]{task.description}") as p:
|
||||
task = p.add_task(description="Discovering across domains...", total=None)
|
||||
results = await seed.many_urls(domains, cfg)
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
total_urls = sum(len(urls) for urls in results.values())
|
||||
console.print(f"[green]✓ Found {total_urls} relevant URLs across {len(domains)} domains in {elapsed:.2f}s\n")
|
||||
|
||||
# Show results per domain
|
||||
for domain, urls in results.items():
|
||||
console.print(f"[bold]{domain}:[/bold] {len(urls)} relevant pages")
|
||||
if urls:
|
||||
top = urls[0]
|
||||
console.print(f" Top result: [{top['relevance_score']:.2f}] {top['head_data'].get('title', 'No title')}")
|
||||
|
||||
|
||||
async def main():
|
||||
async with AsyncUrlSeeder() as seed:
|
||||
# Interactive menu
|
||||
sections = {
|
||||
"1": ("Basic URL Discovery", section_1_basic_exploration),
|
||||
"2": ("Cache Management Demo", section_2_cache_demo),
|
||||
"3": ("Live Check & Metadata Extraction", section_3_live_head),
|
||||
"4": ("BM25 Relevance Scoring", section_4_bm25_scoring),
|
||||
"5": ("Complete Pipeline (Discover → Filter → Crawl)", section_5_keyword_filter_to_agent),
|
||||
"6": ("Multi-Domain Discovery", section_6_multi_domain),
|
||||
"7": ("Run All Demos", None)
|
||||
}
|
||||
|
||||
console.print("\n[bold]Available Demos:[/bold]")
|
||||
for key, (title, _) in sections.items():
|
||||
console.print(f" {key}. {title}")
|
||||
|
||||
choice = Prompt.ask("\n[cyan]Which demo would you like to run?[/cyan]",
|
||||
choices=list(sections.keys()),
|
||||
default="7")
|
||||
|
||||
console.print()
|
||||
|
||||
if choice == "7":
|
||||
# Run all demos
|
||||
for key, (title, func) in sections.items():
|
||||
if key != "7" and func:
|
||||
await func(seed)
|
||||
if key != "6": # Don't pause after the last demo
|
||||
if not Confirm.ask("\n[yellow]Continue to next demo?[/yellow]", default=True):
|
||||
break
|
||||
console.print()
|
||||
else:
|
||||
# Run selected demo
|
||||
_, func = sections[choice]
|
||||
await func(seed)
|
||||
|
||||
console.rule("[bold green]Demo Complete ✔︎")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,128 @@
|
||||
"""
|
||||
🚀 URL Seeder + AsyncWebCrawler = Magic!
|
||||
Quick demo showing discovery → filter → crawl pipeline
|
||||
|
||||
Note: Uses context manager for automatic cleanup of resources.
|
||||
"""
|
||||
import asyncio, os
|
||||
from crawl4ai import AsyncUrlSeeder, AsyncWebCrawler, SeedingConfig, CrawlerRunConfig, AsyncLogger, DefaultMarkdownGenerator
|
||||
from crawl4ai.content_filter_strategy import PruningContentFilter
|
||||
|
||||
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
# 🔍 Example 1: Discover ALL → Filter → Crawl
|
||||
async def discover_and_crawl():
|
||||
"""Find Python module tutorials & extract them all!"""
|
||||
async with AsyncUrlSeeder(logger=AsyncLogger()) as seeder:
|
||||
# Step 1: See how many URLs exist (spoiler: A LOT!)
|
||||
print("📊 Let's see what RealPython has...")
|
||||
all_urls = await seeder.urls("realpython.com",
|
||||
SeedingConfig(source="sitemap"))
|
||||
print(f"😱 Found {len(all_urls)} total URLs!")
|
||||
|
||||
# Step 2: Filter for Python modules (perfect size ~13)
|
||||
print("\n🎯 Filtering for 'python-modules' tutorials...")
|
||||
module_urls = await seeder.urls("realpython.com",
|
||||
SeedingConfig(
|
||||
source="sitemap",
|
||||
pattern="*python-modules*",
|
||||
live_check=True # Make sure they're alive!
|
||||
))
|
||||
|
||||
print(f"✨ Found {len(module_urls)} module tutorials")
|
||||
for url in module_urls[:3]: # Show first 3
|
||||
status = "✅" if url["status"] == "valid" else "❌"
|
||||
print(f"{status} {url['url']}")
|
||||
|
||||
# Step 3: Crawl them all with pruning (keep it lean!)
|
||||
print("\n🕷️ Crawling all module tutorials...")
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
config = CrawlerRunConfig(
|
||||
markdown_generator=DefaultMarkdownGenerator(
|
||||
content_filter=PruningContentFilter( # Smart filtering!
|
||||
threshold=0.48, # Remove fluff
|
||||
threshold_type="fixed",
|
||||
),
|
||||
),
|
||||
only_text=True,
|
||||
stream=True,
|
||||
)
|
||||
|
||||
# Extract just the URLs from the seeder results
|
||||
urls_to_crawl = [u["url"] for u in module_urls[:5]]
|
||||
results = await crawler.arun_many(urls_to_crawl, config=config)
|
||||
|
||||
# Process & save
|
||||
saved = 0
|
||||
async for result in results:
|
||||
if result.success:
|
||||
# Save each tutorial (name from URL)
|
||||
name = result.url.split("/")[-2] + ".md"
|
||||
name = os.path.join(CURRENT_DIR, name)
|
||||
with open(name, "w") as f:
|
||||
f.write(result.markdown.fit_markdown)
|
||||
saved += 1
|
||||
print(f"💾 Saved: {name}")
|
||||
|
||||
print(f"\n🎉 Successfully saved {saved} tutorials!")
|
||||
|
||||
# 🔍 Example 2: Beautiful Soup articles with metadata peek
|
||||
async def explore_beautifulsoup():
|
||||
"""Discover BeautifulSoup content & peek at metadata"""
|
||||
async with AsyncUrlSeeder(logger=AsyncLogger()) as seeder:
|
||||
print("🍲 Looking for Beautiful Soup articles...")
|
||||
soup_urls = await seeder.urls("realpython.com",
|
||||
SeedingConfig(
|
||||
source="sitemap",
|
||||
pattern="*beautiful-soup*",
|
||||
extract_head=True # Get the metadata!
|
||||
))
|
||||
|
||||
print(f"\n📚 Found {len(soup_urls)} Beautiful Soup articles:\n")
|
||||
|
||||
# Show what we discovered
|
||||
for i, url in enumerate(soup_urls, 1):
|
||||
meta = url["head_data"]["meta"]
|
||||
|
||||
print(f"{i}. {url['head_data']['title']}")
|
||||
print(f" 📝 {meta.get('description', 'No description')[:60]}...")
|
||||
print(f" 👤 By: {meta.get('author', 'Unknown')}")
|
||||
print(f" 🔗 {url['url']}\n")
|
||||
|
||||
# 🔍 Example 3: Smart search with BM25 relevance scoring
|
||||
async def smart_search_with_bm25():
|
||||
"""Use AI-powered relevance scoring to find the best content"""
|
||||
async with AsyncUrlSeeder(logger=AsyncLogger()) as seeder:
|
||||
print("🧠 Smart search: 'web scraping tutorial quiz'")
|
||||
|
||||
# Search with BM25 scoring - AI finds the best matches!
|
||||
results = await seeder.urls("realpython.com",
|
||||
SeedingConfig(
|
||||
source="sitemap",
|
||||
pattern="*beautiful-soup*",
|
||||
extract_head=True,
|
||||
query="web scraping tutorial quiz", # Our search
|
||||
scoring_method="bm25",
|
||||
score_threshold=0.2 # Quality filter
|
||||
))
|
||||
|
||||
print(f"\n🎯 Top {len(results)} most relevant results:\n")
|
||||
|
||||
# Show ranked results with relevance scores
|
||||
for i, result in enumerate(results[:3], 1):
|
||||
print(f"{i}. [{result['relevance_score']:.2f}] {result['head_data']['title']}")
|
||||
print(f" 🔗 {result['url'][:60]}...")
|
||||
|
||||
print("\n✨ BM25 automatically ranked by relevance!")
|
||||
|
||||
# 🎬 Run the show!
|
||||
async def main():
|
||||
print("=" * 60)
|
||||
await discover_and_crawl()
|
||||
print("\n" + "=" * 60 + "\n")
|
||||
await explore_beautifulsoup()
|
||||
print("\n" + "=" * 60 + "\n")
|
||||
await smart_search_with_bm25()
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user