chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,244 @@
|
||||
# AdaptiveCrawler
|
||||
|
||||
The `AdaptiveCrawler` class implements intelligent web crawling that automatically determines when sufficient information has been gathered to answer a query. It uses a three-layer scoring system to evaluate coverage, consistency, and saturation.
|
||||
|
||||
## Constructor
|
||||
|
||||
```python
|
||||
AdaptiveCrawler(
|
||||
crawler: AsyncWebCrawler,
|
||||
config: Optional[AdaptiveConfig] = None
|
||||
)
|
||||
```
|
||||
|
||||
### Parameters
|
||||
|
||||
- **crawler** (`AsyncWebCrawler`): The underlying web crawler instance to use for fetching pages
|
||||
- **config** (`Optional[AdaptiveConfig]`): Configuration settings for adaptive crawling behavior. If not provided, uses default settings.
|
||||
|
||||
## Primary Method
|
||||
|
||||
### digest()
|
||||
|
||||
The main method that performs adaptive crawling starting from a URL with a specific query.
|
||||
|
||||
```python
|
||||
async def digest(
|
||||
start_url: str,
|
||||
query: str,
|
||||
resume_from: Optional[Union[str, Path]] = None
|
||||
) -> CrawlState
|
||||
```
|
||||
|
||||
#### Parameters
|
||||
|
||||
- **start_url** (`str`): The starting URL for crawling
|
||||
- **query** (`str`): The search query that guides the crawling process
|
||||
- **resume_from** (`Optional[Union[str, Path]]`): Path to a saved state file to resume from
|
||||
|
||||
#### Returns
|
||||
|
||||
- **CrawlState**: The final crawl state containing all crawled URLs, knowledge base, and metrics
|
||||
|
||||
#### Example
|
||||
|
||||
```python
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
adaptive = AdaptiveCrawler(crawler)
|
||||
state = await adaptive.digest(
|
||||
start_url="https://docs.python.org",
|
||||
query="async context managers"
|
||||
)
|
||||
```
|
||||
|
||||
## Properties
|
||||
|
||||
### confidence
|
||||
|
||||
Current confidence score (0-1) indicating information sufficiency.
|
||||
|
||||
```python
|
||||
@property
|
||||
def confidence(self) -> float
|
||||
```
|
||||
|
||||
### coverage_stats
|
||||
|
||||
Dictionary containing detailed coverage statistics.
|
||||
|
||||
```python
|
||||
@property
|
||||
def coverage_stats(self) -> Dict[str, float]
|
||||
```
|
||||
|
||||
Returns:
|
||||
- **coverage**: Query term coverage score
|
||||
- **consistency**: Information consistency score
|
||||
- **saturation**: Content saturation score
|
||||
- **confidence**: Overall confidence score
|
||||
|
||||
### is_sufficient
|
||||
|
||||
Boolean indicating whether sufficient information has been gathered.
|
||||
|
||||
```python
|
||||
@property
|
||||
def is_sufficient(self) -> bool
|
||||
```
|
||||
|
||||
### state
|
||||
|
||||
Access to the current crawl state.
|
||||
|
||||
```python
|
||||
@property
|
||||
def state(self) -> CrawlState
|
||||
```
|
||||
|
||||
## Methods
|
||||
|
||||
### get_relevant_content()
|
||||
|
||||
Retrieve the most relevant content from the knowledge base.
|
||||
|
||||
```python
|
||||
def get_relevant_content(
|
||||
self,
|
||||
top_k: int = 5
|
||||
) -> List[Dict[str, Any]]
|
||||
```
|
||||
|
||||
#### Parameters
|
||||
|
||||
- **top_k** (`int`): Number of top relevant documents to return (default: 5)
|
||||
|
||||
#### Returns
|
||||
|
||||
List of dictionaries containing:
|
||||
- **url**: The URL of the page
|
||||
- **content**: The page content
|
||||
- **score**: Relevance score
|
||||
- **metadata**: Additional page metadata
|
||||
|
||||
### print_stats()
|
||||
|
||||
Display crawl statistics in formatted output.
|
||||
|
||||
```python
|
||||
def print_stats(
|
||||
self,
|
||||
detailed: bool = False
|
||||
) -> None
|
||||
```
|
||||
|
||||
#### Parameters
|
||||
|
||||
- **detailed** (`bool`): If True, shows detailed metrics with colors. If False, shows summary table.
|
||||
|
||||
### export_knowledge_base()
|
||||
|
||||
Export the collected knowledge base to a JSONL file.
|
||||
|
||||
```python
|
||||
def export_knowledge_base(
|
||||
self,
|
||||
path: Union[str, Path]
|
||||
) -> None
|
||||
```
|
||||
|
||||
#### Parameters
|
||||
|
||||
- **path** (`Union[str, Path]`): Output file path for JSONL export
|
||||
|
||||
#### Example
|
||||
|
||||
```python
|
||||
adaptive.export_knowledge_base("my_knowledge.jsonl")
|
||||
```
|
||||
|
||||
### import_knowledge_base()
|
||||
|
||||
Import a previously exported knowledge base.
|
||||
|
||||
```python
|
||||
async def import_knowledge_base(
|
||||
self,
|
||||
path: Union[str, Path]
|
||||
) -> None
|
||||
```
|
||||
|
||||
#### Parameters
|
||||
|
||||
- **path** (`Union[str, Path]`): Path to JSONL file to import
|
||||
|
||||
## Configuration
|
||||
|
||||
The `AdaptiveConfig` class controls the behavior of adaptive crawling:
|
||||
|
||||
```python
|
||||
@dataclass
|
||||
class AdaptiveConfig:
|
||||
confidence_threshold: float = 0.8 # Stop when confidence reaches this
|
||||
max_pages: int = 50 # Maximum pages to crawl
|
||||
top_k_links: int = 5 # Links to follow per page
|
||||
min_gain_threshold: float = 0.1 # Minimum expected gain to continue
|
||||
save_state: bool = False # Auto-save crawl state
|
||||
state_path: Optional[str] = None # Path for state persistence
|
||||
```
|
||||
|
||||
### Example with Custom Config
|
||||
|
||||
```python
|
||||
config = AdaptiveConfig(
|
||||
confidence_threshold=0.7,
|
||||
max_pages=20,
|
||||
top_k_links=3
|
||||
)
|
||||
|
||||
adaptive = AdaptiveCrawler(crawler, config=config)
|
||||
```
|
||||
|
||||
## Complete Example
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from crawl4ai import AsyncWebCrawler, AdaptiveCrawler, AdaptiveConfig
|
||||
|
||||
async def main():
|
||||
# Configure adaptive crawling
|
||||
config = AdaptiveConfig(
|
||||
confidence_threshold=0.75,
|
||||
max_pages=15,
|
||||
save_state=True,
|
||||
state_path="my_crawl.json"
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
adaptive = AdaptiveCrawler(crawler, config)
|
||||
|
||||
# Start crawling
|
||||
state = await adaptive.digest(
|
||||
start_url="https://example.com/docs",
|
||||
query="authentication oauth2 jwt"
|
||||
)
|
||||
|
||||
# Check results
|
||||
print(f"Confidence achieved: {adaptive.confidence:.0%}")
|
||||
adaptive.print_stats()
|
||||
|
||||
# Get most relevant pages
|
||||
for page in adaptive.get_relevant_content(top_k=3):
|
||||
print(f"- {page['url']} (score: {page['score']:.2f})")
|
||||
|
||||
# Export for later use
|
||||
adaptive.export_knowledge_base("auth_knowledge.jsonl")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
## See Also
|
||||
|
||||
- [digest() Method Reference](digest.md)
|
||||
- [Adaptive Crawling Guide](../core/adaptive-crawling.md)
|
||||
- [Advanced Adaptive Strategies](../advanced/adaptive-strategies.md)
|
||||
@@ -0,0 +1,309 @@
|
||||
# `arun()` Parameter Guide (New Approach)
|
||||
|
||||
In Crawl4AI’s **latest** configuration model, nearly all parameters that once went directly to `arun()` are now part of **`CrawlerRunConfig`**. When calling `arun()`, you provide:
|
||||
|
||||
```python
|
||||
await crawler.arun(
|
||||
url="https://example.com",
|
||||
config=my_run_config
|
||||
)
|
||||
```
|
||||
|
||||
Below is an organized look at the parameters that can go inside `CrawlerRunConfig`, divided by their functional areas. For **Browser** settings (e.g., `headless`, `browser_type`), see [BrowserConfig](./parameters.md).
|
||||
|
||||
---
|
||||
|
||||
## 1. Core Usage
|
||||
|
||||
```python
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
|
||||
|
||||
async def main():
|
||||
run_config = CrawlerRunConfig(
|
||||
verbose=True, # Detailed logging
|
||||
cache_mode=CacheMode.ENABLED, # Use normal read/write cache
|
||||
check_robots_txt=True, # Respect robots.txt rules
|
||||
# ... other parameters
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://example.com",
|
||||
config=run_config
|
||||
)
|
||||
|
||||
# Check if blocked by robots.txt
|
||||
if not result.success and result.status_code == 403:
|
||||
print(f"Error: {result.error_message}")
|
||||
```
|
||||
|
||||
**Key Fields**:
|
||||
- `verbose=True` logs each crawl step.
|
||||
- `cache_mode` decides how to read/write the local crawl cache.
|
||||
|
||||
---
|
||||
|
||||
## 2. Cache Control
|
||||
|
||||
**`cache_mode`** (default: `CacheMode.ENABLED`)
|
||||
Use a built-in enum from `CacheMode`:
|
||||
|
||||
- `ENABLED`: Normal caching—reads if available, writes if missing.
|
||||
- `DISABLED`: No caching—always refetch pages.
|
||||
- `READ_ONLY`: Reads from cache only; no new writes.
|
||||
- `WRITE_ONLY`: Writes to cache but doesn’t read existing data.
|
||||
- `BYPASS`: Skips reading cache for this crawl (though it might still write if set up that way).
|
||||
|
||||
```python
|
||||
run_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS
|
||||
)
|
||||
```
|
||||
|
||||
**Cache modes**:
|
||||
|
||||
- `CacheMode.BYPASS` — Skip cache entirely; always fetch fresh, write result to cache.
|
||||
- `CacheMode.DISABLED` — No caching at all; don't read or write.
|
||||
- `CacheMode.WRITE_ONLY` — Never read from cache, but write results.
|
||||
- `CacheMode.READ_ONLY` — Read from cache if available, never write.
|
||||
|
||||
---
|
||||
|
||||
## 3. Content Processing & Selection
|
||||
|
||||
### 3.1 Text Processing
|
||||
|
||||
```python
|
||||
run_config = CrawlerRunConfig(
|
||||
word_count_threshold=10, # Ignore text blocks <10 words
|
||||
only_text=False, # If True, tries to remove non-text elements
|
||||
keep_data_attributes=False # Keep or discard data-* attributes
|
||||
)
|
||||
```
|
||||
|
||||
### 3.2 Content Selection
|
||||
|
||||
```python
|
||||
run_config = CrawlerRunConfig(
|
||||
css_selector=".main-content", # Focus on .main-content region only
|
||||
excluded_tags=["form", "nav"], # Remove entire tag blocks
|
||||
remove_forms=True, # Specifically strip <form> elements
|
||||
remove_overlay_elements=True, # Attempt to remove modals/popups
|
||||
)
|
||||
```
|
||||
|
||||
### 3.3 Link Handling
|
||||
|
||||
```python
|
||||
run_config = CrawlerRunConfig(
|
||||
exclude_external_links=True, # Remove external links from final content
|
||||
exclude_social_media_links=True, # Remove links to known social sites
|
||||
exclude_domains=["ads.example.com"], # Exclude links to these domains
|
||||
exclude_social_media_domains=["facebook.com","twitter.com"], # Extend the default list
|
||||
)
|
||||
```
|
||||
|
||||
### 3.4 Media Filtering
|
||||
|
||||
```python
|
||||
run_config = CrawlerRunConfig(
|
||||
exclude_external_images=True # Strip images from other domains
|
||||
)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Page Navigation & Timing
|
||||
|
||||
### 4.1 Basic Browser Flow
|
||||
|
||||
```python
|
||||
run_config = CrawlerRunConfig(
|
||||
wait_for="css:.dynamic-content", # Wait for .dynamic-content
|
||||
delay_before_return_html=2.0, # Wait 2s before capturing final HTML
|
||||
page_timeout=60000, # Navigation & script timeout (ms)
|
||||
)
|
||||
```
|
||||
|
||||
**Key Fields**:
|
||||
|
||||
- `wait_for`:
|
||||
- `"css:selector"` or
|
||||
- `"js:() => boolean"`
|
||||
e.g. `js:() => document.querySelectorAll('.item').length > 10`.
|
||||
|
||||
- `mean_delay` & `max_range`: define random delays for `arun_many()` calls.
|
||||
- `semaphore_count`: concurrency limit when crawling multiple URLs.
|
||||
|
||||
### 4.2 JavaScript Execution
|
||||
|
||||
```python
|
||||
run_config = CrawlerRunConfig(
|
||||
js_code=[
|
||||
"window.scrollTo(0, document.body.scrollHeight);",
|
||||
"document.querySelector('.load-more')?.click();"
|
||||
],
|
||||
js_only=False
|
||||
)
|
||||
```
|
||||
|
||||
- `js_code` can be a single string or a list of strings.
|
||||
- `js_only=True` means “I’m continuing in the same session with new JS steps, no new full navigation.”
|
||||
|
||||
### 4.3 Anti-Bot
|
||||
|
||||
```python
|
||||
run_config = CrawlerRunConfig(
|
||||
magic=True,
|
||||
simulate_user=True,
|
||||
override_navigator=True
|
||||
)
|
||||
```
|
||||
- `magic=True` tries multiple stealth features.
|
||||
- `simulate_user=True` mimics mouse movements or random delays.
|
||||
- `override_navigator=True` fakes some navigator properties (like user agent checks).
|
||||
|
||||
---
|
||||
|
||||
## 5. Session Management
|
||||
|
||||
**`session_id`**:
|
||||
```python
|
||||
run_config = CrawlerRunConfig(
|
||||
session_id="my_session123"
|
||||
)
|
||||
```
|
||||
If re-used in subsequent `arun()` calls, the same tab/page context is continued (helpful for multi-step tasks or stateful browsing).
|
||||
|
||||
---
|
||||
|
||||
## 6. Screenshot, PDF & Media Options
|
||||
|
||||
```python
|
||||
run_config = CrawlerRunConfig(
|
||||
screenshot=True, # Grab a screenshot as base64
|
||||
screenshot_wait_for=1.0, # Wait 1s before capturing
|
||||
pdf=True, # Also produce a PDF
|
||||
image_description_min_word_threshold=5, # If analyzing alt text
|
||||
image_score_threshold=3, # Filter out low-score images
|
||||
)
|
||||
```
|
||||
**Where they appear**:
|
||||
- `result.screenshot` → Base64 screenshot string.
|
||||
- `result.pdf` → Byte array with PDF data.
|
||||
|
||||
---
|
||||
|
||||
## 7. Extraction Strategy
|
||||
|
||||
**For advanced data extraction** (CSS/LLM-based), set `extraction_strategy`:
|
||||
|
||||
```python
|
||||
run_config = CrawlerRunConfig(
|
||||
extraction_strategy=my_css_or_llm_strategy
|
||||
)
|
||||
```
|
||||
|
||||
The extracted data will appear in `result.extracted_content`.
|
||||
|
||||
---
|
||||
|
||||
## 8. Comprehensive Example
|
||||
|
||||
Below is a snippet combining many parameters:
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, CacheMode
|
||||
from crawl4ai import JsonCssExtractionStrategy
|
||||
|
||||
async def main():
|
||||
# Example schema
|
||||
schema = {
|
||||
"name": "Articles",
|
||||
"baseSelector": "article.post",
|
||||
"fields": [
|
||||
{"name": "title", "selector": "h2", "type": "text"},
|
||||
{"name": "link", "selector": "a", "type": "attribute", "attribute": "href"}
|
||||
]
|
||||
}
|
||||
|
||||
run_config = CrawlerRunConfig(
|
||||
# Core
|
||||
verbose=True,
|
||||
cache_mode=CacheMode.ENABLED,
|
||||
check_robots_txt=True, # Respect robots.txt rules
|
||||
|
||||
# Content
|
||||
word_count_threshold=10,
|
||||
css_selector="main.content",
|
||||
excluded_tags=["nav", "footer"],
|
||||
exclude_external_links=True,
|
||||
|
||||
# Page & JS
|
||||
js_code="document.querySelector('.show-more')?.click();",
|
||||
wait_for="css:.loaded-block",
|
||||
page_timeout=30000,
|
||||
|
||||
# Extraction
|
||||
extraction_strategy=JsonCssExtractionStrategy(schema),
|
||||
|
||||
# Session
|
||||
session_id="persistent_session",
|
||||
|
||||
# Media
|
||||
screenshot=True,
|
||||
pdf=True,
|
||||
|
||||
# Anti-bot
|
||||
simulate_user=True,
|
||||
magic=True,
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun("https://example.com/posts", config=run_config)
|
||||
if result.success:
|
||||
print("HTML length:", len(result.cleaned_html))
|
||||
print("Extraction JSON:", result.extracted_content)
|
||||
if result.screenshot:
|
||||
print("Screenshot length:", len(result.screenshot))
|
||||
if result.pdf:
|
||||
print("PDF bytes length:", len(result.pdf))
|
||||
else:
|
||||
print("Error:", result.error_message)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
**What we covered**:
|
||||
|
||||
1. **Crawling** the main content region, ignoring external links.
|
||||
2. Running **JavaScript** to click “.show-more”.
|
||||
3. **Waiting** for “.loaded-block” to appear.
|
||||
4. Generating a **screenshot** & **PDF** of the final page.
|
||||
5. Extracting repeated “article.post” elements with a **CSS-based** extraction strategy.
|
||||
|
||||
---
|
||||
|
||||
## 9. Best Practices
|
||||
|
||||
1. **Use `BrowserConfig` for global browser** settings (headless, user agent).
|
||||
2. **Use `CrawlerRunConfig`** to handle the **specific** crawl needs: content filtering, caching, JS, screenshot, extraction, etc.
|
||||
3. Keep your **parameters consistent** in run configs—especially if you’re part of a large codebase with multiple crawls.
|
||||
4. **Limit** large concurrency (`semaphore_count`) if the site or your system can’t handle it.
|
||||
5. For dynamic pages, set `js_code` or `scan_full_page` so you load all content.
|
||||
|
||||
---
|
||||
|
||||
## 10. Conclusion
|
||||
|
||||
All parameters that used to be direct arguments to `arun()` now belong in **`CrawlerRunConfig`**. This approach:
|
||||
|
||||
- Makes code **clearer** and **more maintainable**.
|
||||
- Minimizes confusion about which arguments affect global vs. per-crawl behavior.
|
||||
- Allows you to create **reusable** config objects for different pages or tasks.
|
||||
|
||||
For a **full** reference, check out the [CrawlerRunConfig Docs](./parameters.md).
|
||||
|
||||
Happy crawling with your **structured, flexible** config approach!
|
||||
@@ -0,0 +1,192 @@
|
||||
# `arun_many(...)` Reference
|
||||
|
||||
> **Note**: This function is very similar to [`arun()`](./arun.md) but focused on **concurrent** or **batch** crawling. If you’re unfamiliar with `arun()` usage, please read that doc first, then review this for differences.
|
||||
|
||||
## Function Signature
|
||||
|
||||
```python
|
||||
async def arun_many(
|
||||
urls: Union[List[str], List[Any]],
|
||||
config: Optional[Union[CrawlerRunConfig, List[CrawlerRunConfig]]] = None,
|
||||
dispatcher: Optional[BaseDispatcher] = None,
|
||||
...
|
||||
) -> RunManyReturn:
|
||||
"""
|
||||
Crawl multiple URLs concurrently or in batches.
|
||||
|
||||
:param urls: A list of URLs (or tasks) to crawl.
|
||||
:param config: (Optional) Either:
|
||||
- A single `CrawlerRunConfig` applying to all URLs
|
||||
- A list of `CrawlerRunConfig` objects with url_matcher patterns
|
||||
:param dispatcher: (Optional) A concurrency controller (e.g. MemoryAdaptiveDispatcher).
|
||||
...
|
||||
:return: RunManyReturn containing either a list of `CrawlResult` objects or an async generator if streaming is enabled.
|
||||
"""
|
||||
```
|
||||
|
||||
## Differences from `arun()`
|
||||
|
||||
1. **Multiple URLs**:
|
||||
|
||||
- Instead of crawling a single URL, you pass a list of them (strings or tasks).
|
||||
- The function returns `RunManyReturn` which contains either a **list** of `CrawlResult` or an **async generator** if streaming is enabled.
|
||||
|
||||
2. **Concurrency & Dispatchers**:
|
||||
|
||||
- **`dispatcher`** param allows advanced concurrency control.
|
||||
- If omitted, a default dispatcher (like `MemoryAdaptiveDispatcher`) is used internally.
|
||||
- Dispatchers handle concurrency, rate limiting, and memory-based adaptive throttling (see [Multi-URL Crawling](../advanced/multi-url-crawling.md)).
|
||||
|
||||
3. **Streaming Support**:
|
||||
|
||||
- Enable streaming by setting `stream=True` in your `CrawlerRunConfig`.
|
||||
- When streaming, use `async for` to process results as they become available.
|
||||
- Ideal for processing large numbers of URLs without waiting for all to complete.
|
||||
|
||||
4. **Parallel** Execution**:
|
||||
|
||||
- `arun_many()` can run multiple requests concurrently under the hood.
|
||||
- Each `CrawlResult` might also include a **`dispatch_result`** with concurrency details (like memory usage, start/end times).
|
||||
|
||||
### Basic Example (Batch Mode)
|
||||
|
||||
```python
|
||||
# Minimal usage: The default dispatcher will be used
|
||||
results = await crawler.arun_many(
|
||||
urls=["https://site1.com", "https://site2.com"],
|
||||
config=CrawlerRunConfig(stream=False) # Default behavior
|
||||
)
|
||||
|
||||
for res in results:
|
||||
if res.success:
|
||||
print(res.url, "crawled OK!")
|
||||
else:
|
||||
print("Failed:", res.url, "-", res.error_message)
|
||||
```
|
||||
|
||||
### Streaming Example
|
||||
|
||||
```python
|
||||
config = CrawlerRunConfig(
|
||||
stream=True, # Enable streaming mode
|
||||
cache_mode=CacheMode.BYPASS
|
||||
)
|
||||
|
||||
# Process results as they complete
|
||||
async for result in await crawler.arun_many(
|
||||
urls=["https://site1.com", "https://site2.com", "https://site3.com"],
|
||||
config=config
|
||||
):
|
||||
if result.success:
|
||||
print(f"Just completed: {result.url}")
|
||||
# Process each result immediately
|
||||
process_result(result)
|
||||
```
|
||||
|
||||
### With a Custom Dispatcher
|
||||
|
||||
```python
|
||||
dispatcher = MemoryAdaptiveDispatcher(
|
||||
memory_threshold_percent=70.0,
|
||||
max_session_permit=10
|
||||
)
|
||||
results = await crawler.arun_many(
|
||||
urls=["https://site1.com", "https://site2.com", "https://site3.com"],
|
||||
config=my_run_config,
|
||||
dispatcher=dispatcher
|
||||
)
|
||||
```
|
||||
|
||||
### URL-Specific Configurations
|
||||
|
||||
Instead of using one config for all URLs, provide a list of configs with `url_matcher` patterns:
|
||||
|
||||
```python
|
||||
from crawl4ai import CrawlerRunConfig, MatchMode
|
||||
from crawl4ai.processors.pdf import PDFContentScrapingStrategy
|
||||
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
|
||||
from crawl4ai.content_filter_strategy import PruningContentFilter
|
||||
from crawl4ai.markdown_generation_strategy import DefaultMarkdownGenerator
|
||||
|
||||
# PDF files - specialized extraction
|
||||
pdf_config = CrawlerRunConfig(
|
||||
url_matcher="*.pdf",
|
||||
scraping_strategy=PDFContentScrapingStrategy()
|
||||
)
|
||||
|
||||
# Blog/article pages - content filtering
|
||||
blog_config = CrawlerRunConfig(
|
||||
url_matcher=["*/blog/*", "*/article/*", "*python.org*"],
|
||||
markdown_generator=DefaultMarkdownGenerator(
|
||||
content_filter=PruningContentFilter(threshold=0.48)
|
||||
)
|
||||
)
|
||||
|
||||
# Dynamic pages - JavaScript execution
|
||||
github_config = CrawlerRunConfig(
|
||||
url_matcher=lambda url: 'github.com' in url,
|
||||
js_code="window.scrollTo(0, 500);"
|
||||
)
|
||||
|
||||
# API endpoints - JSON extraction
|
||||
api_config = CrawlerRunConfig(
|
||||
url_matcher=lambda url: 'api' in url or url.endswith('.json'),
|
||||
# Custome settings for JSON extraction
|
||||
)
|
||||
|
||||
# Default fallback config
|
||||
default_config = CrawlerRunConfig() # No url_matcher means it never matches except as fallback
|
||||
|
||||
# Pass the list of configs - first match wins!
|
||||
results = await crawler.arun_many(
|
||||
urls=[
|
||||
"https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf", # → pdf_config
|
||||
"https://blog.python.org/", # → blog_config
|
||||
"https://github.com/microsoft/playwright", # → github_config
|
||||
"https://httpbin.org/json", # → api_config
|
||||
"https://example.com/" # → default_config
|
||||
],
|
||||
config=[pdf_config, blog_config, github_config, api_config, default_config]
|
||||
)
|
||||
```
|
||||
|
||||
**URL Matching Features**:
|
||||
- **String patterns**: `"*.pdf"`, `"*/blog/*"`, `"*python.org*"`
|
||||
- **Function matchers**: `lambda url: 'api' in url`
|
||||
- **Mixed patterns**: Combine strings and functions with `MatchMode.OR` or `MatchMode.AND`
|
||||
- **First match wins**: Configs are evaluated in order
|
||||
|
||||
**Key Points**:
|
||||
- Each URL is processed by the same or separate sessions, depending on the dispatcher’s strategy.
|
||||
- `dispatch_result` in each `CrawlResult` (if using concurrency) can hold memory and timing info.
|
||||
- If you need to handle authentication or session IDs, pass them in each individual task or within your run config.
|
||||
- **Important**: Always include a default config (without `url_matcher`) as the last item if you want to handle all URLs. Otherwise, unmatched URLs will fail.
|
||||
|
||||
### Return Value
|
||||
|
||||
Returns a **`RunManyReturn`** object which contains either a **list** of [`CrawlResult`](./crawl-result.md) objects, or an **async generator** if streaming is enabled. You can iterate to check `result.success` or read each item’s `extracted_content`, `markdown`, or `dispatch_result`.
|
||||
|
||||
---
|
||||
|
||||
## Dispatcher Reference
|
||||
|
||||
- **`MemoryAdaptiveDispatcher`**: Dynamically manages concurrency based on system memory usage.
|
||||
- **`SemaphoreDispatcher`**: Fixed concurrency limit, simpler but less adaptive.
|
||||
|
||||
For advanced usage or custom settings, see [Multi-URL Crawling with Dispatchers](../advanced/multi-url-crawling.md).
|
||||
|
||||
---
|
||||
|
||||
## Common Pitfalls
|
||||
|
||||
1. **Large Lists**: If you pass thousands of URLs, be mindful of memory or rate-limits. A dispatcher can help.
|
||||
|
||||
2. **Session Reuse**: If you need specialized logins or persistent contexts, ensure your dispatcher or tasks handle sessions accordingly.
|
||||
|
||||
3. **Error Handling**: Each `CrawlResult` might fail for different reasons—always check `result.success` or the `error_message` before proceeding.
|
||||
|
||||
---
|
||||
|
||||
## Conclusion
|
||||
|
||||
Use `arun_many()` when you want to **crawl multiple URLs** simultaneously or in controlled parallel tasks. If you need advanced concurrency features (like memory-based adaptive throttling or complex rate-limiting), provide a **dispatcher**. Each result is a standard `CrawlResult`, possibly augmented with concurrency stats (`dispatch_result`) for deeper inspection. For more details on concurrency logic and dispatchers, see the [Advanced Multi-URL Crawling](../advanced/multi-url-crawling.md) docs.
|
||||
@@ -0,0 +1,311 @@
|
||||
# AsyncWebCrawler
|
||||
|
||||
The **`AsyncWebCrawler`** is the core class for asynchronous web crawling in Crawl4AI. You typically create it **once**, optionally customize it with a **`BrowserConfig`** (e.g., headless, user agent), then **run** multiple **`arun()`** calls with different **`CrawlerRunConfig`** objects.
|
||||
|
||||
**Recommended usage**:
|
||||
|
||||
1. **Create** a `BrowserConfig` for global browser settings.
|
||||
|
||||
2. **Instantiate** `AsyncWebCrawler(config=browser_config)`.
|
||||
|
||||
3. **Use** the crawler in an async context manager (`async with`) or manage start/close manually.
|
||||
|
||||
4. **Call** `arun(url, config=crawler_run_config)` for each page you want.
|
||||
|
||||
---
|
||||
|
||||
## 1. Constructor Overview
|
||||
|
||||
```python
|
||||
class AsyncWebCrawler:
|
||||
def __init__(
|
||||
self,
|
||||
crawler_strategy: Optional[AsyncCrawlerStrategy] = None,
|
||||
config: Optional[BrowserConfig] = None,
|
||||
always_bypass_cache: bool = False, # deprecated
|
||||
always_by_pass_cache: Optional[bool] = None, # also deprecated
|
||||
base_directory: str = ...,
|
||||
thread_safe: bool = False,
|
||||
**kwargs,
|
||||
):
|
||||
"""
|
||||
Create an AsyncWebCrawler instance.
|
||||
|
||||
Args:
|
||||
crawler_strategy:
|
||||
(Advanced) Provide a custom crawler strategy if needed.
|
||||
config:
|
||||
A BrowserConfig object specifying how the browser is set up.
|
||||
always_bypass_cache:
|
||||
(Deprecated) Use CrawlerRunConfig.cache_mode instead.
|
||||
base_directory:
|
||||
Folder for storing caches/logs (if relevant).
|
||||
thread_safe:
|
||||
If True, attempts some concurrency safeguards. Usually False.
|
||||
**kwargs:
|
||||
Additional legacy or debugging parameters.
|
||||
"""
|
||||
)
|
||||
|
||||
### Typical Initialization
|
||||
|
||||
```python
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig
|
||||
|
||||
browser_cfg = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
crawler = AsyncWebCrawler(config=browser_cfg)
|
||||
```
|
||||
|
||||
**Notes**:
|
||||
|
||||
- **Legacy** parameters like `always_bypass_cache` remain for backward compatibility, but prefer to set **caching** in `CrawlerRunConfig`.
|
||||
|
||||
---
|
||||
|
||||
## 2. Lifecycle: Start/Close or Context Manager
|
||||
|
||||
### 2.1 Context Manager (Recommended)
|
||||
|
||||
```python
|
||||
async with AsyncWebCrawler(config=browser_cfg) as crawler:
|
||||
result = await crawler.arun("https://example.com")
|
||||
# The crawler automatically starts/closes resources
|
||||
```
|
||||
|
||||
When the `async with` block ends, the crawler cleans up (closes the browser, etc.).
|
||||
|
||||
### 2.2 Manual Start & Close
|
||||
|
||||
```python
|
||||
crawler = AsyncWebCrawler(config=browser_cfg)
|
||||
await crawler.start()
|
||||
|
||||
result1 = await crawler.arun("https://example.com")
|
||||
result2 = await crawler.arun("https://another.com")
|
||||
|
||||
await crawler.close()
|
||||
```
|
||||
|
||||
Use this style if you have a **long-running** application or need full control of the crawler’s lifecycle.
|
||||
|
||||
---
|
||||
|
||||
## 3. Primary Method: `arun()`
|
||||
|
||||
```python
|
||||
async def arun(
|
||||
self,
|
||||
url: str,
|
||||
config: Optional[CrawlerRunConfig] = None,
|
||||
# Legacy parameters for backward compatibility...
|
||||
) -> RunManyReturn:
|
||||
...
|
||||
```
|
||||
|
||||
### 3.1 New Approach
|
||||
|
||||
You pass a `CrawlerRunConfig` object that sets up everything about a crawl—content filtering, caching, session reuse, JS code, screenshots, etc.
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from crawl4ai import CrawlerRunConfig, CacheMode
|
||||
|
||||
run_cfg = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
css_selector="main.article",
|
||||
word_count_threshold=10,
|
||||
screenshot=True
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_cfg) as crawler:
|
||||
result = await crawler.arun("https://example.com/news", config=run_cfg)
|
||||
print("Crawled HTML length:", len(result.cleaned_html))
|
||||
if result.screenshot:
|
||||
print("Screenshot base64 length:", len(result.screenshot))
|
||||
```
|
||||
|
||||
### 3.2 Legacy Parameters Still Accepted
|
||||
|
||||
For **backward** compatibility, `arun()` can still accept direct arguments like `css_selector=...`, `word_count_threshold=...`, etc., but we strongly advise migrating them into a **`CrawlerRunConfig`**.
|
||||
|
||||
---
|
||||
|
||||
## 4. Batch Processing: `arun_many()`
|
||||
|
||||
```python
|
||||
async def arun_many(
|
||||
self,
|
||||
urls: List[str],
|
||||
config: Optional[CrawlerRunConfig] = None,
|
||||
# Legacy parameters maintained for backwards compatibility...
|
||||
) -> RunManyReturn:
|
||||
"""
|
||||
Process multiple URLs with intelligent rate limiting and resource monitoring.
|
||||
"""
|
||||
```
|
||||
|
||||
### 4.1 Resource-Aware Crawling
|
||||
|
||||
The `arun_many()` method now uses an intelligent dispatcher that:
|
||||
|
||||
- Monitors system memory usage
|
||||
- Implements adaptive rate limiting
|
||||
- Provides detailed progress monitoring
|
||||
- Manages concurrent crawls efficiently
|
||||
|
||||
### 4.2 Example Usage
|
||||
|
||||
Check page [Multi-url Crawling](../advanced/multi-url-crawling.md) for a detailed example of how to use `arun_many()`.
|
||||
|
||||
```python
|
||||
|
||||
### 4.3 Key Features
|
||||
|
||||
1. **Rate Limiting**
|
||||
|
||||
- Automatic delay between requests
|
||||
- Exponential backoff on rate limit detection
|
||||
- Domain-specific rate limiting
|
||||
- Configurable retry strategy
|
||||
|
||||
2. **Resource Monitoring**
|
||||
|
||||
- Memory usage tracking
|
||||
- Adaptive concurrency based on system load
|
||||
- Automatic pausing when resources are constrained
|
||||
|
||||
3. **Progress Monitoring**
|
||||
|
||||
- Detailed or aggregated progress display
|
||||
- Real-time status updates
|
||||
- Memory usage statistics
|
||||
|
||||
4. **Error Handling**
|
||||
|
||||
- Graceful handling of rate limits
|
||||
- Automatic retries with backoff
|
||||
- Detailed error reporting
|
||||
|
||||
---
|
||||
|
||||
## 5. `CrawlResult` Output
|
||||
|
||||
Each `arun()` returns a **`CrawlResult`** containing:
|
||||
|
||||
- `url`: Final URL (if redirected).
|
||||
- `html`: Original HTML.
|
||||
- `cleaned_html`: Sanitized HTML.
|
||||
- `markdown_v2`: Removed in v0.5. Accessing it raises `AttributeError`; use `markdown`.
|
||||
- `extracted_content`: If an extraction strategy was used (JSON for CSS/LLM strategies).
|
||||
- `screenshot`, `pdf`: If screenshots/PDF requested.
|
||||
- `media`, `links`: Information about discovered images/links.
|
||||
- `success`, `error_message`: Status info.
|
||||
|
||||
For details, see [CrawlResult doc](./crawl-result.md).
|
||||
|
||||
---
|
||||
|
||||
## 6. Quick Example
|
||||
|
||||
Below is an example hooking it all together:
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
|
||||
from crawl4ai import JsonCssExtractionStrategy
|
||||
import json
|
||||
|
||||
async def main():
|
||||
# 1. Browser config
|
||||
browser_cfg = BrowserConfig(
|
||||
browser_type="firefox",
|
||||
headless=False,
|
||||
verbose=True
|
||||
)
|
||||
|
||||
# 2. Run config
|
||||
schema = {
|
||||
"name": "Articles",
|
||||
"baseSelector": "article.post",
|
||||
"fields": [
|
||||
{
|
||||
"name": "title",
|
||||
"selector": "h2",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "url",
|
||||
"selector": "a",
|
||||
"type": "attribute",
|
||||
"attribute": "href"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
run_cfg = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
extraction_strategy=JsonCssExtractionStrategy(schema),
|
||||
word_count_threshold=15,
|
||||
remove_overlay_elements=True,
|
||||
wait_for="css:.post" # Wait for posts to appear
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_cfg) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://example.com/blog",
|
||||
config=run_cfg
|
||||
)
|
||||
|
||||
if result.success:
|
||||
print("Cleaned HTML length:", len(result.cleaned_html))
|
||||
if result.extracted_content:
|
||||
articles = json.loads(result.extracted_content)
|
||||
print("Extracted articles:", articles[:2])
|
||||
else:
|
||||
print("Error:", result.error_message)
|
||||
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
**Explanation**:
|
||||
|
||||
- We define a **`BrowserConfig`** with Firefox, no headless, and `verbose=True`.
|
||||
- We define a **`CrawlerRunConfig`** that **bypasses cache**, uses a **CSS** extraction schema, has a `word_count_threshold=15`, etc.
|
||||
- We pass them to `AsyncWebCrawler(config=...)` and `arun(url=..., config=...)`.
|
||||
|
||||
---
|
||||
|
||||
## 7. Best Practices & Migration Notes
|
||||
|
||||
1. **Use** `BrowserConfig` for **global** settings about the browser’s environment.
|
||||
2. **Use** `CrawlerRunConfig` for **per-crawl** logic (caching, content filtering, extraction strategies, wait conditions).
|
||||
3. **Avoid** legacy parameters like `css_selector` or `word_count_threshold` directly in `arun()`. Instead:
|
||||
|
||||
```python
|
||||
run_cfg = CrawlerRunConfig(css_selector=".main-content", word_count_threshold=20)
|
||||
result = await crawler.arun(url="...", config=run_cfg)
|
||||
```
|
||||
|
||||
4. **Context Manager** usage is simplest unless you want a persistent crawler across many calls.
|
||||
|
||||
---
|
||||
|
||||
## 8. Summary
|
||||
|
||||
**AsyncWebCrawler** is your entry point to asynchronous crawling:
|
||||
|
||||
- **Constructor** accepts **`BrowserConfig`** (or defaults).
|
||||
- **`arun(url, config=CrawlerRunConfig)`** is the main method for single-page crawls.
|
||||
- **`arun_many(urls, config=CrawlerRunConfig)`** handles concurrency across multiple URLs.
|
||||
- For advanced lifecycle control, use `start()` and `close()` explicitly.
|
||||
|
||||
**Migration**:
|
||||
|
||||
- If you used `AsyncWebCrawler(browser_type="chromium", css_selector="...")`, move browser settings to `BrowserConfig(...)` and content/crawl logic to `CrawlerRunConfig(...)`.
|
||||
|
||||
This modular approach ensures your code is **clean**, **scalable**, and **easy to maintain**. For any advanced or rarely used parameters, see the [BrowserConfig docs](../api/parameters.md).
|
||||
@@ -0,0 +1,992 @@
|
||||
# C4A-Script API Reference
|
||||
|
||||
Complete reference for all C4A-Script commands, syntax, and advanced features.
|
||||
|
||||
## Command Categories
|
||||
|
||||
### 🧭 Navigation Commands
|
||||
|
||||
Navigate between pages and manage browser history.
|
||||
|
||||
#### `GO <url>`
|
||||
Navigate to a specific URL.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
GO <url>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `url` - Target URL (string)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
GO https://example.com
|
||||
GO https://api.example.com/login
|
||||
GO /relative/path
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Supports both absolute and relative URLs
|
||||
- Automatically handles protocol detection
|
||||
- Waits for page load to complete
|
||||
|
||||
---
|
||||
|
||||
#### `RELOAD`
|
||||
Refresh the current page.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
RELOAD
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
RELOAD
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Equivalent to pressing F5 or clicking browser refresh
|
||||
- Waits for page reload to complete
|
||||
- Preserves current URL
|
||||
|
||||
---
|
||||
|
||||
#### `BACK`
|
||||
Navigate back in browser history.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
BACK
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
BACK
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Equivalent to clicking browser back button
|
||||
- Does nothing if no previous page exists
|
||||
- Waits for navigation to complete
|
||||
|
||||
---
|
||||
|
||||
#### `FORWARD`
|
||||
Navigate forward in browser history.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
FORWARD
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
FORWARD
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Equivalent to clicking browser forward button
|
||||
- Does nothing if no next page exists
|
||||
- Waits for navigation to complete
|
||||
|
||||
### ⏱️ Wait Commands
|
||||
|
||||
Control timing and synchronization with page elements.
|
||||
|
||||
#### `WAIT <time>`
|
||||
Wait for a specified number of seconds.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
WAIT <seconds>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `seconds` - Number of seconds to wait (number)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
WAIT 3
|
||||
WAIT 1.5
|
||||
WAIT 10
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Accepts decimal values
|
||||
- Useful for giving dynamic content time to load
|
||||
- Non-blocking for other browser operations
|
||||
|
||||
---
|
||||
|
||||
#### `WAIT <selector> <timeout>`
|
||||
Wait for an element to appear on the page.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
WAIT `<selector>` <timeout>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `selector` - CSS selector for the element (string in backticks)
|
||||
- `timeout` - Maximum seconds to wait (number)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
WAIT `#content` 10
|
||||
WAIT `.loading-spinner` 5
|
||||
WAIT `button[type="submit"]` 15
|
||||
WAIT `.results .item:first-child` 8
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Fails if element doesn't appear within timeout
|
||||
- More reliable than fixed time waits
|
||||
- Supports complex CSS selectors
|
||||
|
||||
---
|
||||
|
||||
#### `WAIT "<text>" <timeout>`
|
||||
Wait for specific text to appear anywhere on the page.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
WAIT "<text>" <timeout>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `text` - Text content to wait for (string in quotes)
|
||||
- `timeout` - Maximum seconds to wait (number)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
WAIT "Loading complete" 10
|
||||
WAIT "Welcome back" 5
|
||||
WAIT "Search results" 15
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Case-sensitive text matching
|
||||
- Searches entire page content
|
||||
- Useful for dynamic status messages
|
||||
|
||||
### 🖱️ Mouse Commands
|
||||
|
||||
Simulate mouse interactions and movements.
|
||||
|
||||
#### `CLICK <selector>`
|
||||
Click on an element specified by CSS selector.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
CLICK `<selector>`
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `selector` - CSS selector for the element (string in backticks)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
CLICK `#submit-button`
|
||||
CLICK `.menu-item:first-child`
|
||||
CLICK `button[data-action="save"]`
|
||||
CLICK `a[href="/dashboard"]`
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Waits for element to be clickable
|
||||
- Scrolls element into view if necessary
|
||||
- Handles overlapping elements intelligently
|
||||
|
||||
---
|
||||
|
||||
#### `CLICK <x> <y>`
|
||||
Click at specific coordinates on the page.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
CLICK <x> <y>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `x` - X coordinate in pixels (number)
|
||||
- `y` - Y coordinate in pixels (number)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
CLICK 100 200
|
||||
CLICK 500 300
|
||||
CLICK 0 0
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Coordinates are relative to viewport
|
||||
- Useful when element selectors are unreliable
|
||||
- Consider responsive design implications
|
||||
|
||||
---
|
||||
|
||||
#### `DOUBLE_CLICK <selector>`
|
||||
Double-click on an element.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
DOUBLE_CLICK `<selector>`
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `selector` - CSS selector for the element (string in backticks)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
DOUBLE_CLICK `.file-icon`
|
||||
DOUBLE_CLICK `#editable-cell`
|
||||
DOUBLE_CLICK `.expandable-item`
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Triggers dblclick event
|
||||
- Common for opening files or editing inline content
|
||||
- Timing between clicks is automatically handled
|
||||
|
||||
---
|
||||
|
||||
#### `RIGHT_CLICK <selector>`
|
||||
Right-click on an element to open context menu.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
RIGHT_CLICK `<selector>`
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `selector` - CSS selector for the element (string in backticks)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
RIGHT_CLICK `#context-target`
|
||||
RIGHT_CLICK `.menu-trigger`
|
||||
RIGHT_CLICK `img.thumbnail`
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Opens browser/application context menu
|
||||
- Useful for testing context menu interactions
|
||||
- May be blocked by some applications
|
||||
|
||||
---
|
||||
|
||||
#### `SCROLL <direction> <amount>`
|
||||
Scroll the page in a specified direction.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
SCROLL <direction> <amount>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `direction` - Direction to scroll: `UP`, `DOWN`, `LEFT`, `RIGHT`
|
||||
- `amount` - Number of pixels to scroll (number)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
SCROLL DOWN 500
|
||||
SCROLL UP 200
|
||||
SCROLL LEFT 100
|
||||
SCROLL RIGHT 300
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Smooth scrolling animation
|
||||
- Useful for infinite scroll pages
|
||||
- Amount can be larger than viewport
|
||||
|
||||
---
|
||||
|
||||
#### `MOVE <x> <y>`
|
||||
Move mouse cursor to specific coordinates.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
MOVE <x> <y>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `x` - X coordinate in pixels (number)
|
||||
- `y` - Y coordinate in pixels (number)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
MOVE 200 100
|
||||
MOVE 500 400
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Triggers hover effects
|
||||
- Useful for testing mouseover interactions
|
||||
- Does not click, only moves cursor
|
||||
|
||||
---
|
||||
|
||||
#### `DRAG <x1> <y1> <x2> <y2>`
|
||||
Drag from one point to another.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
DRAG <x1> <y1> <x2> <y2>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `x1`, `y1` - Starting coordinates (numbers)
|
||||
- `x2`, `y2` - Ending coordinates (numbers)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
DRAG 100 100 500 300
|
||||
DRAG 0 200 400 200
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Simulates click, drag, and release
|
||||
- Useful for sliders, resizing, reordering
|
||||
- Smooth drag animation
|
||||
|
||||
### ⌨️ Keyboard Commands
|
||||
|
||||
Simulate keyboard input and key presses.
|
||||
|
||||
#### `TYPE "<text>"`
|
||||
Type text into the currently focused element.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
TYPE "<text>"
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `text` - Text to type (string in quotes)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
TYPE "Hello, World!"
|
||||
TYPE "user@example.com"
|
||||
TYPE "Password123!"
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Requires an input element to be focused
|
||||
- Types character by character with realistic timing
|
||||
- Supports special characters and Unicode
|
||||
|
||||
---
|
||||
|
||||
#### `TYPE $<variable>`
|
||||
Type the value of a variable.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
TYPE $<variable>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `variable` - Variable name (without quotes)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
SETVAR email = "user@example.com"
|
||||
TYPE $email
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Variable must be defined with SETVAR first
|
||||
- Variable values are strings
|
||||
- Useful for reusable credentials or data
|
||||
|
||||
---
|
||||
|
||||
#### `PRESS <key>`
|
||||
Press and release a special key.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
PRESS <key>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `key` - Key name (see supported keys below)
|
||||
|
||||
**Supported Keys:**
|
||||
- `Tab`, `Enter`, `Escape`, `Space`
|
||||
- `ArrowUp`, `ArrowDown`, `ArrowLeft`, `ArrowRight`
|
||||
- `Delete`, `Backspace`
|
||||
- `Home`, `End`, `PageUp`, `PageDown`
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
PRESS Tab
|
||||
PRESS Enter
|
||||
PRESS Escape
|
||||
PRESS ArrowDown
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Simulates actual key press and release
|
||||
- Useful for form navigation and shortcuts
|
||||
- Case-sensitive key names
|
||||
|
||||
---
|
||||
|
||||
#### `KEY_DOWN <key>`
|
||||
Hold down a modifier key.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
KEY_DOWN <key>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `key` - Modifier key: `Shift`, `Control`, `Alt`, `Meta`
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
KEY_DOWN Shift
|
||||
KEY_DOWN Control
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Must be paired with KEY_UP
|
||||
- Useful for key combinations
|
||||
- Meta key is Cmd on Mac, Windows key on PC
|
||||
|
||||
---
|
||||
|
||||
#### `KEY_UP <key>`
|
||||
Release a modifier key.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
KEY_UP <key>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `key` - Modifier key: `Shift`, `Control`, `Alt`, `Meta`
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
KEY_UP Shift
|
||||
KEY_UP Control
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Must be paired with KEY_DOWN
|
||||
- Releases the specified modifier key
|
||||
- Good practice to always release held keys
|
||||
|
||||
---
|
||||
|
||||
#### `CLEAR <selector>`
|
||||
Clear the content of an input field.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
CLEAR `<selector>`
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `selector` - CSS selector for input element (string in backticks)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
CLEAR `#search-box`
|
||||
CLEAR `input[name="email"]`
|
||||
CLEAR `.form-input:first-child`
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Works with input, textarea elements
|
||||
- Faster than selecting all and deleting
|
||||
- Triggers appropriate change events
|
||||
|
||||
---
|
||||
|
||||
#### `SET <selector> "<value>"`
|
||||
Set the value of an input field directly.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
SET `<selector>` "<value>"
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `selector` - CSS selector for input element (string in backticks)
|
||||
- `value` - Value to set (string in quotes)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
SET `#email` "user@example.com"
|
||||
SET `#age` "25"
|
||||
SET `textarea#message` "Hello, this is a test message."
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Directly sets value without typing animation
|
||||
- Faster than TYPE for long text
|
||||
- Triggers change and input events
|
||||
|
||||
### 🔀 Control Flow Commands
|
||||
|
||||
Add conditional logic and loops to your scripts.
|
||||
|
||||
#### `IF (EXISTS <selector>) THEN <command>`
|
||||
Execute command if element exists.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
IF (EXISTS `<selector>`) THEN <command>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `selector` - CSS selector to check (string in backticks)
|
||||
- `command` - Command to execute if condition is true
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept-cookies`
|
||||
IF (EXISTS `#popup-modal`) THEN CLICK `.close-button`
|
||||
IF (EXISTS `.error-message`) THEN RELOAD
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Checks for element existence at time of execution
|
||||
- Does not wait for element to appear
|
||||
- Can be combined with ELSE
|
||||
|
||||
---
|
||||
|
||||
#### `IF (EXISTS <selector>) THEN <command> ELSE <command>`
|
||||
Execute command based on element existence.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
IF (EXISTS `<selector>`) THEN <command> ELSE <command>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `selector` - CSS selector to check (string in backticks)
|
||||
- First `command` - Execute if condition is true
|
||||
- Second `command` - Execute if condition is false
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
IF (EXISTS `.user-menu`) THEN CLICK `.logout` ELSE CLICK `.login`
|
||||
IF (EXISTS `.loading`) THEN WAIT 5 ELSE CLICK `#continue`
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Exactly one command will be executed
|
||||
- Useful for handling different page states
|
||||
- Commands must be on same line
|
||||
|
||||
---
|
||||
|
||||
#### `IF (NOT EXISTS <selector>) THEN <command>`
|
||||
Execute command if element does not exist.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
IF (NOT EXISTS `<selector>`) THEN <command>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `selector` - CSS selector to check (string in backticks)
|
||||
- `command` - Command to execute if element doesn't exist
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
IF (NOT EXISTS `.logged-in`) THEN GO /login
|
||||
IF (NOT EXISTS `.results`) THEN CLICK `#search-button`
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Inverse of EXISTS condition
|
||||
- Useful for error handling
|
||||
- Can check for missing required elements
|
||||
|
||||
---
|
||||
|
||||
#### `IF (<javascript>) THEN <command>`
|
||||
Execute command based on JavaScript condition.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
IF (`<javascript>`) THEN <command>
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `javascript` - JavaScript expression that returns boolean (string in backticks)
|
||||
- `command` - Command to execute if condition is true
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
IF (`window.innerWidth < 768`) THEN CLICK `.mobile-menu`
|
||||
IF (`document.readyState === "complete"`) THEN CLICK `#start`
|
||||
IF (`localStorage.getItem("user")`) THEN GO /dashboard
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- JavaScript executes in browser context
|
||||
- Must return boolean value
|
||||
- Access to all browser APIs and globals
|
||||
|
||||
---
|
||||
|
||||
#### `REPEAT (<command>, <count>)`
|
||||
Repeat a command a specific number of times.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
REPEAT (<command>, <count>)
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `command` - Command to repeat
|
||||
- `count` - Number of times to repeat (number)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
REPEAT (SCROLL DOWN 300, 5)
|
||||
REPEAT (PRESS Tab, 3)
|
||||
REPEAT (CLICK `.load-more`, 10)
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Executes command exactly count times
|
||||
- Useful for pagination, scrolling, navigation
|
||||
- No delay between repetitions (add WAIT if needed)
|
||||
|
||||
---
|
||||
|
||||
#### `REPEAT (<command>, <condition>)`
|
||||
Repeat a command while condition is true.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
REPEAT (<command>, `<condition>`)
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `command` - Command to repeat
|
||||
- `condition` - JavaScript condition to check (string in backticks)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
REPEAT (SCROLL DOWN 500, `document.querySelector(".load-more")`)
|
||||
REPEAT (PRESS ArrowDown, `window.scrollY < document.body.scrollHeight`)
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Condition checked before each iteration
|
||||
- JavaScript condition must return boolean
|
||||
- Be careful to avoid infinite loops
|
||||
|
||||
### 💾 Variables and Data
|
||||
|
||||
Store and manipulate data within scripts.
|
||||
|
||||
#### `SETVAR <name> = "<value>"`
|
||||
Create or update a variable.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
SETVAR <name> = "<value>"
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `name` - Variable name (alphanumeric, underscore)
|
||||
- `value` - Variable value (string in quotes)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
SETVAR username = "john@example.com"
|
||||
SETVAR password = "secret123"
|
||||
SETVAR base_url = "https://api.example.com"
|
||||
SETVAR counter = "0"
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Variables are global within script scope
|
||||
- Values are always strings
|
||||
- Can be used with TYPE command using $variable syntax
|
||||
|
||||
---
|
||||
|
||||
#### `EVAL <javascript>`
|
||||
Execute arbitrary JavaScript code.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
EVAL `<javascript>`
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `javascript` - JavaScript code to execute (string in backticks)
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
EVAL `console.log("Script started")`
|
||||
EVAL `window.scrollTo(0, 0)`
|
||||
EVAL `localStorage.setItem("test", "value")`
|
||||
EVAL `document.title = "Automated Test"`
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Full access to browser JavaScript APIs
|
||||
- Useful for custom logic and debugging
|
||||
- Return values are not captured
|
||||
- Be careful with security implications
|
||||
|
||||
### 📝 Comments and Documentation
|
||||
|
||||
#### `# <comment>`
|
||||
Add comments to scripts for documentation.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
# <comment text>
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
# This script logs into the application
|
||||
# Step 1: Navigate to login page
|
||||
GO /login
|
||||
|
||||
# Step 2: Fill credentials
|
||||
TYPE "user@example.com"
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Comments are ignored during execution
|
||||
- Useful for documentation and debugging
|
||||
- Can appear anywhere in script
|
||||
- Supports multi-line documentation blocks
|
||||
|
||||
### 🔧 Procedures (Advanced)
|
||||
|
||||
Define reusable command sequences.
|
||||
|
||||
#### `PROC <name> ... ENDPROC`
|
||||
Define a reusable procedure.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
PROC <name>
|
||||
<commands>
|
||||
ENDPROC
|
||||
```
|
||||
|
||||
**Parameters:**
|
||||
- `name` - Procedure name (alphanumeric, underscore)
|
||||
- `commands` - Commands to include in procedure
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
PROC login
|
||||
CLICK `#email`
|
||||
TYPE $email
|
||||
CLICK `#password`
|
||||
TYPE $password
|
||||
CLICK `#submit`
|
||||
ENDPROC
|
||||
|
||||
PROC handle_popups
|
||||
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
|
||||
IF (EXISTS `.newsletter-modal`) THEN CLICK `.close`
|
||||
ENDPROC
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Procedures must be defined before use
|
||||
- Support nested command structures
|
||||
- Variables are shared with main script scope
|
||||
|
||||
---
|
||||
|
||||
#### `<procedure_name>`
|
||||
Call a defined procedure.
|
||||
|
||||
**Syntax:**
|
||||
```c4a
|
||||
<procedure_name>
|
||||
```
|
||||
|
||||
**Examples:**
|
||||
```c4a
|
||||
# Define procedure first
|
||||
PROC setup
|
||||
GO /login
|
||||
WAIT `#form` 5
|
||||
ENDPROC
|
||||
|
||||
# Call procedure
|
||||
setup
|
||||
login
|
||||
```
|
||||
|
||||
**Notes:**
|
||||
- Procedure must be defined before calling
|
||||
- Can be called multiple times
|
||||
- No parameters supported (use variables instead)
|
||||
|
||||
## Error Handling Best Practices
|
||||
|
||||
### 1. Always Use Waits
|
||||
```c4a
|
||||
# Bad - element might not be ready
|
||||
CLICK `#button`
|
||||
|
||||
# Good - wait for element first
|
||||
WAIT `#button` 5
|
||||
CLICK `#button`
|
||||
```
|
||||
|
||||
### 2. Handle Optional Elements
|
||||
```c4a
|
||||
# Check before interacting
|
||||
IF (EXISTS `.popup`) THEN CLICK `.close`
|
||||
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept`
|
||||
|
||||
# Then proceed with main flow
|
||||
CLICK `#main-action`
|
||||
```
|
||||
|
||||
### 3. Use Descriptive Variables
|
||||
```c4a
|
||||
# Set up reusable data
|
||||
SETVAR admin_email = "admin@company.com"
|
||||
SETVAR test_password = "TestPass123!"
|
||||
SETVAR staging_url = "https://staging.example.com"
|
||||
|
||||
# Use throughout script
|
||||
GO $staging_url
|
||||
TYPE $admin_email
|
||||
```
|
||||
|
||||
### 4. Add Debugging Information
|
||||
```c4a
|
||||
# Log progress
|
||||
EVAL `console.log("Starting login process")`
|
||||
GO /login
|
||||
|
||||
# Verify page state
|
||||
IF (`document.title.includes("Login")`) THEN EVAL `console.log("On login page")`
|
||||
|
||||
# Continue with login
|
||||
TYPE $username
|
||||
```
|
||||
|
||||
## Common Patterns
|
||||
|
||||
### Login Flow
|
||||
```c4a
|
||||
# Complete login automation
|
||||
SETVAR email = "user@example.com"
|
||||
SETVAR password = "mypassword"
|
||||
|
||||
GO /login
|
||||
WAIT `#login-form` 5
|
||||
|
||||
# Handle optional cookie banner
|
||||
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept-cookies`
|
||||
|
||||
# Fill and submit form
|
||||
CLICK `#email`
|
||||
TYPE $email
|
||||
PRESS Tab
|
||||
TYPE $password
|
||||
CLICK `button[type="submit"]`
|
||||
|
||||
# Wait for redirect
|
||||
WAIT `.dashboard` 10
|
||||
```
|
||||
|
||||
### Infinite Scroll
|
||||
```c4a
|
||||
# Load all content with infinite scroll
|
||||
GO /products
|
||||
|
||||
# Scroll and load more content
|
||||
REPEAT (SCROLL DOWN 500, `document.querySelector(".load-more")`)
|
||||
|
||||
# Alternative: Fixed number of scrolls
|
||||
REPEAT (SCROLL DOWN 800, 10)
|
||||
WAIT 2
|
||||
```
|
||||
|
||||
### Form Validation
|
||||
```c4a
|
||||
# Handle form with validation
|
||||
SET `#email` "invalid-email"
|
||||
CLICK `#submit`
|
||||
|
||||
# Check for validation error
|
||||
IF (EXISTS `.error-email`) THEN SET `#email` "valid@example.com"
|
||||
|
||||
# Retry submission
|
||||
CLICK `#submit`
|
||||
WAIT `.success-message` 5
|
||||
```
|
||||
|
||||
### Multi-step Process
|
||||
```c4a
|
||||
# Complex multi-step workflow
|
||||
PROC navigate_to_step
|
||||
CLICK `.next-button`
|
||||
WAIT `.step-content` 5
|
||||
ENDPROC
|
||||
|
||||
# Step 1
|
||||
WAIT `.step-1` 5
|
||||
SET `#name` "John Doe"
|
||||
navigate_to_step
|
||||
|
||||
# Step 2
|
||||
SET `#email` "john@example.com"
|
||||
navigate_to_step
|
||||
|
||||
# Step 3
|
||||
CLICK `#submit-final`
|
||||
WAIT `.confirmation` 10
|
||||
```
|
||||
|
||||
## Integration with Crawl4AI
|
||||
|
||||
Use C4A-Script with Crawl4AI for dynamic content interaction:
|
||||
|
||||
```python
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
|
||||
# Define interaction script
|
||||
script = """
|
||||
# Handle dynamic content loading
|
||||
WAIT `.content` 5
|
||||
IF (EXISTS `.load-more-button`) THEN CLICK `.load-more-button`
|
||||
WAIT `.additional-content` 5
|
||||
|
||||
# Accept cookies if needed
|
||||
IF (EXISTS `.cookie-banner`) THEN CLICK `.accept-all`
|
||||
"""
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
c4a_script=script,
|
||||
wait_for=".content",
|
||||
screenshot=True
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
result = await crawler.arun("https://example.com", config=config)
|
||||
print(result.markdown)
|
||||
```
|
||||
|
||||
This reference covers all available C4A-Script commands and patterns. For interactive learning, try the [tutorial](../examples/c4a_script/tutorial/) or [live demo](https://docs.crawl4ai.com/c4a-script/demo).
|
||||
@@ -0,0 +1,431 @@
|
||||
# `CrawlResult` Reference
|
||||
|
||||
The **`CrawlResult`** class encapsulates everything returned after a single crawl operation. It provides the **raw or processed content**, details on links and media, plus optional metadata (like screenshots, PDFs, or extracted JSON).
|
||||
|
||||
**Location**: `crawl4ai/crawler/models.py` (for reference)
|
||||
|
||||
```python
|
||||
class CrawlResult(BaseModel):
|
||||
url: str
|
||||
html: str
|
||||
success: bool
|
||||
cleaned_html: Optional[str] = None
|
||||
fit_html: Optional[str] = None # Preprocessed HTML optimized for extraction
|
||||
media: Dict[str, List[Dict]] = {}
|
||||
links: Dict[str, List[Dict]] = {}
|
||||
downloaded_files: Optional[List[str]] = None
|
||||
screenshot: Optional[str] = None
|
||||
pdf : Optional[bytes] = None
|
||||
mhtml: Optional[str] = None
|
||||
markdown: Optional[Union[str, MarkdownGenerationResult]] = None
|
||||
extracted_content: Optional[str] = None
|
||||
metadata: Optional[dict] = None
|
||||
error_message: Optional[str] = None
|
||||
session_id: Optional[str] = None
|
||||
response_headers: Optional[dict] = None
|
||||
status_code: Optional[int] = None
|
||||
redirected_status_code: Optional[int] = None
|
||||
ssl_certificate: Optional[SSLCertificate] = None
|
||||
dispatch_result: Optional[DispatchResult] = None
|
||||
...
|
||||
```
|
||||
|
||||
Below is a **field-by-field** explanation and possible usage patterns.
|
||||
|
||||
---
|
||||
|
||||
## 1. Basic Crawl Info
|
||||
|
||||
### 1.1 **`url`** *(str)*
|
||||
**What**: The final crawled URL (after any redirects).
|
||||
**Usage**:
|
||||
```python
|
||||
print(result.url) # e.g., "https://example.com/"
|
||||
```
|
||||
|
||||
### 1.2 **`success`** *(bool)*
|
||||
**What**: `True` if the crawl pipeline ended without major errors; `False` otherwise.
|
||||
**Usage**:
|
||||
```python
|
||||
if not result.success:
|
||||
print(f"Crawl failed: {result.error_message}")
|
||||
```
|
||||
|
||||
### 1.3 **`status_code`** *(Optional[int])*
|
||||
**What**: The page's HTTP status code (e.g., 200, 404). When the page was reached via redirect, this is the status code of the **first** response in the redirect chain (e.g., 301 or 302).
|
||||
**Usage**:
|
||||
```python
|
||||
if result.status_code == 404:
|
||||
print("Page not found!")
|
||||
```
|
||||
|
||||
### 1.4 **`redirected_status_code`** *(Optional[int])*
|
||||
**What**: The HTTP status code of the **final** redirect destination. For a 302→200 redirect, `status_code` is 302 and `redirected_status_code` is 200. `None` for non-HTTP requests (raw HTML, local files).
|
||||
**Usage**:
|
||||
```python
|
||||
if result.status_code in (301, 302) and result.redirected_status_code == 200:
|
||||
print(f"Redirected to {result.redirected_url} (OK)")
|
||||
```
|
||||
|
||||
### 1.5 **`error_message`** *(Optional[str])*
|
||||
**What**: If `success=False`, a textual description of the failure.
|
||||
**Usage**:
|
||||
```python
|
||||
if not result.success:
|
||||
print("Error:", result.error_message)
|
||||
```
|
||||
|
||||
### 1.5 **`session_id`** *(Optional[str])*
|
||||
**What**: The ID used for reusing a browser context across multiple calls.
|
||||
**Usage**:
|
||||
```python
|
||||
# If you used session_id="login_session" in CrawlerRunConfig, see it here:
|
||||
print("Session:", result.session_id)
|
||||
```
|
||||
|
||||
### 1.6 **`response_headers`** *(Optional[dict])*
|
||||
**What**: Final HTTP response headers.
|
||||
**Usage**:
|
||||
```python
|
||||
if result.response_headers:
|
||||
print("Server:", result.response_headers.get("Server", "Unknown"))
|
||||
```
|
||||
|
||||
### 1.7 **`ssl_certificate`** *(Optional[SSLCertificate])*
|
||||
**What**: If `fetch_ssl_certificate=True` in your CrawlerRunConfig, **`result.ssl_certificate`** contains a [**`SSLCertificate`**](../advanced/ssl-certificate.md) object describing the site's certificate. You can export the cert in multiple formats (PEM/DER/JSON) or access its properties like `issuer`,
|
||||
`subject`, `valid_from`, `valid_until`, etc.
|
||||
**Usage**:
|
||||
```python
|
||||
if result.ssl_certificate:
|
||||
print("Issuer:", result.ssl_certificate.issuer)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. Raw / Cleaned Content
|
||||
|
||||
### 2.1 **`html`** *(str)*
|
||||
**What**: The **original** unmodified HTML from the final page load.
|
||||
**Usage**:
|
||||
```python
|
||||
# Possibly large
|
||||
print(len(result.html))
|
||||
```
|
||||
|
||||
### 2.2 **`cleaned_html`** *(Optional[str])*
|
||||
**What**: A sanitized HTML version—scripts, styles, or excluded tags are removed based on your `CrawlerRunConfig`.
|
||||
**Usage**:
|
||||
```python
|
||||
print(result.cleaned_html[:500]) # Show a snippet
|
||||
```
|
||||
|
||||
|
||||
---
|
||||
|
||||
## 3. Markdown Fields
|
||||
|
||||
### 3.1 The Markdown Generation Approach
|
||||
|
||||
Crawl4AI can convert HTML→Markdown, optionally including:
|
||||
|
||||
- **Raw** markdown
|
||||
- **Links as citations** (with a references section)
|
||||
- **Fit** markdown if a **content filter** is used (like Pruning or BM25)
|
||||
|
||||
|
||||
**`MarkdownGenerationResult`** includes:
|
||||
- **`raw_markdown`** *(str)*: The full HTML→Markdown conversion.
|
||||
- **`markdown_with_citations`** *(str)*: Same markdown, but with link references as academic-style citations.
|
||||
- **`references_markdown`** *(str)*: The reference list or footnotes at the end.
|
||||
- **`fit_markdown`** *(Optional[str])*: If content filtering (Pruning/BM25) was applied, the filtered "fit" text.
|
||||
- **`fit_html`** *(Optional[str])*: The HTML that led to `fit_markdown`.
|
||||
|
||||
**Usage**:
|
||||
```python
|
||||
if result.markdown:
|
||||
md_res = result.markdown
|
||||
print("Raw MD:", md_res.raw_markdown[:300])
|
||||
print("Citations MD:", md_res.markdown_with_citations[:300])
|
||||
print("References:", md_res.references_markdown)
|
||||
if md_res.fit_markdown:
|
||||
print("Pruned text:", md_res.fit_markdown[:300])
|
||||
```
|
||||
|
||||
### 3.2 **`markdown`** *(Optional[Union[str, MarkdownGenerationResult]])*
|
||||
**What**: Holds the `MarkdownGenerationResult`.
|
||||
**Usage**:
|
||||
```python
|
||||
print(result.markdown.raw_markdown[:200])
|
||||
print(result.markdown.fit_markdown)
|
||||
print(result.markdown.fit_html)
|
||||
```
|
||||
**Important**: "Fit" content (in `fit_markdown`/`fit_html`) exists in result.markdown, only if you used a **filter** (like **PruningContentFilter** or **BM25ContentFilter**) within a `MarkdownGenerationStrategy`.
|
||||
|
||||
---
|
||||
|
||||
## 4. Media & Links
|
||||
|
||||
### 4.1 **`media`** *(Dict[str, List[Dict]])*
|
||||
**What**: Contains info about discovered images, videos, or audio. Typically keys: `"images"`, `"videos"`, `"audios"`.
|
||||
**Common Fields** in each item:
|
||||
|
||||
- `src` *(str)*: Media URL
|
||||
- `alt` or `title` *(str)*: Descriptive text
|
||||
- `score` *(float)*: Relevance score if the crawler's heuristic found it "important"
|
||||
- `desc` or `description` *(Optional[str])*: Additional context extracted from surrounding text
|
||||
|
||||
**Usage**:
|
||||
```python
|
||||
images = result.media.get("images", [])
|
||||
for img in images:
|
||||
if img.get("score", 0) > 5:
|
||||
print("High-value image:", img["src"])
|
||||
```
|
||||
|
||||
### 4.2 **`links`** *(Dict[str, List[Dict]])*
|
||||
**What**: Holds internal and external link data. Usually two keys: `"internal"` and `"external"`.
|
||||
**Common Fields**:
|
||||
|
||||
- `href` *(str)*: The link target
|
||||
- `text` *(str)*: Link text
|
||||
- `title` *(str)*: Title attribute
|
||||
- `context` *(str)*: Surrounding text snippet
|
||||
- `domain` *(str)*: If external, the domain
|
||||
|
||||
**Usage**:
|
||||
```python
|
||||
for link in result.links["internal"]:
|
||||
print(f"Internal link to {link['href']} with text {link['text']}")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5. Additional Fields
|
||||
|
||||
### 5.1 **`extracted_content`** *(Optional[str])*
|
||||
**What**: If you used **`extraction_strategy`** (CSS, LLM, etc.), the structured output (JSON).
|
||||
**Usage**:
|
||||
```python
|
||||
if result.extracted_content:
|
||||
data = json.loads(result.extracted_content)
|
||||
print(data)
|
||||
```
|
||||
|
||||
### 5.2 **`downloaded_files`** *(Optional[List[str]])*
|
||||
**What**: If `accept_downloads=True` in your `BrowserConfig` + `downloads_path`, lists local file paths for downloaded items.
|
||||
**Usage**:
|
||||
```python
|
||||
if result.downloaded_files:
|
||||
for file_path in result.downloaded_files:
|
||||
print("Downloaded:", file_path)
|
||||
```
|
||||
|
||||
### 5.3 **`screenshot`** *(Optional[str])*
|
||||
**What**: Base64-encoded screenshot if `screenshot=True` in `CrawlerRunConfig`.
|
||||
**Usage**:
|
||||
```python
|
||||
import base64
|
||||
if result.screenshot:
|
||||
with open("page.png", "wb") as f:
|
||||
f.write(base64.b64decode(result.screenshot))
|
||||
```
|
||||
|
||||
### 5.4 **`pdf`** *(Optional[bytes])*
|
||||
**What**: Raw PDF bytes if `pdf=True` in `CrawlerRunConfig`.
|
||||
**Usage**:
|
||||
```python
|
||||
if result.pdf:
|
||||
with open("page.pdf", "wb") as f:
|
||||
f.write(result.pdf)
|
||||
```
|
||||
|
||||
### 5.5 **`mhtml`** *(Optional[str])*
|
||||
**What**: MHTML snapshot of the page if `capture_mhtml=True` in `CrawlerRunConfig`. MHTML (MIME HTML) format preserves the entire web page with all its resources (CSS, images, scripts, etc.) in a single file.
|
||||
**Usage**:
|
||||
```python
|
||||
if result.mhtml:
|
||||
with open("page.mhtml", "w", encoding="utf-8") as f:
|
||||
f.write(result.mhtml)
|
||||
```
|
||||
|
||||
### 5.6 **`metadata`** *(Optional[dict])*
|
||||
**What**: Page-level metadata if discovered (title, description, OG data, etc.).
|
||||
**Usage**:
|
||||
```python
|
||||
if result.metadata:
|
||||
print("Title:", result.metadata.get("title"))
|
||||
print("Author:", result.metadata.get("author"))
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. `dispatch_result` (optional)
|
||||
|
||||
A `DispatchResult` object providing additional concurrency and resource usage information when crawling URLs in parallel (e.g., via `arun_many()` with custom dispatchers). It contains:
|
||||
|
||||
- **`task_id`**: A unique identifier for the parallel task.
|
||||
- **`memory_usage`** (float): The memory (in MB) used at the time of completion.
|
||||
- **`peak_memory`** (float): The peak memory usage (in MB) recorded during the task's execution.
|
||||
- **`start_time`** / **`end_time`** (datetime): Time range for this crawling task.
|
||||
- **`error_message`** (str): Any dispatcher- or concurrency-related error encountered.
|
||||
|
||||
```python
|
||||
# Example usage:
|
||||
for result in results:
|
||||
if result.success and result.dispatch_result:
|
||||
dr = result.dispatch_result
|
||||
print(f"URL: {result.url}, Task ID: {dr.task_id}")
|
||||
print(f"Memory: {dr.memory_usage:.1f} MB (Peak: {dr.peak_memory:.1f} MB)")
|
||||
print(f"Duration: {dr.end_time - dr.start_time}")
|
||||
```
|
||||
|
||||
> **Note**: This field is typically populated when using `arun_many(...)` alongside a **dispatcher** (e.g., `MemoryAdaptiveDispatcher` or `SemaphoreDispatcher`). If no concurrency or dispatcher is used, `dispatch_result` may remain `None`.
|
||||
|
||||
---
|
||||
|
||||
## 7. Network Requests & Console Messages
|
||||
|
||||
When you enable network and console message capturing in `CrawlerRunConfig` using `capture_network_requests=True` and `capture_console_messages=True`, the `CrawlResult` will include these fields:
|
||||
|
||||
### 7.1 **`network_requests`** *(Optional[List[Dict[str, Any]]])*
|
||||
**What**: A list of dictionaries containing information about all network requests, responses, and failures captured during the crawl.
|
||||
**Structure**:
|
||||
- Each item has an `event_type` field that can be `"request"`, `"response"`, or `"request_failed"`.
|
||||
- Request events include `url`, `method`, `headers`, `post_data`, `resource_type`, and `is_navigation_request`.
|
||||
- Response events include `url`, `status`, `status_text`, `headers`, and `request_timing`.
|
||||
- Failed request events include `url`, `method`, `resource_type`, and `failure_text`.
|
||||
- All events include a `timestamp` field.
|
||||
|
||||
**Usage**:
|
||||
```python
|
||||
if result.network_requests:
|
||||
# Count different types of events
|
||||
requests = [r for r in result.network_requests if r.get("event_type") == "request"]
|
||||
responses = [r for r in result.network_requests if r.get("event_type") == "response"]
|
||||
failures = [r for r in result.network_requests if r.get("event_type") == "request_failed"]
|
||||
|
||||
print(f"Captured {len(requests)} requests, {len(responses)} responses, and {len(failures)} failures")
|
||||
|
||||
# Analyze API calls
|
||||
api_calls = [r for r in requests if "api" in r.get("url", "")]
|
||||
|
||||
# Identify failed resources
|
||||
for failure in failures:
|
||||
print(f"Failed to load: {failure.get('url')} - {failure.get('failure_text')}")
|
||||
```
|
||||
|
||||
### 7.2 **`console_messages`** *(Optional[List[Dict[str, Any]]])*
|
||||
**What**: A list of dictionaries containing all browser console messages captured during the crawl.
|
||||
**Structure**:
|
||||
- Each item has a `type` field indicating the message type (e.g., `"log"`, `"error"`, `"warning"`, etc.).
|
||||
- The `text` field contains the actual message text.
|
||||
- Some messages include `location` information (URL, line, column).
|
||||
- All messages include a `timestamp` field.
|
||||
|
||||
**Usage**:
|
||||
```python
|
||||
if result.console_messages:
|
||||
# Count messages by type
|
||||
message_types = {}
|
||||
for msg in result.console_messages:
|
||||
msg_type = msg.get("type", "unknown")
|
||||
message_types[msg_type] = message_types.get(msg_type, 0) + 1
|
||||
|
||||
print(f"Message type counts: {message_types}")
|
||||
|
||||
# Display errors (which are usually most important)
|
||||
for msg in result.console_messages:
|
||||
if msg.get("type") == "error":
|
||||
print(f"Error: {msg.get('text')}")
|
||||
```
|
||||
|
||||
These fields provide deep visibility into the page's network activity and browser console, which is invaluable for debugging, security analysis, and understanding complex web applications.
|
||||
|
||||
For more details on network and console capturing, see the [Network & Console Capture documentation](../advanced/network-console-capture.md).
|
||||
|
||||
---
|
||||
|
||||
## 8. Example: Accessing Everything
|
||||
|
||||
```python
|
||||
async def handle_result(result: CrawlResult):
|
||||
if not result.success:
|
||||
print("Crawl error:", result.error_message)
|
||||
return
|
||||
|
||||
# Basic info
|
||||
print("Crawled URL:", result.url)
|
||||
print("Status code:", result.status_code)
|
||||
|
||||
# HTML
|
||||
print("Original HTML size:", len(result.html))
|
||||
print("Cleaned HTML size:", len(result.cleaned_html or ""))
|
||||
|
||||
# Markdown output
|
||||
if result.markdown:
|
||||
print("Raw Markdown:", result.markdown.raw_markdown[:300])
|
||||
print("Citations Markdown:", result.markdown.markdown_with_citations[:300])
|
||||
if result.markdown.fit_markdown:
|
||||
print("Fit Markdown:", result.markdown.fit_markdown[:200])
|
||||
|
||||
# Media & Links
|
||||
if "images" in result.media:
|
||||
print("Image count:", len(result.media["images"]))
|
||||
if "internal" in result.links:
|
||||
print("Internal link count:", len(result.links["internal"]))
|
||||
|
||||
# Extraction strategy result
|
||||
if result.extracted_content:
|
||||
print("Structured data:", result.extracted_content)
|
||||
|
||||
# Screenshot/PDF/MHTML
|
||||
if result.screenshot:
|
||||
print("Screenshot length:", len(result.screenshot))
|
||||
if result.pdf:
|
||||
print("PDF bytes length:", len(result.pdf))
|
||||
if result.mhtml:
|
||||
print("MHTML length:", len(result.mhtml))
|
||||
|
||||
# Network and console capturing
|
||||
if result.network_requests:
|
||||
print(f"Network requests captured: {len(result.network_requests)}")
|
||||
# Analyze request types
|
||||
req_types = {}
|
||||
for req in result.network_requests:
|
||||
if "resource_type" in req:
|
||||
req_types[req["resource_type"]] = req_types.get(req["resource_type"], 0) + 1
|
||||
print(f"Resource types: {req_types}")
|
||||
|
||||
if result.console_messages:
|
||||
print(f"Console messages captured: {len(result.console_messages)}")
|
||||
# Count by message type
|
||||
msg_types = {}
|
||||
for msg in result.console_messages:
|
||||
msg_types[msg.get("type", "unknown")] = msg_types.get(msg.get("type", "unknown"), 0) + 1
|
||||
print(f"Message types: {msg_types}")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. Key Points & Future
|
||||
|
||||
1. **Deprecated legacy properties of CrawlResult**
|
||||
- `markdown_v2` - Removed in v0.5 and now raises `AttributeError`. Use `result.markdown` instead.
|
||||
- `fit_markdown` and `fit_html` - No longer top-level properties in v0.5. Use `result.markdown.fit_markdown` and `result.markdown.fit_html`.
|
||||
|
||||
2. **Fit Content**
|
||||
- **`fit_markdown`** and **`fit_html`** appear in MarkdownGenerationResult, only if you used a content filter (like **PruningContentFilter** or **BM25ContentFilter**) inside your **MarkdownGenerationStrategy** or set them directly.
|
||||
- If no filter is used, they remain `None`.
|
||||
|
||||
3. **References & Citations**
|
||||
- If you enable link citations in your `DefaultMarkdownGenerator` (`options={"citations": True}`), you’ll see `markdown_with_citations` plus a **`references_markdown`** block. This helps large language models or academic-like referencing.
|
||||
|
||||
4. **Links & Media**
|
||||
- `links["internal"]` and `links["external"]` group discovered anchors by domain.
|
||||
- `media["images"]` / `["videos"]` / `["audios"]` store extracted media elements with optional scoring or context.
|
||||
|
||||
5. **Error Cases**
|
||||
- If `success=False`, check `error_message` (e.g., timeouts, invalid URLs).
|
||||
- `status_code` might be `None` if we failed before an HTTP response.
|
||||
|
||||
Use **`CrawlResult`** to glean all final outputs and feed them into your data pipelines, AI models, or archives. With the synergy of a properly configured **BrowserConfig** and **CrawlerRunConfig**, the crawler can produce robust, structured results here in **`CrawlResult`**.
|
||||
@@ -0,0 +1,181 @@
|
||||
# digest()
|
||||
|
||||
The `digest()` method is the primary interface for adaptive web crawling. It intelligently crawls websites starting from a given URL, guided by a query, and automatically determines when sufficient information has been gathered.
|
||||
|
||||
## Method Signature
|
||||
|
||||
```python
|
||||
async def digest(
|
||||
start_url: str,
|
||||
query: str,
|
||||
resume_from: Optional[Union[str, Path]] = None
|
||||
) -> CrawlState
|
||||
```
|
||||
|
||||
## Parameters
|
||||
|
||||
### start_url
|
||||
- **Type**: `str`
|
||||
- **Required**: Yes
|
||||
- **Description**: The starting URL for the crawl. This should be a valid HTTP/HTTPS URL that serves as the entry point for information gathering.
|
||||
|
||||
### query
|
||||
- **Type**: `str`
|
||||
- **Required**: Yes
|
||||
- **Description**: The search query that guides the crawling process. This should contain key terms related to the information you're seeking. The crawler uses this to evaluate relevance and determine which links to follow.
|
||||
|
||||
### resume_from
|
||||
- **Type**: `Optional[Union[str, Path]]`
|
||||
- **Default**: `None`
|
||||
- **Description**: Path to a previously saved crawl state file. When provided, the crawler resumes from the saved state instead of starting fresh.
|
||||
|
||||
## Return Value
|
||||
|
||||
Returns a `CrawlState` object containing:
|
||||
|
||||
- **crawled_urls** (`Set[str]`): All URLs that have been crawled
|
||||
- **knowledge_base** (`List[CrawlResult]`): Collection of crawled pages with content
|
||||
- **pending_links** (`List[Link]`): Links discovered but not yet crawled
|
||||
- **metrics** (`Dict[str, float]`): Performance and quality metrics
|
||||
- **query** (`str`): The original query
|
||||
- Additional statistical information for scoring
|
||||
|
||||
## How It Works
|
||||
|
||||
The `digest()` method implements an intelligent crawling algorithm:
|
||||
|
||||
1. **Initial Crawl**: Starts from the provided URL
|
||||
2. **Link Analysis**: Evaluates all discovered links for relevance
|
||||
3. **Scoring**: Uses three metrics to assess information sufficiency:
|
||||
- **Coverage**: How well the query terms are covered
|
||||
- **Consistency**: Information coherence across pages
|
||||
- **Saturation**: Diminishing returns detection
|
||||
4. **Adaptive Selection**: Chooses the most promising links to follow
|
||||
5. **Stopping Decision**: Automatically stops when confidence threshold is reached
|
||||
|
||||
## Examples
|
||||
|
||||
### Basic Usage
|
||||
|
||||
```python
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
adaptive = AdaptiveCrawler(crawler)
|
||||
|
||||
state = await adaptive.digest(
|
||||
start_url="https://docs.python.org/3/",
|
||||
query="async await context managers"
|
||||
)
|
||||
|
||||
print(f"Crawled {len(state.crawled_urls)} pages")
|
||||
print(f"Confidence: {adaptive.confidence:.0%}")
|
||||
```
|
||||
|
||||
### With Configuration
|
||||
|
||||
```python
|
||||
config = AdaptiveConfig(
|
||||
confidence_threshold=0.9, # Require high confidence
|
||||
max_pages=30, # Allow more pages
|
||||
top_k_links=3 # Follow top 3 links per page
|
||||
)
|
||||
|
||||
adaptive = AdaptiveCrawler(crawler, config=config)
|
||||
|
||||
state = await adaptive.digest(
|
||||
start_url="https://api.example.com/docs",
|
||||
query="authentication endpoints rate limits"
|
||||
)
|
||||
```
|
||||
|
||||
### Resuming a Previous Crawl
|
||||
|
||||
```python
|
||||
# First crawl - may be interrupted
|
||||
state1 = await adaptive.digest(
|
||||
start_url="https://example.com",
|
||||
query="machine learning algorithms"
|
||||
)
|
||||
|
||||
# Save state (if not auto-saved)
|
||||
state1.save("ml_crawl_state.json")
|
||||
|
||||
# Later, resume from saved state
|
||||
state2 = await adaptive.digest(
|
||||
start_url="https://example.com",
|
||||
query="machine learning algorithms",
|
||||
resume_from="ml_crawl_state.json"
|
||||
)
|
||||
```
|
||||
|
||||
### With Progress Monitoring
|
||||
|
||||
```python
|
||||
state = await adaptive.digest(
|
||||
start_url="https://docs.example.com",
|
||||
query="api reference"
|
||||
)
|
||||
|
||||
# Monitor progress
|
||||
print(f"Pages crawled: {len(state.crawled_urls)}")
|
||||
print(f"New terms discovered: {state.new_terms_history}")
|
||||
print(f"Final confidence: {adaptive.confidence:.2%}")
|
||||
|
||||
# View detailed statistics
|
||||
adaptive.print_stats(detailed=True)
|
||||
```
|
||||
|
||||
## Query Best Practices
|
||||
|
||||
1. **Be Specific**: Use descriptive terms that appear in target content
|
||||
```python
|
||||
# Good
|
||||
query = "python async context managers implementation"
|
||||
|
||||
# Too broad
|
||||
query = "python programming"
|
||||
```
|
||||
|
||||
2. **Include Key Terms**: Add technical terms you expect to find
|
||||
```python
|
||||
query = "oauth2 jwt refresh tokens authorization"
|
||||
```
|
||||
|
||||
3. **Multiple Concepts**: Combine related concepts for comprehensive coverage
|
||||
```python
|
||||
query = "rest api pagination sorting filtering"
|
||||
```
|
||||
|
||||
## Performance Considerations
|
||||
|
||||
- **Initial URL**: Choose a page with good navigation (e.g., documentation index)
|
||||
- **Query Length**: 3-8 terms typically work best
|
||||
- **Link Density**: Sites with clear navigation crawl more efficiently
|
||||
- **Caching**: Enable caching for repeated crawls of the same domain
|
||||
|
||||
## Error Handling
|
||||
|
||||
```python
|
||||
try:
|
||||
state = await adaptive.digest(
|
||||
start_url="https://example.com",
|
||||
query="search terms"
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Crawl failed: {e}")
|
||||
# State is auto-saved if save_state=True in config
|
||||
```
|
||||
|
||||
## Stopping Conditions
|
||||
|
||||
The crawl stops when any of these conditions are met:
|
||||
|
||||
1. **Confidence Threshold**: Reached the configured confidence level
|
||||
2. **Page Limit**: Crawled the maximum number of pages
|
||||
3. **Diminishing Returns**: Expected information gain below threshold
|
||||
4. **No Relevant Links**: No promising links remain to follow
|
||||
|
||||
## See Also
|
||||
|
||||
- [AdaptiveCrawler Class](adaptive-crawler.md)
|
||||
- [Adaptive Crawling Guide](../core/adaptive-crawling.md)
|
||||
- [Configuration Options](../core/adaptive-crawling.md#configuration-options)
|
||||
@@ -0,0 +1,534 @@
|
||||
# 1. **BrowserConfig** – Controlling the Browser
|
||||
|
||||
`BrowserConfig` focuses on **how** the browser is launched and behaves. This includes headless mode, proxies, user agents, and other environment tweaks.
|
||||
|
||||
```python
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig
|
||||
|
||||
browser_cfg = BrowserConfig(
|
||||
browser_type="chromium",
|
||||
headless=True,
|
||||
viewport_width=1280,
|
||||
viewport_height=720,
|
||||
proxy_config="http://user:pass@proxy:8080",
|
||||
user_agent="Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/116.0.0.0 Safari/537.36",
|
||||
)
|
||||
```
|
||||
|
||||
## 1.1 Parameter Highlights
|
||||
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|-----------------------|----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| **`browser_type`** | `"chromium"`, `"firefox"`, `"webkit"`<br/>*(default: `"chromium"`)* | Which browser engine to use. `"chromium"` is typical for many sites, `"firefox"` or `"webkit"` for specialized tests. |
|
||||
| **`headless`** | `bool` (default: `True`) | Headless means no visible UI. `False` is handy for debugging. |
|
||||
| **`browser_mode`** | `str` (default: `"dedicated"`) | How browser is initialized: `"dedicated"` (new instance), `"builtin"` (CDP background), `"custom"` (explicit CDP), `"docker"` (container). |
|
||||
| **`use_managed_browser`** | `bool` (default: `False`) | Launch browser via CDP for advanced control. Set automatically based on `browser_mode`. |
|
||||
| **`cdp_url`** | `str` (default: `None`) | Chrome DevTools Protocol endpoint URL (e.g., `"ws://localhost:9222/devtools/browser/"`). Set automatically based on `browser_mode`. |
|
||||
| **`debugging_port`** | `int` (default: `9222`) | Port for browser debugging protocol. |
|
||||
| **`host`** | `str` (default: `"localhost"`) | Host for browser connection. |
|
||||
| **`viewport_width`** | `int` (default: `1080`) | Initial page width (in px). Useful for testing responsive layouts. |
|
||||
| **`viewport_height`** | `int` (default: `600`) | Initial page height (in px). |
|
||||
| **`viewport`** | `dict` (default: `None`) | Viewport dimensions dict. If set, overrides `viewport_width` and `viewport_height`. |
|
||||
| **`device_scale_factor`** | `float` (default: `1.0`) | Device pixel ratio for rendering. Use `2.0` for Retina-quality screenshots. Higher values produce larger images and use more memory. |
|
||||
| **`proxy`** | `str` (deprecated) | Deprecated. Use `proxy_config` instead. If set, it will be auto-converted internally. |
|
||||
| **`proxy_config`** | `ProxyConfig or dict` (default: `None`)| For advanced or multi-proxy needs, specify `ProxyConfig` object or dict like `{"server": "...", "username": "...", "password": "..."}`. |
|
||||
| **`use_persistent_context`** | `bool` (default: `False`) | If `True`, uses a **persistent** browser context (keep cookies, sessions across runs). Also sets `use_managed_browser=True`. |
|
||||
| **`user_data_dir`** | `str or None` (default: `None`) | Directory to store user data (profiles, cookies). Must be set if you want permanent sessions. |
|
||||
| **`chrome_channel`** | `str` (default: `"chromium"`) | Chrome channel to launch (e.g., "chrome", "msedge"). Only for `browser_type="chromium"`. Auto-set to empty for Firefox/WebKit. |
|
||||
| **`channel`** | `str` (default: `"chromium"`) | Alias for `chrome_channel`. |
|
||||
| **`accept_downloads`** | `bool` (default: `False`) | Whether to allow file downloads. Requires `downloads_path` if `True`. |
|
||||
| **`downloads_path`** | `str or None` (default: `None`) | Directory to store downloaded files. |
|
||||
| **`storage_state`** | `str or dict or None` (default: `None`)| In-memory storage state (cookies, localStorage) to restore browser state. |
|
||||
| **`ignore_https_errors`** | `bool` (default: `True`) | If `True`, continues despite invalid certificates (common in dev/staging). |
|
||||
| **`java_script_enabled`** | `bool` (default: `True`) | Disable if you want no JS overhead, or if only static content is needed. |
|
||||
| **`sleep_on_close`** | `bool` (default: `False`) | Add a small delay when closing browser (can help with cleanup issues). |
|
||||
| **`cookies`** | `list` (default: `[]`) | Pre-set cookies, each a dict like `{"name": "session", "value": "...", "url": "..."}`. |
|
||||
| **`headers`** | `dict` (default: `{}`) | Extra HTTP headers for every request, e.g. `{"Accept-Language": "en-US"}`. |
|
||||
| **`user_agent`** | `str` (default: Chrome-based UA) | Your custom user agent string. |
|
||||
| **`user_agent_mode`** | `str` (default: `""`) | Set to `"random"` to randomize user agent from a pool (helps with bot detection). |
|
||||
| **`user_agent_generator_config`** | `dict` (default: `{}`) | Configuration dict for user agent generation when `user_agent_mode="random"`. |
|
||||
| **`text_mode`** | `bool` (default: `False`) | If `True`, tries to disable images/other heavy content for speed. |
|
||||
| **`light_mode`** | `bool` (default: `False`) | Disables some background features for performance gains. |
|
||||
| **`avoid_ads`** | `bool` (default: `False`) | If `True`, blocks requests to common ad/tracker domains (Google Analytics, DoubleClick, Facebook, Hotjar, etc.) at the browser context level. |
|
||||
| **`avoid_css`** | `bool` (default: `False`) | If `True`, blocks loading of CSS files (`.css`, `.less`, `.scss`, `.sass`) for faster, leaner crawls when only text content is needed. |
|
||||
| **`extra_args`** | `list` (default: `[]`) | Additional flags for the underlying browser process, e.g. `["--disable-extensions"]`. |
|
||||
| **`enable_stealth`** | `bool` (default: `False`) | Enable playwright-stealth mode to bypass bot detection. Cannot be used with `browser_mode="builtin"`. |
|
||||
|
||||
**Tips**:
|
||||
- Set `headless=False` to visually **debug** how pages load or how interactions proceed.
|
||||
- If you need **authentication** storage or repeated sessions, consider `use_persistent_context=True` and specify `user_data_dir`.
|
||||
- For large pages, you might need a bigger `viewport_width` and `viewport_height` to handle dynamic content.
|
||||
|
||||
---
|
||||
|
||||
# 2. **CrawlerRunConfig** – Controlling Each Crawl
|
||||
|
||||
While `BrowserConfig` sets up the **environment**, `CrawlerRunConfig` details **how** each **crawl operation** should behave: caching, content filtering, link or domain blocking, timeouts, JavaScript code, etc.
|
||||
|
||||
```python
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig
|
||||
|
||||
run_cfg = CrawlerRunConfig(
|
||||
wait_for="css:.main-content",
|
||||
word_count_threshold=15,
|
||||
excluded_tags=["nav", "footer"],
|
||||
exclude_external_links=True,
|
||||
stream=True, # Enable streaming for arun_many()
|
||||
)
|
||||
```
|
||||
|
||||
## 2.1 Parameter Highlights
|
||||
|
||||
We group them by category.
|
||||
|
||||
### A) **Content Processing**
|
||||
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|------------------------------|--------------------------------------|-------------------------------------------------------------------------------------------------|
|
||||
| **`word_count_threshold`** | `int` (default: ~200) | Skips text blocks below X words. Helps ignore trivial sections. |
|
||||
| **`extraction_strategy`** | `ExtractionStrategy` (default: None) | If set, extracts structured data (CSS-based, LLM-based, etc.). |
|
||||
| **`chunking_strategy`** | `ChunkingStrategy` (default: RegexChunking()) | Strategy to chunk content before extraction. Can be customized for different chunking approaches. |
|
||||
| **`markdown_generator`** | `MarkdownGenerationStrategy` (None) | If you want specialized markdown output (citations, filtering, chunking, etc.). Can be customized with options such as `content_source` parameter to select the HTML input source ('cleaned_html', 'raw_html', or 'fit_html'). |
|
||||
| **`css_selector`** | `str` (None) | Retains only the part of the page matching this selector. Affects the entire extraction process. |
|
||||
| **`target_elements`** | `List[str]` (None) | List of CSS selectors for elements to focus on for markdown generation and data extraction, while still processing the entire page for links, media, etc. Provides more flexibility than `css_selector`. |
|
||||
| **`excluded_tags`** | `list` (None) | Removes entire tags (e.g. `["script", "style"]`). |
|
||||
| **`excluded_selector`** | `str` (None) | Like `css_selector` but to exclude. E.g. `"#ads, .tracker"`. |
|
||||
| **`only_text`** | `bool` (False) | If `True`, tries to extract text-only content. |
|
||||
| **`prettiify`** | `bool` (False) | If `True`, beautifies final HTML (slower, purely cosmetic). |
|
||||
| **`keep_data_attributes`** | `bool` (False) | If `True`, preserve `data-*` attributes in cleaned HTML. |
|
||||
| **`keep_attrs`** | `list` (default: []) | List of HTML attributes to keep during processing (e.g., `["id", "class", "data-value"]`). |
|
||||
| **`remove_forms`** | `bool` (False) | If `True`, remove all `<form>` elements. |
|
||||
| **`parser_type`** | `str` (default: "lxml") | HTML parser to use (e.g., "lxml", "html.parser"). |
|
||||
| **`scraping_strategy`** | `ContentScrapingStrategy` (default: LXMLWebScrapingStrategy()) | Strategy to use for content scraping. Can be customized for different scraping needs (e.g., PDF extraction). |
|
||||
|
||||
---
|
||||
|
||||
### B) **Browser Location and Identity**
|
||||
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|------------------------|---------------------------|--------------------------------------------------------------------------------------------------------|
|
||||
| **`locale`** | `str or None` (None) | Browser's locale (e.g., "en-US", "fr-FR") for language preferences. |
|
||||
| **`timezone_id`** | `str or None` (None) | Browser's timezone (e.g., "America/New_York", "Europe/Paris"). |
|
||||
| **`geolocation`** | `GeolocationConfig or None` (None) | GPS coordinates configuration. Use `GeolocationConfig(latitude=..., longitude=..., accuracy=...)`. |
|
||||
| **`fetch_ssl_certificate`** | `bool` (False) | If `True`, fetches and includes SSL certificate information in the result. |
|
||||
| **`proxy_config`** | `ProxyConfig`, `list[ProxyConfig]`, or `None` (None) | Proxy configuration for this specific crawl. Pass a single proxy or an ordered list of proxies to try. See [Anti-Bot & Fallback](../advanced/anti-bot-and-fallback.md). |
|
||||
| **`proxy_rotation_strategy`** | `ProxyRotationStrategy` (None) | Strategy for rotating proxies during crawl operations. |
|
||||
| **`max_retries`** | `int` (0) | Number of retry rounds when anti-bot blocking is detected. Each round tries all proxies in `proxy_config`. |
|
||||
| **`fallback_fetch_function`**| `async (str) -> str or None` (None) | Async function called as last resort after all retries are exhausted. Takes URL, returns raw HTML. See [Anti-Bot & Fallback](../advanced/anti-bot-and-fallback.md). |
|
||||
|
||||
---
|
||||
|
||||
### C) **Caching & Session**
|
||||
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|-------------------------|------------------------|------------------------------------------------------------------------------------------------------------------------------|
|
||||
| **`cache_mode`** | `CacheMode or None` | Controls how caching is handled (`ENABLED`, `BYPASS`, `DISABLED`, etc.). If `None`, typically defaults to `ENABLED`. |
|
||||
| **`session_id`** | `str or None` | Assign a unique ID to reuse a single browser session across multiple `arun()` calls. |
|
||||
| **`bypass_cache`** | `bool` (False) | **Deprecated.** If `True`, acts like `CacheMode.BYPASS`. Use `cache_mode` instead. |
|
||||
| **`disable_cache`** | `bool` (False) | **Deprecated.** If `True`, acts like `CacheMode.DISABLED`. Use `cache_mode` instead. |
|
||||
| **`no_cache_read`** | `bool` (False) | **Deprecated.** If `True`, acts like `CacheMode.WRITE_ONLY` (writes cache but never reads). Use `cache_mode` instead. |
|
||||
| **`no_cache_write`** | `bool` (False) | **Deprecated.** If `True`, acts like `CacheMode.READ_ONLY` (reads cache but never writes). Use `cache_mode` instead. |
|
||||
| **`shared_data`** | `dict or None` (None) | Shared data to be passed between hooks and accessible across crawl operations. |
|
||||
|
||||
Use these for controlling whether you read or write from a local content cache. Handy for large batch crawls or repeated site visits.
|
||||
|
||||
---
|
||||
|
||||
### D) **Page Navigation & Timing**
|
||||
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|----------------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------|
|
||||
| **`wait_until`** | `str` (domcontentloaded)| Condition for navigation to "complete". Often `"networkidle"` or `"domcontentloaded"`. |
|
||||
| **`page_timeout`** | `int` (60000 ms) | Timeout for page navigation or JS steps. Increase for slow sites. |
|
||||
| **`wait_for`** | `str or None` | Wait for a CSS (`"css:selector"`) or JS (`"js:() => bool"`) condition before content extraction. |
|
||||
| **`wait_for_timeout`** | `int or None` (None) | Specific timeout in ms for the `wait_for` condition. If None, uses `page_timeout`. |
|
||||
| **`wait_for_images`** | `bool` (False) | Wait for images to load before finishing. Slows down if you only want text. |
|
||||
| **`delay_before_return_html`** | `float` (0.1) | Additional pause (seconds) before final HTML is captured. Good for last-second updates. |
|
||||
| **`check_robots_txt`** | `bool` (False) | Whether to check and respect robots.txt rules before crawling. If True, caches robots.txt for efficiency. |
|
||||
| **`mean_delay`** and **`max_range`** | `float` (0.1, 0.3) | If you call `arun_many()`, these define random delay intervals between crawls, helping avoid detection or rate limits. |
|
||||
| **`semaphore_count`** | `int` (5) | Max concurrency for `arun_many()`. Increase if you have resources for parallel crawls. |
|
||||
|
||||
---
|
||||
|
||||
### E) **Page Interaction**
|
||||
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|----------------------------|--------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| **`js_code`** | `str or list[str]` (None) | JavaScript to run **after** `wait_for` and `delay_before_return_html`, on the fully-loaded page. E.g. `"document.querySelector('button')?.click();"`. |
|
||||
| **`js_code_before_wait`** | `str or list[str]` (None) | JavaScript to run **before** `wait_for`. Use for triggering loading that `wait_for` then checks (e.g. clicking a tab, then waiting for its content). |
|
||||
| **`c4a_script`** | `str or list[str]` (None) | C4A script that compiles to JavaScript. Alternative to writing raw JS. |
|
||||
| **`js_only`** | `bool` (False) | If `True`, indicates we're reusing an existing session and only applying JS. No full reload. |
|
||||
| **`ignore_body_visibility`** | `bool` (True) | Skip checking if `<body>` is visible. Usually best to keep `True`. |
|
||||
| **`scan_full_page`** | `bool` (False) | If `True`, auto-scroll the page to load dynamic content (infinite scroll). |
|
||||
| **`scroll_delay`** | `float` (0.2) | Delay between scroll steps when scanning the full page (`scan_full_page=True`) or capturing full-page screenshots. |
|
||||
| **`max_scroll_steps`** | `int or None` (None) | Maximum number of scroll steps during full page scan. If None, scrolls until entire page is loaded. |
|
||||
| **`process_iframes`** | `bool` (False) | Inlines iframe content for single-page extraction. |
|
||||
| **`flatten_shadow_dom`** | `bool` (False) | Flattens Shadow DOM content into the light DOM before HTML capture. Resolves slots, strips shadow-scoped styles, and force-opens closed shadow roots. Essential for sites built with Web Components (Stencil, Lit, Shoelace, etc.). |
|
||||
| **`remove_overlay_elements`** | `bool` (False) | Removes potential modals/popups blocking the main content. |
|
||||
| **`remove_consent_popups`** | `bool` (False) | Removes GDPR/cookie consent popups from known CMP providers (OneTrust, Cookiebot, TrustArc, Quantcast, Didomi, Sourcepoint, FundingChoices, etc.). Tries clicking "Accept All" first, then falls back to DOM removal. |
|
||||
| **`simulate_user`** | `bool` (False) | Simulate user interactions (mouse movements) to avoid bot detection. |
|
||||
| **`override_navigator`** | `bool` (False) | Override `navigator` properties in JS for stealth. |
|
||||
| **`magic`** | `bool` (False) | Automatic handling of popups/consent banners. Experimental. |
|
||||
| **`adjust_viewport_to_content`** | `bool` (False) | Resizes viewport to match page content height. |
|
||||
|
||||
If your page is a single-page app with repeated JS updates, set `js_only=True` in subsequent calls, plus a `session_id` for reusing the same tab.
|
||||
|
||||
---
|
||||
|
||||
### F) **Media Handling**
|
||||
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|--------------------------------------------|---------------------|-----------------------------------------------------------------------------------------------------------|
|
||||
| **`screenshot`** | `bool` (False) | Capture a screenshot (base64) in `result.screenshot`. |
|
||||
| **`screenshot_wait_for`** | `float or None` | Extra wait time before the screenshot. |
|
||||
| **`screenshot_height_threshold`** | `int` (~20000) | If the page is taller than this, alternate screenshot strategies are used. |
|
||||
| **`force_viewport_screenshot`** | `bool` (False) | If `True`, always captures a viewport-only screenshot regardless of page height. Faster and smaller than full-page screenshots. |
|
||||
| **`pdf`** | `bool` (False) | If `True`, returns a PDF in `result.pdf`. |
|
||||
| **`capture_mhtml`** | `bool` (False) | If `True`, captures an MHTML snapshot of the page in `result.mhtml`. MHTML includes all page resources (CSS, images, etc.) in a single file. |
|
||||
| **`image_description_min_word_threshold`** | `int` (~50) | Minimum words for an image's alt text or description to be considered valid. |
|
||||
| **`image_score_threshold`** | `int` (~3) | Filter out low-scoring images. The crawler scores images by relevance (size, context, etc.). |
|
||||
| **`exclude_external_images`** | `bool` (False) | Exclude images from other domains. |
|
||||
| **`exclude_all_images`** | `bool` (False) | If `True`, excludes all images from processing (both internal and external). |
|
||||
| **`table_score_threshold`** | `int` (7) | Minimum score threshold for processing a table. Lower values include more tables. |
|
||||
| **`table_extraction`** | `TableExtractionStrategy` (DefaultTableExtraction) | Strategy for table extraction. Defaults to DefaultTableExtraction with configured threshold. |
|
||||
|
||||
---
|
||||
|
||||
### G) **Link/Domain Handling**
|
||||
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|------------------------------|-------------------------|-----------------------------------------------------------------------------------------------------------------------------|
|
||||
| **`exclude_social_media_domains`** | `list` (e.g. Facebook/Twitter) | A default list can be extended. Any link to these domains is removed from final output. |
|
||||
| **`exclude_external_links`** | `bool` (False) | Removes all links pointing outside the current domain. |
|
||||
| **`exclude_social_media_links`** | `bool` (False) | Strips links specifically to social sites (like Facebook or Twitter). |
|
||||
| **`exclude_domains`** | `list` ([]) | Provide a custom list of domains to exclude (like `["ads.com", "trackers.io"]`). |
|
||||
| **`exclude_internal_links`** | `bool` (False) | If `True`, excludes internal links from the results. |
|
||||
| **`score_links`** | `bool` (False) | If `True`, calculates intrinsic quality scores for all links using URL structure, text quality, and contextual metrics. |
|
||||
| **`preserve_https_for_internal_links`** | `bool` (False) | If `True`, preserves HTTPS scheme for internal links even when the server redirects to HTTP. Useful for security-conscious crawling. |
|
||||
|
||||
Use these for link-level content filtering (often to keep crawls “internal” or to remove spammy domains).
|
||||
|
||||
---
|
||||
|
||||
### H) **Debug, Logging & Network Monitoring**
|
||||
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|----------------|--------------------|---------------------------------------------------------------------------|
|
||||
| **`verbose`** | `bool` (True) | Prints logs detailing each step of crawling, interactions, or errors. |
|
||||
| **`log_console`** | `bool` (False) | Logs the page's JavaScript console output if you want deeper JS debugging.|
|
||||
| **`capture_network_requests`** | `bool` (False) | If `True`, captures network requests made by the page in `result.captured_requests`. |
|
||||
| **`capture_console_messages`** | `bool` (False) | If `True`, captures console messages from the page in `result.console_messages`. |
|
||||
|
||||
---
|
||||
|
||||
### I) **Connection & HTTP Parameters**
|
||||
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|-----------------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------|
|
||||
| **`method`** | `str` ("GET") | HTTP method to use when using AsyncHTTPCrawlerStrategy (e.g., "GET", "POST"). |
|
||||
| **`stream`** | `bool` (False) | If `True`, enables streaming mode for `arun_many()` to process URLs as they complete rather than waiting for all. |
|
||||
| **`url`** | `str or None` (None) | URL for this specific config. Not typically set directly but used internally for URL-specific configurations. |
|
||||
| **`user_agent`** | `str or None` (None) | Custom User-Agent string for this crawl. Can override browser-level user agent. |
|
||||
| **`user_agent_mode`** | `str or None` (None) | Set to `"random"` to randomize user agent. Can override browser-level setting. |
|
||||
| **`user_agent_generator_config`** | `dict` ({}) | Configuration for user agent generation when `user_agent_mode="random"`. |
|
||||
|
||||
---
|
||||
|
||||
### J) **Virtual Scroll Configuration**
|
||||
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|------------------------------|------------------------------|-------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| **`virtual_scroll_config`** | `VirtualScrollConfig or dict` (None) | Configuration for handling virtualized scrolling on sites like Twitter/Instagram where content is replaced rather than appended. |
|
||||
|
||||
When sites use virtual scrolling (content replaced as you scroll), use `VirtualScrollConfig`:
|
||||
|
||||
```python
|
||||
from crawl4ai import VirtualScrollConfig
|
||||
|
||||
virtual_config = VirtualScrollConfig(
|
||||
container_selector="#timeline", # CSS selector for scrollable container
|
||||
scroll_count=30, # Number of times to scroll
|
||||
scroll_by="container_height", # How much to scroll: "container_height", "page_height", or pixels (e.g. 500)
|
||||
wait_after_scroll=0.5 # Seconds to wait after each scroll for content to load
|
||||
)
|
||||
|
||||
config = CrawlerRunConfig(
|
||||
virtual_scroll_config=virtual_config
|
||||
)
|
||||
```
|
||||
|
||||
**VirtualScrollConfig Parameters:**
|
||||
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|------------------------|---------------------------|-------------------------------------------------------------------------------------------|
|
||||
| **`container_selector`** | `str` (required) | CSS selector for the scrollable container (e.g., `"#feed"`, `".timeline"`) |
|
||||
| **`scroll_count`** | `int` (10) | Maximum number of scrolls to perform |
|
||||
| **`scroll_by`** | `str or int` ("container_height") | Scroll amount: `"container_height"`, `"page_height"`, or pixels (e.g., `500`) |
|
||||
| **`wait_after_scroll`** | `float` (0.5) | Time in seconds to wait after each scroll for new content to load |
|
||||
|
||||
**When to use Virtual Scroll vs scan_full_page:**
|
||||
- Use `virtual_scroll_config` when content is **replaced** during scroll (Twitter, Instagram)
|
||||
- Use `scan_full_page` when content is **appended** during scroll (traditional infinite scroll)
|
||||
|
||||
See [Virtual Scroll documentation](../../advanced/virtual-scroll.md) for detailed examples.
|
||||
|
||||
---
|
||||
|
||||
### K) **URL Matching Configuration**
|
||||
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|------------------------|------------------------------|-------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| **`url_matcher`** | `UrlMatcher` (None) | Pattern(s) to match URLs against. Can be: string (glob), function, or list of mixed types. **None means match ALL URLs** |
|
||||
| **`match_mode`** | `MatchMode` (MatchMode.OR) | How to combine multiple matchers in a list: `MatchMode.OR` (any match) or `MatchMode.AND` (all must match) |
|
||||
|
||||
The `url_matcher` parameter enables URL-specific configurations when used with `arun_many()`:
|
||||
|
||||
```python
|
||||
from crawl4ai import CrawlerRunConfig, MatchMode
|
||||
from crawl4ai.processors.pdf import PDFContentScrapingStrategy
|
||||
from crawl4ai.extraction_strategy import JsonCssExtractionStrategy
|
||||
|
||||
# Simple string pattern (glob-style)
|
||||
pdf_config = CrawlerRunConfig(
|
||||
url_matcher="*.pdf",
|
||||
scraping_strategy=PDFContentScrapingStrategy()
|
||||
)
|
||||
|
||||
# Multiple patterns with OR logic (default)
|
||||
blog_config = CrawlerRunConfig(
|
||||
url_matcher=["*/blog/*", "*/article/*", "*/news/*"],
|
||||
match_mode=MatchMode.OR # Any pattern matches
|
||||
)
|
||||
|
||||
# Function matcher
|
||||
api_config = CrawlerRunConfig(
|
||||
url_matcher=lambda url: 'api' in url or url.endswith('.json'),
|
||||
# Other settings like extraction_strategy
|
||||
)
|
||||
|
||||
# Mixed: String + Function with AND logic
|
||||
complex_config = CrawlerRunConfig(
|
||||
url_matcher=[
|
||||
lambda url: url.startswith('https://'), # Must be HTTPS
|
||||
"*.org/*", # Must be .org domain
|
||||
lambda url: 'docs' in url # Must contain 'docs'
|
||||
],
|
||||
match_mode=MatchMode.AND # ALL conditions must match
|
||||
)
|
||||
|
||||
# Combined patterns and functions with AND logic
|
||||
secure_docs = CrawlerRunConfig(
|
||||
url_matcher=["https://*", lambda url: '.doc' in url],
|
||||
match_mode=MatchMode.AND # Must be HTTPS AND contain .doc
|
||||
)
|
||||
|
||||
# Default config - matches ALL URLs
|
||||
default_config = CrawlerRunConfig() # No url_matcher = matches everything
|
||||
```
|
||||
|
||||
**UrlMatcher Types:**
|
||||
- **None (default)**: When `url_matcher` is None or not set, the config matches ALL URLs
|
||||
- **String patterns**: Glob-style patterns like `"*.pdf"`, `"*/api/*"`, `"https://*.example.com/*"`
|
||||
- **Functions**: `lambda url: bool` - Custom logic for complex matching
|
||||
- **Lists**: Mix strings and functions, combined with `MatchMode.OR` or `MatchMode.AND`
|
||||
|
||||
**Important Behavior:**
|
||||
- When passing a list of configs to `arun_many()`, URLs are matched against each config's `url_matcher` in order. First match wins!
|
||||
- If no config matches a URL and there's no default config (one without `url_matcher`), the URL will fail with "No matching configuration found"
|
||||
- Always include a default config as the last item if you want to handle all URLs
|
||||
|
||||
---
|
||||
|
||||
### L) **Advanced Crawling Features**
|
||||
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|-----------------------------|------------------------------|-------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| **`deep_crawl_strategy`** | `DeepCrawlStrategy or None` (None) | Strategy for deep/recursive crawling. Enables automatic link following and multi-level site crawling. |
|
||||
| **`link_preview_config`** | `LinkPreviewConfig or dict or None` (None) | Configuration for link head extraction and scoring. Fetches and scores link metadata without full page loads. |
|
||||
| **`experimental`** | `dict or None` (None) | Dictionary for experimental/beta features not yet integrated into main parameters. Use with caution. |
|
||||
|
||||
**Deep Crawl Strategy** enables automatic site exploration by following links according to defined rules. Useful for sitemap generation or comprehensive site archiving.
|
||||
|
||||
**Link Preview Config** allows efficient link discovery and scoring by fetching only the `<head>` section of linked pages, enabling smart crawl prioritization without the overhead of full page loads.
|
||||
|
||||
**Experimental** parameters are features in beta testing. They may change or be removed in future versions. Check documentation for currently available experimental features.
|
||||
|
||||
---
|
||||
|
||||
## 2.2 Helper Methods
|
||||
|
||||
Both `BrowserConfig` and `CrawlerRunConfig` provide a `clone()` method to create modified copies:
|
||||
|
||||
```python
|
||||
# Create a base configuration
|
||||
base_config = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.ENABLED,
|
||||
word_count_threshold=200
|
||||
)
|
||||
|
||||
# Create variations using clone()
|
||||
stream_config = base_config.clone(stream=True)
|
||||
no_cache_config = base_config.clone(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
stream=True
|
||||
)
|
||||
```
|
||||
|
||||
The `clone()` method is particularly useful when you need slightly different configurations for different use cases, without modifying the original config.
|
||||
|
||||
### Class-Level Defaults (`set_defaults` / `get_defaults` / `reset_defaults`)
|
||||
|
||||
Both config classes support class-level default overrides. When deploying in a server or cloud context, this eliminates the need to pass the same parameters at every call site.
|
||||
|
||||
**Resolution order:** explicit arg > class-level default > hardcoded default
|
||||
|
||||
```python
|
||||
from crawl4ai import BrowserConfig, CrawlerRunConfig
|
||||
|
||||
# Set once at application startup
|
||||
BrowserConfig.set_defaults(
|
||||
cache_cdp_connection=True,
|
||||
cdp_close_delay=0,
|
||||
create_isolated_context=True,
|
||||
)
|
||||
CrawlerRunConfig.set_defaults(verbose=False)
|
||||
|
||||
# All new instances inherit the class defaults
|
||||
cfg1 = BrowserConfig(cdp_url="ws://localhost:9222")
|
||||
# → cache_cdp_connection=True, cdp_close_delay=0
|
||||
|
||||
cfg2 = BrowserConfig(cdp_url="ws://localhost:9222", cache_cdp_connection=False)
|
||||
# → cache_cdp_connection=False (explicit value wins)
|
||||
|
||||
# Inspect current defaults
|
||||
BrowserConfig.get_defaults()
|
||||
# → {"cache_cdp_connection": True, "cdp_close_delay": 0, "create_isolated_context": True}
|
||||
|
||||
# Remove a single default
|
||||
BrowserConfig.reset_defaults("cdp_close_delay")
|
||||
|
||||
# Remove all defaults
|
||||
BrowserConfig.reset_defaults()
|
||||
```
|
||||
|
||||
**API Reference:**
|
||||
|
||||
| Method | Signature | Description |
|
||||
|--------|-----------|-------------|
|
||||
| `set_defaults` | `set_defaults(**kwargs)` | Set class-level defaults for new instances. Raises `ValueError` if any key is not a valid `__init__` parameter. |
|
||||
| `get_defaults` | `get_defaults() → dict` | Return a deep copy of the current class-level defaults. |
|
||||
| `reset_defaults` | `reset_defaults(*names)` | With no args, clears all defaults. With args, removes only the named defaults. |
|
||||
|
||||
**Notes:**
|
||||
- Defaults are independent per class — `BrowserConfig.set_defaults()` has no effect on `CrawlerRunConfig`.
|
||||
- Mutable values (lists, dicts) are deep-copied on storage and on each instance creation, so instances do not share objects.
|
||||
- `clone()`, `dump()`/`load()`, and `from_kwargs()` all work correctly with class defaults — serialized data is self-contained and independent of the current class defaults.
|
||||
- Defaults are stored in memory for the lifetime of the process. They are not persisted to disk.
|
||||
|
||||
## 2.3 Example Usage
|
||||
|
||||
```python
|
||||
import asyncio
|
||||
from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode
|
||||
|
||||
async def main():
|
||||
# Configure the browser
|
||||
browser_cfg = BrowserConfig(
|
||||
headless=False,
|
||||
viewport_width=1280,
|
||||
viewport_height=720,
|
||||
proxy_config="http://user:pass@myproxy:8080",
|
||||
text_mode=True
|
||||
)
|
||||
|
||||
# Configure the run
|
||||
run_cfg = CrawlerRunConfig(
|
||||
cache_mode=CacheMode.BYPASS,
|
||||
session_id="my_session",
|
||||
css_selector="main.article",
|
||||
excluded_tags=["script", "style"],
|
||||
exclude_external_links=True,
|
||||
wait_for="css:.article-loaded",
|
||||
screenshot=True,
|
||||
stream=True
|
||||
)
|
||||
|
||||
async with AsyncWebCrawler(config=browser_cfg) as crawler:
|
||||
result = await crawler.arun(
|
||||
url="https://example.com/news",
|
||||
config=run_cfg
|
||||
)
|
||||
if result.success:
|
||||
print("Final cleaned_html length:", len(result.cleaned_html))
|
||||
if result.screenshot:
|
||||
print("Screenshot captured (base64, length):", len(result.screenshot))
|
||||
else:
|
||||
print("Crawl failed:", result.error_message)
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
```
|
||||
|
||||
## 2.4 Compliance & Ethics
|
||||
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|-----------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------|
|
||||
| **`check_robots_txt`**| `bool` (False) | When True, checks and respects robots.txt rules before crawling. Uses efficient caching with SQLite backend. |
|
||||
| **`user_agent`** | `str` (None) | User agent string to identify your crawler. Used for robots.txt checking when enabled. |
|
||||
|
||||
```python
|
||||
run_config = CrawlerRunConfig(
|
||||
check_robots_txt=True, # Enable robots.txt compliance
|
||||
user_agent="MyBot/1.0" # Identify your crawler
|
||||
)
|
||||
```
|
||||
|
||||
# 3. **LLMConfig** - Setting up LLM providers
|
||||
LLMConfig is useful to pass LLM provider config to strategies and functions that rely on LLMs to do extraction, filtering, schema generation etc. Currently it can be used in the following -
|
||||
|
||||
1. LLMExtractionStrategy
|
||||
2. LLMContentFilter
|
||||
3. JsonCssExtractionStrategy.generate_schema
|
||||
4. JsonXPathExtractionStrategy.generate_schema
|
||||
5. AdaptiveConfig.embedding_llm_config (embedding model for adaptive crawling)
|
||||
6. AdaptiveConfig.query_llm_config (chat completion model for query expansion in adaptive crawling)
|
||||
|
||||
## 3.1 Parameters
|
||||
| **Parameter** | **Type / Default** | **What It Does** |
|
||||
|-----------------------|----------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|
|
||||
| **`provider`** | `"ollama/llama3","groq/llama3-70b-8192","groq/llama3-8b-8192", "openai/gpt-4o-mini" ,"openai/gpt-4o","openai/o1-mini","openai/o1-preview","openai/o3-mini","openai/o3-mini-high","anthropic/claude-3-haiku-20240307","anthropic/claude-3-opus-20240229","anthropic/claude-3-sonnet-20240229","anthropic/claude-3-5-sonnet-20240620","gemini/gemini-pro","gemini/gemini-1.5-pro","gemini/gemini-2.0-flash","gemini/gemini-2.0-flash-exp","gemini/gemini-2.0-flash-lite-preview-02-05","deepseek/deepseek-chat"`<br/>*(default: `"openai/gpt-4o-mini"`)* | Which LLM provider to use.
|
||||
| **`api_token`** |1.Optional. When not provided explicitly, api_token will be read from environment variables based on provider. For example: If a gemini model is passed as provider then,`"GEMINI_API_KEY"` will be read from environment variables <br/> 2. API token of LLM provider <br/> eg: `api_token = "gsk_1ClHGGJ7Lpn4WGybR7vNWGdyb3FY7zXEw3SCiy0BAVM9lL8CQv"` <br/> 3. Environment variable - use with prefix "env:" <br/> eg:`api_token = "env: GROQ_API_KEY"` | API token to use for the given provider
|
||||
| **`base_url`** |Optional. Custom API endpoint | If your provider has a custom endpoint
|
||||
| **`backoff_base_delay`** |Optional. `int` *(default: `2`)* | Seconds to wait before the first retry when the provider throttles a request.
|
||||
| **`backoff_max_attempts`** |Optional. `int` *(default: `3`)* | Total tries (initial call + retries) before surfacing an error.
|
||||
| **`backoff_exponential_factor`** |Optional. `int` *(default: `2`)* | Multiplier that increases the wait time for each retry (`delay = base_delay * factor^attempt`).
|
||||
|
||||
## 3.2 Example Usage
|
||||
```python
|
||||
llm_config = LLMConfig(
|
||||
provider="openai/gpt-4o-mini",
|
||||
api_token=os.getenv("OPENAI_API_KEY"),
|
||||
backoff_base_delay=1, # optional
|
||||
backoff_max_attempts=5, # optional
|
||||
backoff_exponential_factor=3, # optional
|
||||
)
|
||||
```
|
||||
|
||||
## 4. Putting It All Together
|
||||
|
||||
- **Use** `BrowserConfig` for **global** browser settings: engine, headless, proxy, user agent.
|
||||
- **Use** `CrawlerRunConfig` for each crawl’s **context**: how to filter content, handle caching, wait for dynamic elements, or run JS.
|
||||
- **Pass** both configs to `AsyncWebCrawler` (the `BrowserConfig`) and then to `arun()` (the `CrawlerRunConfig`).
|
||||
- **Use** `LLMConfig` for LLM provider configurations that can be used across all extraction, filtering, schema generation, and adaptive crawling tasks. Can be used in - `LLMExtractionStrategy`, `LLMContentFilter`, `JsonCssExtractionStrategy.generate_schema`, `JsonXPathExtractionStrategy.generate_schema`, and `AdaptiveConfig` (`embedding_llm_config` / `query_llm_config`)
|
||||
|
||||
```python
|
||||
# Create a modified copy with the clone() method
|
||||
stream_cfg = run_cfg.clone(
|
||||
stream=True,
|
||||
cache_mode=CacheMode.BYPASS
|
||||
)
|
||||
|
||||
# Or set project-wide defaults once at startup
|
||||
BrowserConfig.set_defaults(headless=True, text_mode=True)
|
||||
CrawlerRunConfig.set_defaults(cache_mode=CacheMode.BYPASS)
|
||||
```
|
||||
@@ -0,0 +1,401 @@
|
||||
# Extraction & Chunking Strategies API
|
||||
|
||||
This documentation covers the API reference for extraction and chunking strategies in Crawl4AI.
|
||||
|
||||
## Extraction Strategies
|
||||
|
||||
All extraction strategies inherit from the base `ExtractionStrategy` class and implement two key methods:
|
||||
- `extract(url: str, html: str) -> List[Dict[str, Any]]`
|
||||
- `run(url: str, sections: List[str]) -> List[Dict[str, Any]]`
|
||||
|
||||
### LLMExtractionStrategy
|
||||
|
||||
Used for extracting structured data using Language Models.
|
||||
|
||||
```python
|
||||
LLMExtractionStrategy(
|
||||
# Required Parameters
|
||||
provider: str = DEFAULT_PROVIDER, # LLM provider (e.g., "ollama/llama2")
|
||||
api_token: Optional[str] = None, # API token
|
||||
|
||||
# Extraction Configuration
|
||||
instruction: str = None, # Custom extraction instruction
|
||||
schema: Dict = None, # Pydantic model schema for structured data
|
||||
extraction_type: str = "block", # "block" or "schema"
|
||||
|
||||
# Chunking Parameters
|
||||
chunk_token_threshold: int = 4000, # Maximum tokens per chunk
|
||||
overlap_rate: float = 0.1, # Overlap between chunks
|
||||
word_token_rate: float = 0.75, # Word to token conversion rate
|
||||
apply_chunking: bool = True, # Enable/disable chunking
|
||||
|
||||
# API Configuration
|
||||
base_url: str = None, # Base URL for API
|
||||
extra_args: Dict = {}, # Additional provider arguments
|
||||
verbose: bool = False # Enable verbose logging
|
||||
)
|
||||
```
|
||||
|
||||
### RegexExtractionStrategy
|
||||
|
||||
Used for fast pattern-based extraction of common entities using regular expressions.
|
||||
|
||||
```python
|
||||
RegexExtractionStrategy(
|
||||
# Pattern Configuration
|
||||
pattern: IntFlag = RegexExtractionStrategy.Nothing, # Bit flags of built-in patterns to use
|
||||
custom: Optional[Dict[str, str]] = None, # Custom pattern dictionary {label: regex}
|
||||
|
||||
# Input Format
|
||||
input_format: str = "fit_html", # "html", "markdown", "text" or "fit_html"
|
||||
)
|
||||
|
||||
# Built-in Patterns as Bit Flags
|
||||
RegexExtractionStrategy.Email # Email addresses
|
||||
RegexExtractionStrategy.PhoneIntl # International phone numbers
|
||||
RegexExtractionStrategy.PhoneUS # US-format phone numbers
|
||||
RegexExtractionStrategy.Url # HTTP/HTTPS URLs
|
||||
RegexExtractionStrategy.IPv4 # IPv4 addresses
|
||||
RegexExtractionStrategy.IPv6 # IPv6 addresses
|
||||
RegexExtractionStrategy.Uuid # UUIDs
|
||||
RegexExtractionStrategy.Currency # Currency values (USD, EUR, etc)
|
||||
RegexExtractionStrategy.Percentage # Percentage values
|
||||
RegexExtractionStrategy.Number # Numeric values
|
||||
RegexExtractionStrategy.DateIso # ISO format dates
|
||||
RegexExtractionStrategy.DateUS # US format dates
|
||||
RegexExtractionStrategy.Time24h # 24-hour format times
|
||||
RegexExtractionStrategy.PostalUS # US postal codes
|
||||
RegexExtractionStrategy.PostalUK # UK postal codes
|
||||
RegexExtractionStrategy.HexColor # HTML hex color codes
|
||||
RegexExtractionStrategy.TwitterHandle # Twitter handles
|
||||
RegexExtractionStrategy.Hashtag # Hashtags
|
||||
RegexExtractionStrategy.MacAddr # MAC addresses
|
||||
RegexExtractionStrategy.Iban # International bank account numbers
|
||||
RegexExtractionStrategy.CreditCard # Credit card numbers
|
||||
RegexExtractionStrategy.All # All available patterns
|
||||
```
|
||||
|
||||
### CosineStrategy
|
||||
|
||||
Used for content similarity-based extraction and clustering.
|
||||
|
||||
```python
|
||||
CosineStrategy(
|
||||
# Content Filtering
|
||||
semantic_filter: str = None, # Topic/keyword filter
|
||||
word_count_threshold: int = 10, # Minimum words per cluster
|
||||
sim_threshold: float = 0.3, # Similarity threshold
|
||||
|
||||
# Clustering Parameters
|
||||
max_dist: float = 0.2, # Maximum cluster distance
|
||||
linkage_method: str = 'ward', # Clustering method
|
||||
top_k: int = 3, # Top clusters to return
|
||||
|
||||
# Model Configuration
|
||||
model_name: str = 'sentence-transformers/all-MiniLM-L6-v2', # Embedding model
|
||||
|
||||
verbose: bool = False # Enable verbose logging
|
||||
)
|
||||
```
|
||||
|
||||
### JsonCssExtractionStrategy
|
||||
|
||||
Used for CSS selector-based structured data extraction.
|
||||
|
||||
```python
|
||||
JsonCssExtractionStrategy(
|
||||
schema: Dict[str, Any], # Extraction schema
|
||||
verbose: bool = False # Enable verbose logging
|
||||
)
|
||||
|
||||
# Schema Structure
|
||||
schema = {
|
||||
"name": str, # Schema name
|
||||
"baseSelector": str, # Base CSS selector
|
||||
"fields": [ # List of fields to extract
|
||||
{
|
||||
"name": str, # Field name
|
||||
"selector": str, # CSS selector
|
||||
"type": str, # Field type: "text", "attribute", "html", "regex"
|
||||
"attribute": str, # For type="attribute"
|
||||
"pattern": str, # For type="regex"
|
||||
"transform": str, # Optional: "lowercase", "uppercase", "strip"
|
||||
"default": Any, # Default value if extraction fails
|
||||
"source": str, # Optional: navigate to sibling first, e.g. "+ tr"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
## Chunking Strategies
|
||||
|
||||
All chunking strategies inherit from `ChunkingStrategy` and implement the `chunk(text: str) -> list` method.
|
||||
|
||||
### RegexChunking
|
||||
|
||||
Splits text based on regex patterns.
|
||||
|
||||
```python
|
||||
RegexChunking(
|
||||
patterns: List[str] = None # Regex patterns for splitting
|
||||
# Default: [r'\n\n']
|
||||
)
|
||||
```
|
||||
|
||||
### SlidingWindowChunking
|
||||
|
||||
Creates overlapping chunks with a sliding window approach.
|
||||
|
||||
```python
|
||||
SlidingWindowChunking(
|
||||
window_size: int = 100, # Window size in words
|
||||
step: int = 50 # Step size between windows
|
||||
)
|
||||
```
|
||||
|
||||
### OverlappingWindowChunking
|
||||
|
||||
Creates chunks with specified overlap.
|
||||
|
||||
```python
|
||||
OverlappingWindowChunking(
|
||||
window_size: int = 1000, # Chunk size in words
|
||||
overlap: int = 100 # Overlap size in words
|
||||
)
|
||||
```
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### LLM Extraction
|
||||
|
||||
```python
|
||||
from pydantic import BaseModel
|
||||
from crawl4ai import LLMExtractionStrategy
|
||||
from crawl4ai import LLMConfig
|
||||
|
||||
# Define schema
|
||||
class Article(BaseModel):
|
||||
title: str
|
||||
content: str
|
||||
author: str
|
||||
|
||||
# Create strategy
|
||||
strategy = LLMExtractionStrategy(
|
||||
llm_config = LLMConfig(provider="ollama/llama2"),
|
||||
schema=Article.schema(),
|
||||
instruction="Extract article details"
|
||||
)
|
||||
|
||||
# Use with crawler
|
||||
result = await crawler.arun(
|
||||
url="https://example.com/article",
|
||||
extraction_strategy=strategy
|
||||
)
|
||||
|
||||
# Access extracted data
|
||||
data = json.loads(result.extracted_content)
|
||||
```
|
||||
|
||||
### Regex Extraction
|
||||
|
||||
```python
|
||||
import json
|
||||
from crawl4ai import AsyncWebCrawler, CrawlerRunConfig, RegexExtractionStrategy
|
||||
|
||||
# Method 1: Use built-in patterns
|
||||
strategy = RegexExtractionStrategy(
|
||||
pattern = RegexExtractionStrategy.Email | RegexExtractionStrategy.Url
|
||||
)
|
||||
|
||||
# Method 2: Use custom patterns
|
||||
price_pattern = {"usd_price": r"\$\s?\d{1,3}(?:,\d{3})*(?:\.\d{2})?"}
|
||||
strategy = RegexExtractionStrategy(custom=price_pattern)
|
||||
|
||||
# Method 3: Generate pattern with LLM assistance (one-time)
|
||||
from crawl4ai import LLMConfig
|
||||
|
||||
async with AsyncWebCrawler() as crawler:
|
||||
# Get sample HTML first
|
||||
sample_result = await crawler.arun("https://example.com/products")
|
||||
html = sample_result.markdown.fit_html
|
||||
|
||||
# Generate regex pattern once
|
||||
pattern = RegexExtractionStrategy.generate_pattern(
|
||||
label="price",
|
||||
html=html,
|
||||
query="Product prices in USD format",
|
||||
llm_config=LLMConfig(provider="openai/gpt-4o-mini")
|
||||
)
|
||||
|
||||
# Save pattern for reuse
|
||||
import json
|
||||
with open("price_pattern.json", "w") as f:
|
||||
json.dump(pattern, f)
|
||||
|
||||
# Use pattern for extraction (no LLM calls)
|
||||
strategy = RegexExtractionStrategy(custom=pattern)
|
||||
result = await crawler.arun(
|
||||
url="https://example.com/products",
|
||||
config=CrawlerRunConfig(extraction_strategy=strategy)
|
||||
)
|
||||
|
||||
# Process results
|
||||
data = json.loads(result.extracted_content)
|
||||
for item in data:
|
||||
print(f"{item['label']}: {item['value']}")
|
||||
```
|
||||
|
||||
### CSS Extraction
|
||||
|
||||
```python
|
||||
from crawl4ai import JsonCssExtractionStrategy
|
||||
|
||||
# Define schema
|
||||
schema = {
|
||||
"name": "Product List",
|
||||
"baseSelector": ".product-card",
|
||||
"fields": [
|
||||
{
|
||||
"name": "title",
|
||||
"selector": "h2.title",
|
||||
"type": "text"
|
||||
},
|
||||
{
|
||||
"name": "price",
|
||||
"selector": ".price",
|
||||
"type": "text",
|
||||
"transform": "strip"
|
||||
},
|
||||
{
|
||||
"name": "image",
|
||||
"selector": "img",
|
||||
"type": "attribute",
|
||||
"attribute": "src"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
# Create and use strategy
|
||||
strategy = JsonCssExtractionStrategy(schema)
|
||||
result = await crawler.arun(
|
||||
url="https://example.com/products",
|
||||
extraction_strategy=strategy
|
||||
)
|
||||
```
|
||||
|
||||
### Content Chunking
|
||||
|
||||
```python
|
||||
from crawl4ai.chunking_strategy import OverlappingWindowChunking
|
||||
from crawl4ai import LLMConfig
|
||||
|
||||
# Create chunking strategy
|
||||
chunker = OverlappingWindowChunking(
|
||||
window_size=500, # 500 words per chunk
|
||||
overlap=50 # 50 words overlap
|
||||
)
|
||||
|
||||
# Use with extraction strategy
|
||||
strategy = LLMExtractionStrategy(
|
||||
llm_config = LLMConfig(provider="ollama/llama2"),
|
||||
chunking_strategy=chunker
|
||||
)
|
||||
|
||||
result = await crawler.arun(
|
||||
url="https://example.com/long-article",
|
||||
extraction_strategy=strategy
|
||||
)
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Choose the Right Strategy**
|
||||
- Use `RegexExtractionStrategy` for common data types like emails, phones, URLs, dates
|
||||
- Use `JsonCssExtractionStrategy` for well-structured HTML with consistent patterns
|
||||
- Use `LLMExtractionStrategy` for complex, unstructured content requiring reasoning
|
||||
- Use `CosineStrategy` for content similarity and clustering
|
||||
|
||||
2. **Strategy Selection Guide**
|
||||
```
|
||||
Is the target data a common type (email/phone/date/URL)?
|
||||
→ RegexExtractionStrategy
|
||||
|
||||
Does the page have consistent HTML structure?
|
||||
→ JsonCssExtractionStrategy or JsonXPathExtractionStrategy
|
||||
|
||||
Is the data semantically complex or unstructured?
|
||||
→ LLMExtractionStrategy
|
||||
|
||||
Need to find content similar to a specific topic?
|
||||
→ CosineStrategy
|
||||
```
|
||||
|
||||
3. **Optimize Chunking**
|
||||
```python
|
||||
# For long documents
|
||||
strategy = LLMExtractionStrategy(
|
||||
chunk_token_threshold=2000, # Smaller chunks
|
||||
overlap_rate=0.1 # 10% overlap
|
||||
)
|
||||
```
|
||||
|
||||
4. **Combine Strategies for Best Performance**
|
||||
```python
|
||||
# First pass: Extract structure with CSS
|
||||
css_strategy = JsonCssExtractionStrategy(product_schema)
|
||||
css_result = await crawler.arun(url, config=CrawlerRunConfig(extraction_strategy=css_strategy))
|
||||
product_data = json.loads(css_result.extracted_content)
|
||||
|
||||
# Second pass: Extract specific fields with regex
|
||||
descriptions = [product["description"] for product in product_data]
|
||||
regex_strategy = RegexExtractionStrategy(
|
||||
pattern=RegexExtractionStrategy.Email | RegexExtractionStrategy.PhoneUS,
|
||||
custom={"dimension": r"\d+x\d+x\d+ (?:cm|in)"}
|
||||
)
|
||||
|
||||
# Process descriptions with regex
|
||||
for text in descriptions:
|
||||
matches = regex_strategy.extract("", text) # Direct extraction
|
||||
```
|
||||
|
||||
5. **Handle Errors**
|
||||
```python
|
||||
try:
|
||||
result = await crawler.arun(
|
||||
url="https://example.com",
|
||||
extraction_strategy=strategy
|
||||
)
|
||||
if result.success:
|
||||
content = json.loads(result.extracted_content)
|
||||
except Exception as e:
|
||||
print(f"Extraction failed: {e}")
|
||||
```
|
||||
|
||||
6. **Monitor Performance**
|
||||
```python
|
||||
strategy = CosineStrategy(
|
||||
verbose=True, # Enable logging
|
||||
word_count_threshold=20, # Filter short content
|
||||
top_k=5 # Limit results
|
||||
)
|
||||
```
|
||||
|
||||
7. **Cache Generated Patterns**
|
||||
```python
|
||||
# For RegexExtractionStrategy pattern generation
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
cache_dir = Path("./pattern_cache")
|
||||
cache_dir.mkdir(exist_ok=True)
|
||||
pattern_file = cache_dir / "product_pattern.json"
|
||||
|
||||
if pattern_file.exists():
|
||||
with open(pattern_file) as f:
|
||||
pattern = json.load(f)
|
||||
else:
|
||||
# Generate once with LLM
|
||||
pattern = RegexExtractionStrategy.generate_pattern(...)
|
||||
with open(pattern_file, "w") as f:
|
||||
json.dump(pattern, f)
|
||||
```
|
||||
Reference in New Issue
Block a user