diff --git a/app/tool/__init__.py b/app/tool/__init__.py index b1d25e2..df15a0a 100644 --- a/app/tool/__init__.py +++ b/app/tool/__init__.py @@ -7,6 +7,7 @@ from app.tool.str_replace_editor import StrReplaceEditor from app.tool.terminate import Terminate from app.tool.tool_collection import ToolCollection from app.tool.web_search import WebSearch +from app.tool.crawl4ai import Crawl4aiTool __all__ = [ @@ -19,4 +20,5 @@ __all__ = [ "ToolCollection", "CreateChatCompletion", "PlanningTool", + "Crawl4aiTool" ] diff --git a/app/tool/crawl4ai.py b/app/tool/crawl4ai.py new file mode 100644 index 0000000..7198749 --- /dev/null +++ b/app/tool/crawl4ai.py @@ -0,0 +1,249 @@ + +""" +Crawl4AI Web Crawler Tool for OpenManus + +This tool integrates Crawl4AI, a high-performance web crawler designed for LLMs and AI agents, +providing fast, precise, and AI-ready data extraction with clean Markdown generation. +""" + +import asyncio +from typing import Union, List +from urllib.parse import urlparse + +from app.logger import logger +from app.tool.base import BaseTool, ToolResult + + +class Crawl4aiTool(BaseTool): + """ + Web crawler tool powered by Crawl4AI. + + Provides clean markdown extraction optimized for AI processing. + """ + + name: str = "crawl4ai" + description: str = """Web crawler that extracts clean, AI-ready content from web pages. + + Features: + - Extracts clean markdown content optimized for LLMs + - Handles JavaScript-heavy sites and dynamic content + - Supports multiple URLs in a single request + - Fast and reliable with built-in error handling + + Perfect for content analysis, research, and feeding web content to AI models.""" + + parameters: dict = { + "type": "object", + "properties": { + "urls": { + "type": "array", + "items": {"type": "string"}, + "description": "(required) List of URLs to crawl. Can be a single URL or multiple URLs.", + "minItems": 1 + }, + "timeout": { + "type": "integer", + "description": "(optional) Timeout in seconds for each URL. Default is 30.", + "default": 30, + "minimum": 5, + "maximum": 120 + }, + "bypass_cache": { + "type": "boolean", + "description": "(optional) Whether to bypass cache and fetch fresh content. Default is false.", + "default": False + }, + "word_count_threshold": { + "type": "integer", + "description": "(optional) Minimum word count for content blocks. Default is 10.", + "default": 10, + "minimum": 1 + } + }, + "required": ["urls"] + } + + async def execute( + self, + urls: Union[str, List[str]], + timeout: int = 30, + bypass_cache: bool = False, + word_count_threshold: int = 10, + ) -> ToolResult: + """ + Execute web crawling for the specified URLs. + + Args: + urls: Single URL string or list of URLs to crawl + timeout: Timeout in seconds for each URL + bypass_cache: Whether to bypass cache + word_count_threshold: Minimum word count for content blocks + + Returns: + ToolResult with crawl results + """ + # Normalize URLs to list + if isinstance(urls, str): + url_list = [urls] + else: + url_list = urls + + # Validate URLs + valid_urls = [] + for url in url_list: + if self._is_valid_url(url): + valid_urls.append(url) + else: + logger.warning(f"Invalid URL skipped: {url}") + + if not valid_urls: + return ToolResult(error="No valid URLs provided") + + try: + # Import crawl4ai components + from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CacheMode + + # Configure browser settings + browser_config = BrowserConfig( + headless=True, + verbose=False, + browser_type="chromium", + ignore_https_errors=True, + java_script_enabled=True + ) + + # Configure crawler settings + run_config = CrawlerRunConfig( + cache_mode=CacheMode.BYPASS if bypass_cache else CacheMode.ENABLED, + word_count_threshold=word_count_threshold, + process_iframes=True, + remove_overlay_elements=True, + excluded_tags=['script', 'style'], + page_timeout=timeout * 1000, # Convert to milliseconds + verbose=False, + wait_until="domcontentloaded" + ) + + results = [] + successful_count = 0 + failed_count = 0 + + # Process each URL + async with AsyncWebCrawler(config=browser_config) as crawler: + for url in valid_urls: + try: + logger.info(f"🕷️ Crawling URL: {url}") + start_time = asyncio.get_event_loop().time() + + result = await crawler.arun(url=url, config=run_config) + + end_time = asyncio.get_event_loop().time() + execution_time = end_time - start_time + + if result.success: + # Count words in markdown + word_count = 0 + if hasattr(result, 'markdown') and result.markdown: + word_count = len(result.markdown.split()) + + # Count links + links_count = 0 + if hasattr(result, 'links') and result.links: + internal_links = result.links.get('internal', []) + external_links = result.links.get('external', []) + links_count = len(internal_links) + len(external_links) + + # Count images + images_count = 0 + if hasattr(result, 'media') and result.media: + images = result.media.get('images', []) + images_count = len(images) + + results.append({ + 'url': url, + 'success': True, + 'status_code': getattr(result, 'status_code', 200), + 'title': result.metadata.get('title') if result.metadata else None, + 'markdown': result.markdown if hasattr(result, 'markdown') else None, + 'word_count': word_count, + 'links_count': links_count, + 'images_count': images_count, + 'execution_time': execution_time + }) + successful_count += 1 + logger.info(f"✅ Successfully crawled {url} in {execution_time:.2f}s") + + else: + results.append({ + 'url': url, + 'success': False, + 'error_message': getattr(result, 'error_message', 'Unknown error'), + 'execution_time': execution_time + }) + failed_count += 1 + logger.warning(f"❌ Failed to crawl {url}") + + except Exception as e: + error_msg = f"Error crawling {url}: {str(e)}" + logger.error(error_msg) + results.append({ + 'url': url, + 'success': False, + 'error_message': error_msg + }) + failed_count += 1 + + # Format output + output_lines = [f"🕷️ Crawl4AI Results Summary:"] + output_lines.append(f"📊 Total URLs: {len(valid_urls)}") + output_lines.append(f"✅ Successful: {successful_count}") + output_lines.append(f"❌ Failed: {failed_count}") + output_lines.append("") + + for i, result in enumerate(results, 1): + output_lines.append(f"{i}. {result['url']}") + + if result['success']: + output_lines.append(f" ✅ Status: Success (HTTP {result.get('status_code', 'N/A')})") + if result.get('title'): + output_lines.append(f" 📄 Title: {result['title']}") + + if result.get('markdown'): + # Show first 300 characters of markdown content + content_preview = result['markdown'] + if len(result['markdown']) > 300: + content_preview += "..." + output_lines.append(f" 📝 Content: {content_preview}") + + output_lines.append(f" 📊 Stats: {result.get('word_count', 0)} words, {result.get('links_count', 0)} links, {result.get('images_count', 0)} images") + + if result.get('execution_time'): + output_lines.append(f" ⏱️ Time: {result['execution_time']:.2f}s") + else: + output_lines.append(f" ❌ Status: Failed") + if result.get('error_message'): + output_lines.append(f" 🚫 Error: {result['error_message']}") + + output_lines.append("") + + return ToolResult(output="\n".join(output_lines)) + + except ImportError as e: + error_msg = "Crawl4AI is not installed. Please install it with: pip install crawl4ai" + logger.error(error_msg) + return ToolResult(error=error_msg) + except Exception as e: + error_msg = f"Crawl4AI execution failed: {str(e)}" + logger.error(error_msg) + return ToolResult(error=error_msg) + + def _is_valid_url(self, url: str) -> bool: + """Validate if a URL is properly formatted.""" + try: + result = urlparse(url) + return all([result.scheme, result.netloc]) and result.scheme in ['http', 'https'] + except Exception: + return False + + + diff --git a/requirements.txt b/requirements.txt index 3324283..aa7e6dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,6 +36,7 @@ boto3~=1.37.18 requests~=2.32.3 beautifulsoup4~=4.13.3 +crawl4ai~=0.6.3 huggingface-hub~=0.29.2 setuptools~=75.8.0